zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2026-07-19 14:30:37 +00:00
Code Activity

fix: csv stream special cases (#902)

Browse Source 
* chore: add tests for csv stream

* fix: csv values must have quotes when they contain a new line

* fix: csv reader did not properly handle special cases
This commit is contained in:
Jan Laupetin
2026-07-12 18:00:46 +02:00
committed by GitHub
parent e45272137f
commit a87850d36e
2 changed files with 260 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ObjCommon/Csv/CsvStream.cpp
+66 -17
View File
@@ -5,7 +5,10 @@
#include <cstdlib>
#include <sstream>
constexpr char CSV_SEPARATOR = ',';
namespace
{
constexpr char CSV_SEPARATOR = ',';
}
CsvCell::CsvCell(std::string value)
: m_value(std::move(value))
@@ -110,12 +113,44 @@ bool CsvInputStream::NextRow(std::vector<const char*>& out, MemoryManager& memor
bool CsvInputStream::EmitNextRow(const std::function<void(std::string)>& cb) const
{
auto c = m_stream.get();
const auto isEof = c == EOF;
const auto startedWithEof = c == EOF;
auto inQuotes = c == '"';
std::ostringstream col;
auto content = false;
if (inQuotes)
c = m_stream.get();
while (c != EOF)
{
if (c == CSV_SEPARATOR)
if (inQuotes)
{
if (c == '"')
{
c = m_stream.peek();
if (c == '"')
{
m_stream.get();
content = true;
col << '"';
}
else
{
inQuotes = false;
}
}
else if (isspace(c))
{
if (content)
col << static_cast<char>(c);
}
else
{
content = true;
col << static_cast<char>(c);
}
}
else if (c == CSV_SEPARATOR)
{
auto value = col.str();
utils::StringTrimR(value);
@@ -123,6 +158,13 @@ bool CsvInputStream::EmitNextRow(const std::function<void(std::string)>& cb) con
col.clear();
col.str(std::string());
content = false;
c = m_stream.peek();
if (c == '"')
{
m_stream.get();
inQuotes = true;
}
}
else if (c == '\r')
{
@@ -149,16 +191,32 @@ bool CsvInputStream::EmitNextRow(const std::function<void(std::string)>& cb) con
c = m_stream.get();
}
if (!isEof)
if (!startedWithEof)
{
auto value = col.str();
utils::StringTrimR(value);
cb(std::move(value));
}
return !isEof;
return !startedWithEof;
}
namespace
{
void InspectCsvValue(const std::string& value, bool& outContainsSeparator, bool& outContainsQuote, bool& outContainsNewLine)
{
for (const auto& c : value)
{
if (c == '"')
outContainsQuote = true;
else if (c == CSV_SEPARATOR)
outContainsSeparator = true;
else if (c == '\n')
outContainsNewLine = true;
}
}
} // namespace
CsvOutputStream::CsvOutputStream(std::ostream& stream)
: m_stream(stream),
m_column_count(0),
@@ -174,17 +232,8 @@ void CsvOutputStream::WriteColumn(const std::string& value)
auto containsSeparator = false;
auto containsQuote = false;
for (const auto& c : value)
{
if (c == '"')
{
containsQuote = true;
break;
}
if (c == CSV_SEPARATOR)
containsSeparator = true;
}
auto containsNewLine = false;
InspectCsvValue(value, containsSeparator, containsQuote, containsNewLine);
if (containsQuote)
{
@@ -199,7 +248,7 @@ void CsvOutputStream::WriteColumn(const std::string& value)
m_stream << "\"";
}
else if (containsSeparator)
else if (containsSeparator || containsNewLine)
{
m_stream << "\"" << value << "\"";
}
test/ObjCommonTests/Csv/CsvStreamTests.cpp
+194
View File
@@ -0,0 +1,194 @@
#include "Csv/CsvStream.h"
#include <catch2/catch_test_macros.hpp>
#include <catch2/generators/catch_generators.hpp>
#include <sstream>
namespace
{
#define NEW_LINE "\n"
TEST_CASE("CsvOutputStream", "[csv]")
{
std::ostringstream ss;
CsvOutputStream outputStream(ss);
SECTION("Ensure can write normal single-line csv")
{
outputStream.WriteColumn("foo");
outputStream.WriteColumn("bar");
REQUIRE(ss.str() == "foo,bar");
}
SECTION("Ensure can write normal multi-line csv")
{
outputStream.WriteColumn("one");
outputStream.WriteColumn("two");
outputStream.NextRow();
outputStream.WriteColumn("foo");
outputStream.WriteColumn("bar");
REQUIRE(ss.str() == ("one,two" NEW_LINE "foo,bar"));
}
SECTION("Ensure can write csv value with comma")
{
outputStream.WriteColumn("first,value");
outputStream.WriteColumn("two");
outputStream.NextRow();
outputStream.WriteColumn("foo");
outputStream.WriteColumn("second,value");
outputStream.NextRow();
outputStream.WriteColumn("crazy");
outputStream.WriteColumn("comma,value");
outputStream.WriteColumn("omg");
REQUIRE(ss.str() == ("\"first,value\",two" NEW_LINE "foo,\"second,value\"" NEW_LINE "crazy,\"comma,value\",omg"));
}
SECTION("Ensure can chain multiple values with commas")
{
outputStream.WriteColumn("first,value");
outputStream.WriteColumn("second,value");
outputStream.WriteColumn("third,value");
REQUIRE(ss.str() == R"("first,value","second,value","third,value")");
}
SECTION("Ensure can write csv value with quotes")
{
outputStream.WriteColumn("one");
outputStream.WriteColumn("two\"value");
outputStream.WriteColumn("three");
REQUIRE(ss.str() == R"(one,"two""value",three)");
}
SECTION("Ensure can write csv value with newlines")
{
outputStream.WriteColumn("one");
outputStream.WriteColumn("two\nvalue");
outputStream.WriteColumn("three");
REQUIRE(ss.str() == ("one,\"two\nvalue\",three"));
}
SECTION("Ensure can write csv value with all special cases")
{
outputStream.WriteColumn("one");
outputStream.WriteColumn("two\nvalue,with\"alot\nof,special cases");
outputStream.WriteColumn("three");
REQUIRE(ss.str() == ("one,\"two\nvalue,with\"\"alot\nof,special cases\",three"));
}
}
TEST_CASE("CsvInputStream", "[csv]")
{
SECTION("Ensure can write normal single-line csv")
{
std::istringstream ss("foo,bar");
CsvInputStream inputStream(ss);
std::vector<std::string> row;
inputStream.NextRow(row);
REQUIRE(row.size() == 2);
REQUIRE(row[0] == "foo");
REQUIRE(row[1] == "bar");
}
SECTION("Ensure can write normal multi-line csv")
{
std::istringstream ss("one,two" NEW_LINE "foo,bar");
CsvInputStream inputStream(ss);
std::vector<std::string> row;
inputStream.NextRow(row);
REQUIRE(row.size() == 2);
REQUIRE(row[0] == "one");
REQUIRE(row[1] == "two");
inputStream.NextRow(row);
REQUIRE(row.size() == 2);
REQUIRE(row[0] == "foo");
REQUIRE(row[1] == "bar");
}
SECTION("Ensure can write csv value with comma")
{
std::istringstream ss("\"first,value\",two" NEW_LINE "foo,\"second,value\"" NEW_LINE "crazy,\"comma,value\",omg");
CsvInputStream inputStream(ss);
std::vector<std::string> row;
inputStream.NextRow(row);
REQUIRE(row.size() == 2);
REQUIRE(row[0] == "first,value");
REQUIRE(row[1] == "two");
inputStream.NextRow(row);
REQUIRE(row.size() == 2);
REQUIRE(row[0] == "foo");
REQUIRE(row[1] == "second,value");
inputStream.NextRow(row);
REQUIRE(row.size() == 3);
REQUIRE(row[0] == "crazy");
REQUIRE(row[1] == "comma,value");
REQUIRE(row[2] == "omg");
}
SECTION("Ensure can chain multiple values with commas")
{
std::istringstream ss(R"("first,value","second,value","third,value")");
CsvInputStream inputStream(ss);
std::vector<std::string> row;
inputStream.NextRow(row);
REQUIRE(row.size() == 3);
REQUIRE(row[0] == "first,value");
REQUIRE(row[1] == "second,value");
REQUIRE(row[2] == "third,value");
}
SECTION("Ensure can write csv value with quotes")
{
std::istringstream ss(R"(one,"two""value",three)");
CsvInputStream inputStream(ss);
std::vector<std::string> row;
inputStream.NextRow(row);
REQUIRE(row.size() == 3);
REQUIRE(row[0] == "one");
REQUIRE(row[1] == "two\"value");
REQUIRE(row[2] == "three");
}
SECTION("Ensure can write csv value with newlines")
{
std::istringstream ss("one,\"two\nvalue\",three");
CsvInputStream inputStream(ss);
std::vector<std::string> row;
inputStream.NextRow(row);
REQUIRE(row.size() == 3);
REQUIRE(row[0] == "one");
REQUIRE(row[1] == "two\nvalue");
REQUIRE(row[2] == "three");
}
SECTION("Ensure can write csv value with all special cases")
{
std::istringstream ss("one,\"two\nvalue,with\"\"alot\nof,special cases\",three");
CsvInputStream inputStream(ss);
std::vector<std::string> row;
inputStream.NextRow(row);
REQUIRE(row.size() == 3);
REQUIRE(row[0] == "one");
REQUIRE(row[1] == "two\nvalue,with\"alot\nof,special cases");
REQUIRE(row[2] == "three");
}
}
} // namespace