diff --git a/src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp b/src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp index eb9ea67a..073a3487 100644 --- a/src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp +++ b/src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp @@ -32,6 +32,7 @@ std::vector LocalizeFileReader::ReadLocalizeFile() SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = true; lexerConfig.m_read_strings = true; + lexerConfig.m_string_escape_sequences = false; // Escape sequences are handled inside the Parser itself in the Sequence lexerConfig.m_read_integer_numbers = false; lexerConfig.m_read_floating_point_numbers = false; const auto lexer = std::make_unique(m_stream, std::move(lexerConfig)); diff --git a/src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileLanguageValue.cpp b/src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileLanguageValue.cpp index 8a746551..8edcc3de 100644 --- a/src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileLanguageValue.cpp +++ b/src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileLanguageValue.cpp @@ -18,37 +18,7 @@ SequenceLocalizeFileLanguageValue::SequenceLocalizeFileLanguageValue() std::string SequenceLocalizeFileLanguageValue::UnescapeValue(const std::string& value) { std::ostringstream str; - - auto isEscaped = false; - for(const auto c : value) - { - if(isEscaped) - { - switch(c) - { - case 'n': - str << '\n'; - break; - - case 'r': - str << '\r'; - break; - - default: - str << c; - break; - } - isEscaped = false; - } - else if(c == '\\') - { - isEscaped = true; - } - else - { - str << c; - } - } + utils::UnescapeStringFromQuotationMarks(str, value); return str.str(); } diff --git a/src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp b/src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp index 726bbd2b..5dba0a64 100644 --- a/src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp +++ b/src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp @@ -47,6 +47,7 @@ std::vector> StructuredDataDefReader::R SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = true; + lexerConfig.m_string_escape_sequences = true; lexerConfig.m_read_integer_numbers = true; lexerConfig.m_read_floating_point_numbers = true; const auto lexer = std::make_unique(m_stream, std::move(lexerConfig)); diff --git a/src/ObjLoading/Techset/TechniqueFileReader.cpp b/src/ObjLoading/Techset/TechniqueFileReader.cpp index 959bcf21..67d37fc7 100644 --- a/src/ObjLoading/Techset/TechniqueFileReader.cpp +++ b/src/ObjLoading/Techset/TechniqueFileReader.cpp @@ -22,6 +22,7 @@ bool TechniqueFileReader::ReadTechniqueDefinition() const SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = true; + lexerConfig.m_string_escape_sequences = false; lexerConfig.m_read_integer_numbers = true; lexerConfig.m_read_floating_point_numbers = true; const auto lexer = std::make_unique(m_comment_proxy.get(), std::move(lexerConfig)); diff --git a/src/ObjLoading/Techset/TechsetFileReader.cpp b/src/ObjLoading/Techset/TechsetFileReader.cpp index 26f57aa9..52fdd964 100644 --- a/src/ObjLoading/Techset/TechsetFileReader.cpp +++ b/src/ObjLoading/Techset/TechsetFileReader.cpp @@ -22,6 +22,7 @@ std::unique_ptr TechsetFileReader::ReadTechsetDefini SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = true; + lexerConfig.m_string_escape_sequences = false; lexerConfig.m_read_integer_numbers = false; lexerConfig.m_read_floating_point_numbers = false; const auto lexer = std::make_unique(m_comment_proxy.get(), std::move(lexerConfig)); diff --git a/src/ObjWriting/Dumping/Localize/StringFileDumper.cpp b/src/ObjWriting/Dumping/Localize/StringFileDumper.cpp index 33eb1575..b54b8a6b 100644 --- a/src/ObjWriting/Dumping/Localize/StringFileDumper.cpp +++ b/src/ObjWriting/Dumping/Localize/StringFileDumper.cpp @@ -1,6 +1,8 @@ #include "StringFileDumper.h" #include +#include "Utils/StringUtils.h" + StringFileDumper::StringFileDumper(Zone* zone, std::ostream& stream) : AbstractTextDumper(stream), m_zone(zone), @@ -44,11 +46,10 @@ void StringFileDumper::WriteLocalizeEntry(const std::string& reference, const st m_stream << "\n"; m_stream << "REFERENCE " << reference << "\n"; - auto escapedValue = std::regex_replace(value, std::regex("\n"), "\\n"); - escapedValue = std::regex_replace(escapedValue, std::regex("\r"), "\\r"); - const auto valueSpacing = std::string(15 - m_language_caps.length(), ' '); - m_stream << "LANG_" << m_language_caps << valueSpacing << "\"" << escapedValue << "\"\n"; + m_stream << "LANG_" << m_language_caps << valueSpacing << "\""; + utils::EscapeStringForQuotationMarks(m_stream, value); + m_stream << "\"\n"; } void StringFileDumper::Finalize() diff --git a/src/ObjWriting/Menu/AbstractMenuDumper.cpp b/src/ObjWriting/Menu/AbstractMenuDumper.cpp index 914926d2..6f02352e 100644 --- a/src/ObjWriting/Menu/AbstractMenuDumper.cpp +++ b/src/ObjWriting/Menu/AbstractMenuDumper.cpp @@ -70,6 +70,7 @@ std::vector AbstractMenuDumper::CreateScriptTokenList(const char* s SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = true; + lexerConfig.m_string_escape_sequences = true; lexerConfig.m_read_integer_numbers = false; lexerConfig.m_read_floating_point_numbers = false; SimpleLexer lexer(&inputStream, std::move(lexerConfig)); diff --git a/src/ObjWriting/StructuredDataDef/StructuredDataDefDumper.cpp b/src/ObjWriting/StructuredDataDef/StructuredDataDefDumper.cpp index 823a0862..a8fed89f 100644 --- a/src/ObjWriting/StructuredDataDef/StructuredDataDefDumper.cpp +++ b/src/ObjWriting/StructuredDataDef/StructuredDataDefDumper.cpp @@ -4,6 +4,7 @@ #include #include "Utils/Alignment.h" +#include "Utils/StringUtils.h" StructuredDataDefDumper::StructuredDataDefDumper(std::ostream& stream) : AbstractTextDumper(stream), @@ -37,7 +38,9 @@ void StructuredDataDefDumper::DumpEnum(const CommonStructuredDataEnum& _enum) for (auto i = 0u; i < entryCount; i++) { Indent(); - m_stream << "\"" << _enum.m_entries[i].m_name << "\""; + m_stream << "\""; + utils::EscapeStringForQuotationMarks(m_stream, _enum.m_entries[i].m_name); + m_stream << "\""; if (i + 1 < entryCount) m_stream << ","; diff --git a/src/Parser/Parsing/Simple/SimpleExpressionInterpreter.cpp b/src/Parser/Parsing/Simple/SimpleExpressionInterpreter.cpp index f8b0bbaf..f4975734 100644 --- a/src/Parser/Parsing/Simple/SimpleExpressionInterpreter.cpp +++ b/src/Parser/Parsing/Simple/SimpleExpressionInterpreter.cpp @@ -67,6 +67,7 @@ std::unique_ptr SimpleExpressionInterpreter::Evaluate() const lexerConfig.m_read_integer_numbers = true; lexerConfig.m_read_floating_point_numbers = true; lexerConfig.m_read_strings = true; + lexerConfig.m_string_escape_sequences = true; SimpleExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig); SimpleLexer lexer(m_input, std::move(lexerConfig)); diff --git a/src/Parser/Parsing/Simple/SimpleLexer.h b/src/Parser/Parsing/Simple/SimpleLexer.h index 8a4315d5..fd051f2d 100644 --- a/src/Parser/Parsing/Simple/SimpleLexer.h +++ b/src/Parser/Parsing/Simple/SimpleLexer.h @@ -22,11 +22,11 @@ public: MultiCharacterToken(int id, std::string value); }; - bool m_emit_new_line_tokens; - bool m_read_strings; - bool m_string_escape_sequences; - bool m_read_integer_numbers; - bool m_read_floating_point_numbers; + bool m_emit_new_line_tokens = false; + bool m_read_strings = true; + bool m_string_escape_sequences = false; + bool m_read_integer_numbers = true; + bool m_read_floating_point_numbers = true; std::vector m_multi_character_tokens; }; diff --git a/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp b/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp index f9484977..14082bd3 100644 --- a/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp +++ b/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp @@ -112,6 +112,7 @@ namespace test::parsing::simple::expression SimpleLexer::Config lexerConfig; lexerConfig.m_read_strings = true; + lexerConfig.m_string_escape_sequences = true; lexerConfig.m_read_integer_numbers = true; lexerConfig.m_read_floating_point_numbers = true; lexerConfig.m_emit_new_line_tokens = false;