Use escaped strings with simple lexer whenever appropriate

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp

@@ -32,6 +32,7 @@ std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = true;
     lexerConfig.m_read_strings = true;
+    lexerConfig.m_string_escape_sequences = false; // Escape sequences are handled inside the Parser itself in the Sequence
     lexerConfig.m_read_integer_numbers = false;
     lexerConfig.m_read_floating_point_numbers = false;
     const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileLanguageValue.cpp

@@ -18,37 +18,7 @@ SequenceLocalizeFileLanguageValue::SequenceLocalizeFileLanguageValue()
 std::string SequenceLocalizeFileLanguageValue::UnescapeValue(const std::string& value)
 {
     std::ostringstream str;
-
+    utils::UnescapeStringFromQuotationMarks(str, value);
-    auto isEscaped = false;
-    for(const auto c : value)
-    {
-        if(isEscaped)
-        {
-            switch(c)
-            {
-            case 'n':
-                str << '\n';
-                break;
-
-            case 'r':
-                str << '\r';
-                break;
-
-            default:
-                str << c;
-                break;
-            }
-            isEscaped = false;
-        }
-        else if(c == '\\')
-        {
-            isEscaped = true;
-        }
-        else
-        {
-            str << c;
-        }
-    }
 
     return str.str();
 }

src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp

@@ -47,6 +47,7 @@ std::vector<std::unique_ptr<CommonStructuredDataDef>> StructuredDataDefReader::R
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = true;
+    lexerConfig.m_string_escape_sequences = true;
     lexerConfig.m_read_integer_numbers = true;
     lexerConfig.m_read_floating_point_numbers = true;
     const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));

src/ObjLoading/Techset/TechniqueFileReader.cpp

@@ -22,6 +22,7 @@ bool TechniqueFileReader::ReadTechniqueDefinition() const
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = true;
+    lexerConfig.m_string_escape_sequences = false;
     lexerConfig.m_read_integer_numbers = true;
     lexerConfig.m_read_floating_point_numbers = true;
     const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig));

src/ObjLoading/Techset/TechsetFileReader.cpp

@@ -22,6 +22,7 @@ std::unique_ptr<techset::TechsetDefinition> TechsetFileReader::ReadTechsetDefini
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = true;
+    lexerConfig.m_string_escape_sequences = false;
     lexerConfig.m_read_integer_numbers = false;
     lexerConfig.m_read_floating_point_numbers = false;
     const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig));

src/ObjWriting/Dumping/Localize/StringFileDumper.cpp

@@ -1,6 +1,8 @@
 #include "StringFileDumper.h"
 #include <regex>
 
+#include "Utils/StringUtils.h"
+
 StringFileDumper::StringFileDumper(Zone* zone, std::ostream& stream)
     : AbstractTextDumper(stream),
       m_zone(zone),
@@ -44,11 +46,10 @@ void StringFileDumper::WriteLocalizeEntry(const std::string& reference, const st
     m_stream << "\n";
     m_stream << "REFERENCE           " << reference << "\n";
 
-    auto escapedValue = std::regex_replace(value, std::regex("\n"), "\\n");
-    escapedValue = std::regex_replace(escapedValue, std::regex("\r"), "\\r");
-
     const auto valueSpacing = std::string(15 - m_language_caps.length(), ' ');
-    m_stream << "LANG_" << m_language_caps << valueSpacing << "\"" << escapedValue << "\"\n";
+    m_stream << "LANG_" << m_language_caps << valueSpacing << "\"";
+    utils::EscapeStringForQuotationMarks(m_stream, value);
+    m_stream << "\"\n";
 }
 
 void StringFileDumper::Finalize()

src/ObjWriting/Menu/AbstractMenuDumper.cpp

@@ -70,6 +70,7 @@ std::vector<std::string> AbstractMenuDumper::CreateScriptTokenList(const char* s
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = true;
+    lexerConfig.m_string_escape_sequences = true;
     lexerConfig.m_read_integer_numbers = false;
     lexerConfig.m_read_floating_point_numbers = false;
     SimpleLexer lexer(&inputStream, std::move(lexerConfig));

src/ObjWriting/StructuredDataDef/StructuredDataDefDumper.cpp

@@ -4,6 +4,7 @@
 #include <sstream>
 
 #include "Utils/Alignment.h"
+#include "Utils/StringUtils.h"
 
 StructuredDataDefDumper::StructuredDataDefDumper(std::ostream& stream)
     : AbstractTextDumper(stream),
@@ -37,7 +38,9 @@ void StructuredDataDefDumper::DumpEnum(const CommonStructuredDataEnum& _enum)
     for (auto i = 0u; i < entryCount; i++)
     {
         Indent();
-        m_stream << "\"" << _enum.m_entries[i].m_name << "\"";
+        m_stream << "\"";
+        utils::EscapeStringForQuotationMarks(m_stream, _enum.m_entries[i].m_name);
+        m_stream << "\"";
 
         if (i + 1 < entryCount)
             m_stream << ",";

src/Parser/Parsing/Simple/SimpleExpressionInterpreter.cpp

@@ -67,6 +67,7 @@ std::unique_ptr<ISimpleExpression> SimpleExpressionInterpreter::Evaluate() const
     lexerConfig.m_read_integer_numbers = true;
     lexerConfig.m_read_floating_point_numbers = true;
     lexerConfig.m_read_strings = true;
+    lexerConfig.m_string_escape_sequences = true;
     SimpleExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig);
 
     SimpleLexer lexer(m_input, std::move(lexerConfig));

src/Parser/Parsing/Simple/SimpleLexer.h

@@ -22,11 +22,11 @@ public:
             MultiCharacterToken(int id, std::string value);
         };
 
-        bool m_emit_new_line_tokens;
+        bool m_emit_new_line_tokens = false;
-        bool m_read_strings;
+        bool m_read_strings = true;
-        bool m_string_escape_sequences;
+        bool m_string_escape_sequences = false;
-        bool m_read_integer_numbers;
+        bool m_read_integer_numbers = true;
-        bool m_read_floating_point_numbers;
+        bool m_read_floating_point_numbers = true;
         std::vector<MultiCharacterToken> m_multi_character_tokens;
     };
 

test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp

@@ -112,6 +112,7 @@ namespace test::parsing::simple::expression
 
             SimpleLexer::Config lexerConfig;
             lexerConfig.m_read_strings = true;
+            lexerConfig.m_string_escape_sequences = true;
             lexerConfig.m_read_integer_numbers = true;
             lexerConfig.m_read_floating_point_numbers = true;
             lexerConfig.m_emit_new_line_tokens = false;