Add option to not parse floating point numbers when using SimpleLexer

src/ObjLoading/AssetLoading/SndCurve/SndCurveReader.cpp

@@ -191,7 +191,8 @@ std::unique_ptr<SndCurveReader::Result> SndCurveReader::Read() const
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = false;
-    lexerConfig.m_read_numbers = true;
+    lexerConfig.m_read_integer_numbers = true;
+    lexerConfig.m_read_floating_point_numbers = true;
 
     SimpleLexer lexer(&parserStream, std::move(lexerConfig));
     SndCurveParser parser(&lexer);

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp

@@ -32,7 +32,8 @@ std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = true;
     lexerConfig.m_read_strings = true;
-    lexerConfig.m_read_numbers = false;
+    lexerConfig.m_read_integer_numbers = false;
+    lexerConfig.m_read_floating_point_numbers = false;
     const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
 
     const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language);

src/ObjLoading/Parsing/Menu/MenuFileReader.cpp

@@ -129,7 +129,8 @@ std::unique_ptr<ParsingResult> MenuFileReader::ReadMenuFile()
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = true;
-    lexerConfig.m_read_numbers = true;
+    lexerConfig.m_read_integer_numbers = true;
+    lexerConfig.m_read_floating_point_numbers = true;
     MenuExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig);
 
     const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));

src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp

@@ -47,7 +47,8 @@ std::vector<std::unique_ptr<CommonStructuredDataDef>> StructuredDataDefReader::R
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = true;
-    lexerConfig.m_read_numbers = true;
+    lexerConfig.m_read_integer_numbers = true;
+    lexerConfig.m_read_floating_point_numbers = true;
     const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
 
     const auto parser = std::make_unique<StructuredDataDefParser>(lexer.get());

src/ObjLoading/Techset/TechniqueFileReader.cpp

@@ -22,7 +22,8 @@ bool TechniqueFileReader::ReadTechniqueDefinition() const
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = true;
-    lexerConfig.m_read_numbers = true;
+    lexerConfig.m_read_integer_numbers = true;
+    lexerConfig.m_read_floating_point_numbers = true;
     const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig));
 
     const auto parser = std::make_unique<TechniqueParser>(lexer.get(), m_acceptor);

src/ObjLoading/Techset/TechsetFileReader.cpp

@@ -22,7 +22,8 @@ std::unique_ptr<techset::TechsetDefinition> TechsetFileReader::ReadTechsetDefini
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = true;
-    lexerConfig.m_read_numbers = false;
+    lexerConfig.m_read_integer_numbers = false;
+    lexerConfig.m_read_floating_point_numbers = false;
     const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig));
 
     const auto parser = std::make_unique<techset::TechsetParser>(lexer.get(), m_valid_technique_type_names, m_valid_technique_type_name_count);

src/ObjWriting/Menu/AbstractMenuDumper.cpp

@@ -70,7 +70,8 @@ std::vector<std::string> AbstractMenuDumper::CreateScriptTokenList(const char* s
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
     lexerConfig.m_read_strings = true;
-    lexerConfig.m_read_numbers = false;
+    lexerConfig.m_read_integer_numbers = false;
+    lexerConfig.m_read_floating_point_numbers = false;
     SimpleLexer lexer(&inputStream, std::move(lexerConfig));
 
     std::vector<std::string> result;

src/Parser/Parsing/Impl/DefinesStreamProxy.cpp

@@ -285,7 +285,8 @@ std::unique_ptr<ISimpleExpression> DefinesStreamProxy::ParseIfExpression(const s
 
     SimpleLexer::Config lexerConfig;
     lexerConfig.m_emit_new_line_tokens = false;
-    lexerConfig.m_read_numbers = true;
+    lexerConfig.m_read_integer_numbers = true;
+    lexerConfig.m_read_floating_point_numbers = true;
     lexerConfig.m_read_strings = false;
     SimpleExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig);
 

src/Parser/Parsing/Simple/SimpleLexer.cpp

@@ -14,7 +14,7 @@ SimpleLexer::MultiCharacterTokenLookupEntry::MultiCharacterTokenLookupEntry(cons
 
 SimpleLexer::SimpleLexer(IParserLineStream* stream)
     : AbstractLexer(stream),
-      m_config{false, true, true, {}},
+      m_config{false, true, true, true, {}},
       m_check_for_multi_character_tokens(false),
       m_last_line(1)
 {
@@ -29,6 +29,9 @@ SimpleLexer::SimpleLexer(IParserLineStream* stream, Config config)
     for (auto tokenConfig : m_config.m_multi_character_tokens)
         AddMultiCharacterTokenConfigToLookup(std::move(tokenConfig));
     m_config.m_multi_character_tokens.clear();
+
+    // If reading floating point numbers then must be reading integers
+    assert(config.m_read_floating_point_numbers == false || config.m_read_floating_point_numbers == config.m_read_integer_numbers);
 }
 
 void SimpleLexer::AddMultiCharacterTokenConfigToLookup(Config::MultiCharacterToken tokenConfig)
@@ -120,21 +123,26 @@ SimpleParserValue SimpleLexer::GetNextToken()
     if (m_config.m_read_strings && c == '\"')
         return SimpleParserValue::String(pos, new std::string(ReadString()));
 
-    if (m_config.m_read_numbers && (isdigit(c) || (c == '+' || c == '-' || c == '.') && isdigit(PeekChar())))
+    if (m_config.m_read_integer_numbers && (isdigit(c) || (c == '+' || c == '-' || (m_config.m_read_floating_point_numbers && c == '.')) && isdigit(PeekChar())))
+    {
+        if(m_config.m_read_floating_point_numbers)
         {
             bool isFloatingPointValue;
             bool hasSignPrefix;
-        double doubleValue;
+            double floatingPointValue;
             int integerValue;
 
-        ReadNumber(isFloatingPointValue, hasSignPrefix, doubleValue, integerValue);
+            ReadNumber(isFloatingPointValue, hasSignPrefix, floatingPointValue, integerValue);
 
             if (isFloatingPointValue)
-            return SimpleParserValue::FloatingPoint(pos, doubleValue, hasSignPrefix);
+                return SimpleParserValue::FloatingPoint(pos, floatingPointValue, hasSignPrefix);
 
             return SimpleParserValue::Integer(pos, integerValue, hasSignPrefix);
         }
 
+        return SimpleParserValue::Integer(pos, ReadInteger(), c == '+' || c == '-');
+    }
+
     if (isalpha(c) || c == '_')
     {
         auto identifier = ReadIdentifier();

src/Parser/Parsing/Simple/SimpleLexer.h

@@ -24,7 +24,8 @@ public:
 
         bool m_emit_new_line_tokens;
         bool m_read_strings;
-        bool m_read_numbers;
+        bool m_read_integer_numbers;
+        bool m_read_floating_point_numbers;
         std::vector<MultiCharacterToken> m_multi_character_tokens;
     };
 

test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp

@@ -111,7 +111,8 @@ namespace test::parsing::simple::expression
 
             SimpleLexer::Config lexerConfig;
             lexerConfig.m_read_strings = true;
-            lexerConfig.m_read_numbers = true;
+            lexerConfig.m_read_integer_numbers = true;
+            lexerConfig.m_read_floating_point_numbers = true;
             lexerConfig.m_emit_new_line_tokens = false;
             SimpleExpressionMatchers(true, true, true, true, true).ApplyTokensToLexerConfig(lexerConfig);
             m_lexer = std::make_unique<SimpleLexer>(m_stream.get(), std::move(lexerConfig));