diff --git a/src/ObjLoading/AssetLoading/SndCurve/SndCurveReader.cpp b/src/ObjLoading/AssetLoading/SndCurve/SndCurveReader.cpp index eacfd680..5649a25e 100644 --- a/src/ObjLoading/AssetLoading/SndCurve/SndCurveReader.cpp +++ b/src/ObjLoading/AssetLoading/SndCurve/SndCurveReader.cpp @@ -191,7 +191,8 @@ std::unique_ptr SndCurveReader::Read() const SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = false; - lexerConfig.m_read_numbers = true; + lexerConfig.m_read_integer_numbers = true; + lexerConfig.m_read_floating_point_numbers = true; SimpleLexer lexer(&parserStream, std::move(lexerConfig)); SndCurveParser parser(&lexer); diff --git a/src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp b/src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp index 1a4af3fb..eb9ea67a 100644 --- a/src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp +++ b/src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp @@ -32,7 +32,8 @@ std::vector LocalizeFileReader::ReadLocalizeFile() SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = true; lexerConfig.m_read_strings = true; - lexerConfig.m_read_numbers = false; + lexerConfig.m_read_integer_numbers = false; + lexerConfig.m_read_floating_point_numbers = false; const auto lexer = std::make_unique(m_stream, std::move(lexerConfig)); const auto parser = std::make_unique(lexer.get(), m_language); diff --git a/src/ObjLoading/Parsing/Menu/MenuFileReader.cpp b/src/ObjLoading/Parsing/Menu/MenuFileReader.cpp index 751d9103..edbc58d3 100644 --- a/src/ObjLoading/Parsing/Menu/MenuFileReader.cpp +++ b/src/ObjLoading/Parsing/Menu/MenuFileReader.cpp @@ -129,7 +129,8 @@ std::unique_ptr MenuFileReader::ReadMenuFile() SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = true; - lexerConfig.m_read_numbers = true; + lexerConfig.m_read_integer_numbers = true; + lexerConfig.m_read_floating_point_numbers = true; MenuExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig); const auto lexer = std::make_unique(m_stream, std::move(lexerConfig)); diff --git a/src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp b/src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp index 2eade578..726bbd2b 100644 --- a/src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp +++ b/src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp @@ -47,7 +47,8 @@ std::vector> StructuredDataDefReader::R SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = true; - lexerConfig.m_read_numbers = true; + lexerConfig.m_read_integer_numbers = true; + lexerConfig.m_read_floating_point_numbers = true; const auto lexer = std::make_unique(m_stream, std::move(lexerConfig)); const auto parser = std::make_unique(lexer.get()); diff --git a/src/ObjLoading/Techset/TechniqueFileReader.cpp b/src/ObjLoading/Techset/TechniqueFileReader.cpp index 3ced3da0..959bcf21 100644 --- a/src/ObjLoading/Techset/TechniqueFileReader.cpp +++ b/src/ObjLoading/Techset/TechniqueFileReader.cpp @@ -22,7 +22,8 @@ bool TechniqueFileReader::ReadTechniqueDefinition() const SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = true; - lexerConfig.m_read_numbers = true; + lexerConfig.m_read_integer_numbers = true; + lexerConfig.m_read_floating_point_numbers = true; const auto lexer = std::make_unique(m_comment_proxy.get(), std::move(lexerConfig)); const auto parser = std::make_unique(lexer.get(), m_acceptor); diff --git a/src/ObjLoading/Techset/TechsetFileReader.cpp b/src/ObjLoading/Techset/TechsetFileReader.cpp index 8255fa48..26f57aa9 100644 --- a/src/ObjLoading/Techset/TechsetFileReader.cpp +++ b/src/ObjLoading/Techset/TechsetFileReader.cpp @@ -22,7 +22,8 @@ std::unique_ptr TechsetFileReader::ReadTechsetDefini SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = true; - lexerConfig.m_read_numbers = false; + lexerConfig.m_read_integer_numbers = false; + lexerConfig.m_read_floating_point_numbers = false; const auto lexer = std::make_unique(m_comment_proxy.get(), std::move(lexerConfig)); const auto parser = std::make_unique(lexer.get(), m_valid_technique_type_names, m_valid_technique_type_name_count); diff --git a/src/ObjWriting/Menu/AbstractMenuDumper.cpp b/src/ObjWriting/Menu/AbstractMenuDumper.cpp index 2bdeaa06..60fef6c9 100644 --- a/src/ObjWriting/Menu/AbstractMenuDumper.cpp +++ b/src/ObjWriting/Menu/AbstractMenuDumper.cpp @@ -70,7 +70,8 @@ std::vector AbstractMenuDumper::CreateScriptTokenList(const char* s SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_read_strings = true; - lexerConfig.m_read_numbers = false; + lexerConfig.m_read_integer_numbers = false; + lexerConfig.m_read_floating_point_numbers = false; SimpleLexer lexer(&inputStream, std::move(lexerConfig)); std::vector result; diff --git a/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp b/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp index 8b2e51e3..8eaf8bb6 100644 --- a/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp +++ b/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp @@ -285,7 +285,8 @@ std::unique_ptr DefinesStreamProxy::ParseIfExpression(const s SimpleLexer::Config lexerConfig; lexerConfig.m_emit_new_line_tokens = false; - lexerConfig.m_read_numbers = true; + lexerConfig.m_read_integer_numbers = true; + lexerConfig.m_read_floating_point_numbers = true; lexerConfig.m_read_strings = false; SimpleExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig); diff --git a/src/Parser/Parsing/Simple/SimpleLexer.cpp b/src/Parser/Parsing/Simple/SimpleLexer.cpp index bb862a37..fc7db4d3 100644 --- a/src/Parser/Parsing/Simple/SimpleLexer.cpp +++ b/src/Parser/Parsing/Simple/SimpleLexer.cpp @@ -14,7 +14,7 @@ SimpleLexer::MultiCharacterTokenLookupEntry::MultiCharacterTokenLookupEntry(cons SimpleLexer::SimpleLexer(IParserLineStream* stream) : AbstractLexer(stream), - m_config{false, true, true, {}}, + m_config{false, true, true, true, {}}, m_check_for_multi_character_tokens(false), m_last_line(1) { @@ -29,6 +29,9 @@ SimpleLexer::SimpleLexer(IParserLineStream* stream, Config config) for (auto tokenConfig : m_config.m_multi_character_tokens) AddMultiCharacterTokenConfigToLookup(std::move(tokenConfig)); m_config.m_multi_character_tokens.clear(); + + // If reading floating point numbers then must be reading integers + assert(config.m_read_floating_point_numbers == false || config.m_read_floating_point_numbers == config.m_read_integer_numbers); } void SimpleLexer::AddMultiCharacterTokenConfigToLookup(Config::MultiCharacterToken tokenConfig) @@ -105,12 +108,12 @@ SimpleParserValue SimpleLexer::GetNextToken() if (c == EOF) return SimpleParserValue::EndOfFile(TokenPos()); - if(m_check_for_multi_character_tokens) + if (m_check_for_multi_character_tokens) { const auto* multiTokenLookup = m_multi_character_token_lookup[static_cast(c)].get(); - while(multiTokenLookup) + while (multiTokenLookup) { - if(ReadMultiCharacterToken(multiTokenLookup)) + if (ReadMultiCharacterToken(multiTokenLookup)) return SimpleParserValue::MultiCharacter(pos, multiTokenLookup->m_id); multiTokenLookup = multiTokenLookup->m_next.get(); @@ -120,19 +123,24 @@ SimpleParserValue SimpleLexer::GetNextToken() if (m_config.m_read_strings && c == '\"') return SimpleParserValue::String(pos, new std::string(ReadString())); - if (m_config.m_read_numbers && (isdigit(c) || (c == '+' || c == '-' || c == '.') && isdigit(PeekChar()))) + if (m_config.m_read_integer_numbers && (isdigit(c) || (c == '+' || c == '-' || (m_config.m_read_floating_point_numbers && c == '.')) && isdigit(PeekChar()))) { - bool isFloatingPointValue; - bool hasSignPrefix; - double doubleValue; - int integerValue; + if(m_config.m_read_floating_point_numbers) + { + bool isFloatingPointValue; + bool hasSignPrefix; + double floatingPointValue; + int integerValue; - ReadNumber(isFloatingPointValue, hasSignPrefix, doubleValue, integerValue); + ReadNumber(isFloatingPointValue, hasSignPrefix, floatingPointValue, integerValue); - if (isFloatingPointValue) - return SimpleParserValue::FloatingPoint(pos, doubleValue, hasSignPrefix); + if (isFloatingPointValue) + return SimpleParserValue::FloatingPoint(pos, floatingPointValue, hasSignPrefix); - return SimpleParserValue::Integer(pos, integerValue, hasSignPrefix); + return SimpleParserValue::Integer(pos, integerValue, hasSignPrefix); + } + + return SimpleParserValue::Integer(pos, ReadInteger(), c == '+' || c == '-'); } if (isalpha(c) || c == '_') diff --git a/src/Parser/Parsing/Simple/SimpleLexer.h b/src/Parser/Parsing/Simple/SimpleLexer.h index dceae153..7563e2db 100644 --- a/src/Parser/Parsing/Simple/SimpleLexer.h +++ b/src/Parser/Parsing/Simple/SimpleLexer.h @@ -24,7 +24,8 @@ public: bool m_emit_new_line_tokens; bool m_read_strings; - bool m_read_numbers; + bool m_read_integer_numbers; + bool m_read_floating_point_numbers; std::vector m_multi_character_tokens; }; diff --git a/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp b/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp index 8eaec2e8..a7fde18b 100644 --- a/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp +++ b/test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp @@ -111,7 +111,8 @@ namespace test::parsing::simple::expression SimpleLexer::Config lexerConfig; lexerConfig.m_read_strings = true; - lexerConfig.m_read_numbers = true; + lexerConfig.m_read_integer_numbers = true; + lexerConfig.m_read_floating_point_numbers = true; lexerConfig.m_emit_new_line_tokens = false; SimpleExpressionMatchers(true, true, true, true, true).ApplyTokensToLexerConfig(lexerConfig); m_lexer = std::make_unique(m_stream.get(), std::move(lexerConfig));