zone/OpenAssetTools
zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-04-19 15:52:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add option to not parse floating point numbers when using SimpleLexer

Browse Source
This commit is contained in:
Jan 2022-03-30 19:07:54 +02:00
parent b5cd357014
commit a092f76ec9
11 changed files with 41 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/ObjLoading/AssetLoading/SndCurve/SndCurveReader.cpp
View File

@ -191,7 +191,8 @@ std::unique_ptr<SndCurveReader::Result> SndCurveReader::Read() const
SimpleLexer::Config lexerConfig; SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = false; lexerConfig.m_read_strings = false;
lexerConfig.m_read_numbers = true; lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
SimpleLexer lexer(&parserStream, std::move(lexerConfig)); SimpleLexer lexer(&parserStream, std::move(lexerConfig));
SndCurveParser parser(&lexer); SndCurveParser parser(&lexer);

3
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp
View File

@ -32,7 +32,8 @@ std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
SimpleLexer::Config lexerConfig; SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = true; lexerConfig.m_emit_new_line_tokens = true;
lexerConfig.m_read_strings = true; lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = false; lexerConfig.m_read_integer_numbers = false;
lexerConfig.m_read_floating_point_numbers = false;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig)); const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language); const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language);

3
src/ObjLoading/Parsing/Menu/MenuFileReader.cpp
View File

@ -129,7 +129,8 @@ std::unique_ptr<ParsingResult> MenuFileReader::ReadMenuFile()
SimpleLexer::Config lexerConfig; SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true; lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true; lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
MenuExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig); MenuExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig);
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig)); const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));

3
src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp
View File

@ -47,7 +47,8 @@ std::vector<std::unique_ptr<CommonStructuredDataDef>> StructuredDataDefReader::R
SimpleLexer::Config lexerConfig; SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true; lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true; lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig)); const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<StructuredDataDefParser>(lexer.get()); const auto parser = std::make_unique<StructuredDataDefParser>(lexer.get());

3
src/ObjLoading/Techset/TechniqueFileReader.cpp
View File

@ -22,7 +22,8 @@ bool TechniqueFileReader::ReadTechniqueDefinition() const
SimpleLexer::Config lexerConfig; SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true; lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true; lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig)); const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig));
const auto parser = std::make_unique<TechniqueParser>(lexer.get(), m_acceptor); const auto parser = std::make_unique<TechniqueParser>(lexer.get(), m_acceptor);

3
src/ObjLoading/Techset/TechsetFileReader.cpp
View File

@ -22,7 +22,8 @@ std::unique_ptr<techset::TechsetDefinition> TechsetFileReader::ReadTechsetDefini
SimpleLexer::Config lexerConfig; SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true; lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = false; lexerConfig.m_read_integer_numbers = false;
lexerConfig.m_read_floating_point_numbers = false;
const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig)); const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig));
const auto parser = std::make_unique<techset::TechsetParser>(lexer.get(), m_valid_technique_type_names, m_valid_technique_type_name_count); const auto parser = std::make_unique<techset::TechsetParser>(lexer.get(), m_valid_technique_type_names, m_valid_technique_type_name_count);

3
src/ObjWriting/Menu/AbstractMenuDumper.cpp
View File

@ -70,7 +70,8 @@ std::vector<std::string> AbstractMenuDumper::CreateScriptTokenList(const char* s
SimpleLexer::Config lexerConfig; SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true; lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = false; lexerConfig.m_read_integer_numbers = false;
lexerConfig.m_read_floating_point_numbers = false;
SimpleLexer lexer(&inputStream, std::move(lexerConfig)); SimpleLexer lexer(&inputStream, std::move(lexerConfig));
std::vector<std::string> result; std::vector<std::string> result;

3
src/Parser/Parsing/Impl/DefinesStreamProxy.cpp
View File

@ -285,7 +285,8 @@ std::unique_ptr<ISimpleExpression> DefinesStreamProxy::ParseIfExpression(const s
SimpleLexer::Config lexerConfig; SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_numbers = true; lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
lexerConfig.m_read_strings = false; lexerConfig.m_read_strings = false;
SimpleExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig); SimpleExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig);

24
src/Parser/Parsing/Simple/SimpleLexer.cpp
View File

@ -14,7 +14,7 @@ SimpleLexer::MultiCharacterTokenLookupEntry::MultiCharacterTokenLookupEntry(cons
SimpleLexer::SimpleLexer(IParserLineStream* stream) SimpleLexer::SimpleLexer(IParserLineStream* stream)
: AbstractLexer(stream), : AbstractLexer(stream),
m_config{false, true, true, {}}, m_config{false, true, true, true, {}},
m_check_for_multi_character_tokens(false), m_check_for_multi_character_tokens(false),
m_last_line(1) m_last_line(1)
{ {
@ -29,6 +29,9 @@ SimpleLexer::SimpleLexer(IParserLineStream* stream, Config config)
for (auto tokenConfig : m_config.m_multi_character_tokens) for (auto tokenConfig : m_config.m_multi_character_tokens)
AddMultiCharacterTokenConfigToLookup(std::move(tokenConfig)); AddMultiCharacterTokenConfigToLookup(std::move(tokenConfig));
m_config.m_multi_character_tokens.clear(); m_config.m_multi_character_tokens.clear();
// If reading floating point numbers then must be reading integers
assert(config.m_read_floating_point_numbers == false || config.m_read_floating_point_numbers == config.m_read_integer_numbers);
} }
void SimpleLexer::AddMultiCharacterTokenConfigToLookup(Config::MultiCharacterToken tokenConfig) void SimpleLexer::AddMultiCharacterTokenConfigToLookup(Config::MultiCharacterToken tokenConfig)
@ -105,12 +108,12 @@ SimpleParserValue SimpleLexer::GetNextToken()
if (c == EOF) if (c == EOF)
return SimpleParserValue::EndOfFile(TokenPos()); return SimpleParserValue::EndOfFile(TokenPos());
if(m_check_for_multi_character_tokens) if (m_check_for_multi_character_tokens)
{ {
const auto* multiTokenLookup = m_multi_character_token_lookup[static_cast<uint8_t>(c)].get(); const auto* multiTokenLookup = m_multi_character_token_lookup[static_cast<uint8_t>(c)].get();
while(multiTokenLookup) while (multiTokenLookup)
{ {
if(ReadMultiCharacterToken(multiTokenLookup)) if (ReadMultiCharacterToken(multiTokenLookup))
return SimpleParserValue::MultiCharacter(pos, multiTokenLookup->m_id); return SimpleParserValue::MultiCharacter(pos, multiTokenLookup->m_id);
multiTokenLookup = multiTokenLookup->m_next.get(); multiTokenLookup = multiTokenLookup->m_next.get();
@ -120,21 +123,26 @@ SimpleParserValue SimpleLexer::GetNextToken()
if (m_config.m_read_strings && c == '\"') if (m_config.m_read_strings && c == '\"')
return SimpleParserValue::String(pos, new std::string(ReadString())); return SimpleParserValue::String(pos, new std::string(ReadString()));
if (m_config.m_read_numbers && (isdigit(c) || (c == '+' || c == '-' || c == '.') && isdigit(PeekChar()))) if (m_config.m_read_integer_numbers && (isdigit(c) || (c == '+' || c == '-' || (m_config.m_read_floating_point_numbers && c == '.')) && isdigit(PeekChar())))
{
if(m_config.m_read_floating_point_numbers)
{ {
bool isFloatingPointValue; bool isFloatingPointValue;
bool hasSignPrefix; bool hasSignPrefix;
double doubleValue; double floatingPointValue;
int integerValue; int integerValue;
ReadNumber(isFloatingPointValue, hasSignPrefix, doubleValue, integerValue); ReadNumber(isFloatingPointValue, hasSignPrefix, floatingPointValue, integerValue);
if (isFloatingPointValue) if (isFloatingPointValue)
return SimpleParserValue::FloatingPoint(pos, doubleValue, hasSignPrefix); return SimpleParserValue::FloatingPoint(pos, floatingPointValue, hasSignPrefix);
return SimpleParserValue::Integer(pos, integerValue, hasSignPrefix); return SimpleParserValue::Integer(pos, integerValue, hasSignPrefix);
} }
return SimpleParserValue::Integer(pos, ReadInteger(), c == '+' || c == '-');
}
if (isalpha(c) || c == '_') if (isalpha(c) || c == '_')
{ {
auto identifier = ReadIdentifier(); auto identifier = ReadIdentifier();

3
src/Parser/Parsing/Simple/SimpleLexer.h
View File

@ -24,7 +24,8 @@ public:
bool m_emit_new_line_tokens; bool m_emit_new_line_tokens;
bool m_read_strings; bool m_read_strings;
bool m_read_numbers; bool m_read_integer_numbers;
bool m_read_floating_point_numbers;
std::vector<MultiCharacterToken> m_multi_character_tokens; std::vector<MultiCharacterToken> m_multi_character_tokens;
}; };

3
test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp
View File

@ -111,7 +111,8 @@ namespace test::parsing::simple::expression
SimpleLexer::Config lexerConfig; SimpleLexer::Config lexerConfig;
lexerConfig.m_read_strings = true; lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true; lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
lexerConfig.m_emit_new_line_tokens = false; lexerConfig.m_emit_new_line_tokens = false;
SimpleExpressionMatchers(true, true, true, true, true).ApplyTokensToLexerConfig(lexerConfig); SimpleExpressionMatchers(true, true, true, true, true).ApplyTokensToLexerConfig(lexerConfig);
m_lexer = std::make_unique<SimpleLexer>(m_stream.get(), std::move(lexerConfig)); m_lexer = std::make_unique<SimpleLexer>(m_stream.get(), std::move(lexerConfig));