zone/OpenAssetTools
zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-04-19 15:52:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add option to not parse floating point numbers when using SimpleLexer

Browse Source
This commit is contained in:
Jan 2022-03-30 19:07:54 +02:00
parent b5cd357014
commit a092f76ec9
11 changed files with 41 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/ObjLoading/AssetLoading/SndCurve/SndCurveReader.cpp
View File

@ -191,7 +191,8 @@ std::unique_ptr<SndCurveReader::Result> SndCurveReader::Read() const
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = false;
lexerConfig.m_read_numbers = true;
lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
SimpleLexer lexer(&parserStream, std::move(lexerConfig));
SndCurveParser parser(&lexer);

3
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp
View File

@ -32,7 +32,8 @@ std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = true;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = false;
lexerConfig.m_read_integer_numbers = false;
lexerConfig.m_read_floating_point_numbers = false;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language);

3
src/ObjLoading/Parsing/Menu/MenuFileReader.cpp
View File

@ -129,7 +129,8 @@ std::unique_ptr<ParsingResult> MenuFileReader::ReadMenuFile()
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true;
lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
MenuExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig);
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));

3
src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp
View File

@ -47,7 +47,8 @@ std::vector<std::unique_ptr<CommonStructuredDataDef>> StructuredDataDefReader::R
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true;
lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<StructuredDataDefParser>(lexer.get());

3
src/ObjLoading/Techset/TechniqueFileReader.cpp
View File

@ -22,7 +22,8 @@ bool TechniqueFileReader::ReadTechniqueDefinition() const
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true;
lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig));
const auto parser = std::make_unique<TechniqueParser>(lexer.get(), m_acceptor);

3
src/ObjLoading/Techset/TechsetFileReader.cpp
View File

@ -22,7 +22,8 @@ std::unique_ptr<techset::TechsetDefinition> TechsetFileReader::ReadTechsetDefini
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = false;
lexerConfig.m_read_integer_numbers = false;
lexerConfig.m_read_floating_point_numbers = false;
const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig));
const auto parser = std::make_unique<techset::TechsetParser>(lexer.get(), m_valid_technique_type_names, m_valid_technique_type_name_count);

3
src/ObjWriting/Menu/AbstractMenuDumper.cpp
View File

@ -70,7 +70,8 @@ std::vector<std::string> AbstractMenuDumper::CreateScriptTokenList(const char* s
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = false;
lexerConfig.m_read_integer_numbers = false;
lexerConfig.m_read_floating_point_numbers = false;
SimpleLexer lexer(&inputStream, std::move(lexerConfig));
std::vector<std::string> result;

3
src/Parser/Parsing/Impl/DefinesStreamProxy.cpp
View File

@ -285,7 +285,8 @@ std::unique_ptr<ISimpleExpression> DefinesStreamProxy::ParseIfExpression(const s
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_numbers = true;
lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
lexerConfig.m_read_strings = false;
SimpleExpressionMatchers().ApplyTokensToLexerConfig(lexerConfig);

34
src/Parser/Parsing/Simple/SimpleLexer.cpp
View File

@ -14,7 +14,7 @@ SimpleLexer::MultiCharacterTokenLookupEntry::MultiCharacterTokenLookupEntry(cons
SimpleLexer::SimpleLexer(IParserLineStream* stream)
: AbstractLexer(stream),
m_config{false, true, true, {}},
m_config{false, true, true, true, {}},
m_check_for_multi_character_tokens(false),
m_last_line(1)
{
@ -29,6 +29,9 @@ SimpleLexer::SimpleLexer(IParserLineStream* stream, Config config)
for (auto tokenConfig : m_config.m_multi_character_tokens)
AddMultiCharacterTokenConfigToLookup(std::move(tokenConfig));
m_config.m_multi_character_tokens.clear();
// If reading floating point numbers then must be reading integers
assert(config.m_read_floating_point_numbers == false || config.m_read_floating_point_numbers == config.m_read_integer_numbers);
}
void SimpleLexer::AddMultiCharacterTokenConfigToLookup(Config::MultiCharacterToken tokenConfig)
@ -105,12 +108,12 @@ SimpleParserValue SimpleLexer::GetNextToken()
if (c == EOF)
return SimpleParserValue::EndOfFile(TokenPos());
if(m_check_for_multi_character_tokens)
if (m_check_for_multi_character_tokens)
{
const auto* multiTokenLookup = m_multi_character_token_lookup[static_cast<uint8_t>(c)].get();
while(multiTokenLookup)
while (multiTokenLookup)
{
if(ReadMultiCharacterToken(multiTokenLookup))
if (ReadMultiCharacterToken(multiTokenLookup))
return SimpleParserValue::MultiCharacter(pos, multiTokenLookup->m_id);
multiTokenLookup = multiTokenLookup->m_next.get();
@ -120,19 +123,24 @@ SimpleParserValue SimpleLexer::GetNextToken()
if (m_config.m_read_strings && c == '\"')
return SimpleParserValue::String(pos, new std::string(ReadString()));
if (m_config.m_read_numbers && (isdigit(c) || (c == '+' || c == '-' || c == '.') && isdigit(PeekChar())))
if (m_config.m_read_integer_numbers && (isdigit(c) || (c == '+' || c == '-' || (m_config.m_read_floating_point_numbers && c == '.')) && isdigit(PeekChar())))
{
bool isFloatingPointValue;
bool hasSignPrefix;
double doubleValue;
int integerValue;
if(m_config.m_read_floating_point_numbers)
{
bool isFloatingPointValue;
bool hasSignPrefix;
double floatingPointValue;
int integerValue;
ReadNumber(isFloatingPointValue, hasSignPrefix, doubleValue, integerValue);
ReadNumber(isFloatingPointValue, hasSignPrefix, floatingPointValue, integerValue);
if (isFloatingPointValue)
return SimpleParserValue::FloatingPoint(pos, doubleValue, hasSignPrefix);
if (isFloatingPointValue)
return SimpleParserValue::FloatingPoint(pos, floatingPointValue, hasSignPrefix);
return SimpleParserValue::Integer(pos, integerValue, hasSignPrefix);
return SimpleParserValue::Integer(pos, integerValue, hasSignPrefix);
}
return SimpleParserValue::Integer(pos, ReadInteger(), c == '+' || c == '-');
}
if (isalpha(c) || c == '_')

3
src/Parser/Parsing/Simple/SimpleLexer.h
View File

@ -24,7 +24,8 @@ public:
bool m_emit_new_line_tokens;
bool m_read_strings;
bool m_read_numbers;
bool m_read_integer_numbers;
bool m_read_floating_point_numbers;
std::vector<MultiCharacterToken> m_multi_character_tokens;
};

3
test/ParserTests/Parsing/Simple/SimpleExpressionTests.cpp
View File

@ -111,7 +111,8 @@ namespace test::parsing::simple::expression
SimpleLexer::Config lexerConfig;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true;
lexerConfig.m_read_integer_numbers = true;
lexerConfig.m_read_floating_point_numbers = true;
lexerConfig.m_emit_new_line_tokens = false;
SimpleExpressionMatchers(true, true, true, true, true).ApplyTokensToLexerConfig(lexerConfig);
m_lexer = std::make_unique<SimpleLexer>(m_stream.get(), std::move(lexerConfig));