zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-09-07 09:17:26 +00:00
Code Activity

Add multicharacter tokens to simple lexer

Browse Source
This commit is contained in:
Jan
2021-11-03 22:02:04 +01:00
parent ab7b516918
commit c5d7d71a51
11 changed files with 181 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp
View File

@@ -29,7 +29,12 @@ void LocalizeFileReader::SetupStreamProxies()
std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
{
const auto lexer = std::make_unique<SimpleLexer>(m_stream, SimpleLexer::Config{true, true, false});
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = true;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = false;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language);
if (parser->Parse())