zone/OpenAssetTools
2
0 Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2026-11-10 18:01:59 +00:00
Code Activity

Add multicharacter tokens to simple lexer

Browse Source
This commit is contained in:
Jan committed 2021-11-03 22:02:04 +01:00
1 parent ab7b516918
commit c5d7d71a51
11 files changed
+181 -11
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp
+6 -1
View File
@@ -29,7 +29,12 @@ void LocalizeFileReader::SetupStreamProxies()
std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
{
const auto lexer = std::make_unique<SimpleLexer>(m_stream, SimpleLexer::Config{true, true, false});
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = true;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = false;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language);
if (parser->Parse())
src/ObjLoading/Parsing/Menu/MenuFileReader.cpp
+6 -1
View File
@@ -111,7 +111,12 @@ std::unique_ptr<ParsingResult> MenuFileReader::CreateParsingResult(MenuFileParse
std::unique_ptr<ParsingResult> MenuFileReader::ReadMenuFile()
{
const auto lexer = std::make_unique<SimpleLexer>(m_stream, SimpleLexer::Config{false, true, true});
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<MenuFileParser>(lexer.get(), m_feature_level);
if (!parser->Parse())