zone/OpenAssetTools
zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-04-20 16:15:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix simple lexer not recognizing newlines after spaces

Browse Source
This commit is contained in:
Jan 2021-07-10 10:53:18 +02:00
parent eedfccc617
commit c6ea52018a
1 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/Parser/Parsing/Simple/SimpleLexer.cpp
View File

@ -26,7 +26,7 @@ void SimpleLexer::SetShouldReadNumbers(const bool value)
SimpleParserValue SimpleLexer::GetNextToken() SimpleParserValue SimpleLexer::GetNextToken()
{ {
PeekChar(); auto c = PeekChar();
const auto nextCharPos = GetNextCharacterPos(); const auto nextCharPos = GetNextCharacterPos();
if (m_emit_new_line_tokens && nextCharPos.m_line > m_last_line) if (m_emit_new_line_tokens && nextCharPos.m_line > m_last_line)
{ {
@ -34,23 +34,30 @@ SimpleParserValue SimpleLexer::GetNextToken()
return SimpleParserValue::NewLine(GetPreviousCharacterPos()); return SimpleParserValue::NewLine(GetPreviousCharacterPos());
} }
auto c = NextChar();
while (isspace(c)) while (isspace(c))
{ {
if (m_emit_new_line_tokens && c == '\n') if (m_emit_new_line_tokens && c == '\n')
return SimpleParserValue::NewLine(GetPreviousCharacterPos()); return SimpleParserValue::NewLine(GetPreviousCharacterPos());
c = NextChar(); NextChar();
c = PeekChar();
} }
const auto pos = GetNextCharacterPos();
if (m_emit_new_line_tokens && pos.m_line > m_last_line)
{
m_last_line++;
return SimpleParserValue::NewLine(GetPreviousCharacterPos());
}
c = NextChar();
if (c == EOF) if (c == EOF)
return SimpleParserValue::EndOfFile(TokenPos()); return SimpleParserValue::EndOfFile(TokenPos());
if (m_read_strings && c == '\"') if (m_read_strings && c == '\"')
return SimpleParserValue::String(GetPreviousCharacterPos(), new std::string(ReadString())); return SimpleParserValue::String(GetPreviousCharacterPos(), new std::string(ReadString()));
const auto pos = GetPreviousCharacterPos();
if (m_read_numbers && isdigit(c)) if (m_read_numbers && isdigit(c))
{ {
bool isFloatingPointValue; bool isFloatingPointValue;