mirror of
https://github.com/Laupetin/OpenAssetTools.git
synced 2025-04-21 00:25:44 +00:00
Cache lines in Lexer and show original input when running into an error
This commit is contained in:
parent
40fedc905d
commit
216125739c
@ -129,9 +129,9 @@ HeaderParserValue HeaderLexer::GetNextToken()
|
|||||||
if (isspace(c))
|
if (isspace(c))
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
const auto pos = GetPreviousCharacterPos();
|
||||||
if(isdigit(c))
|
if(isdigit(c))
|
||||||
{
|
{
|
||||||
const auto pos = GetPreviousCharacterPos();
|
|
||||||
bool isFloatingPointValue;
|
bool isFloatingPointValue;
|
||||||
double doubleValue;
|
double doubleValue;
|
||||||
int integerValue;
|
int integerValue;
|
||||||
@ -150,12 +150,12 @@ HeaderParserValue HeaderLexer::GetNextToken()
|
|||||||
|
|
||||||
const auto foundKeyword = m_keywords.find(identifier);
|
const auto foundKeyword = m_keywords.find(identifier);
|
||||||
if (foundKeyword != m_keywords.end())
|
if (foundKeyword != m_keywords.end())
|
||||||
return HeaderParserValue::Keyword(GetPreviousCharacterPos(), foundKeyword->second);
|
return HeaderParserValue::Keyword(pos, foundKeyword->second);
|
||||||
|
|
||||||
return HeaderParserValue::Identifier(GetPreviousCharacterPos(), new std::string(std::move(identifier)));
|
return HeaderParserValue::Identifier(pos, new std::string(std::move(identifier)));
|
||||||
}
|
}
|
||||||
|
|
||||||
return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast<char>(c));
|
return HeaderParserValue::Character(pos, static_cast<char>(c));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -134,7 +134,8 @@ HeaderParserValue::~HeaderParserValue()
|
|||||||
}
|
}
|
||||||
|
|
||||||
HeaderParserValue::HeaderParserValue(HeaderParserValue&& other) noexcept
|
HeaderParserValue::HeaderParserValue(HeaderParserValue&& other) noexcept
|
||||||
: m_type(other.m_type),
|
: m_pos(other.m_pos),
|
||||||
|
m_type(other.m_type),
|
||||||
m_value(other.m_value)
|
m_value(other.m_value)
|
||||||
{
|
{
|
||||||
other.m_value = ValueType();
|
other.m_value = ValueType();
|
||||||
@ -142,6 +143,7 @@ HeaderParserValue::HeaderParserValue(HeaderParserValue&& other) noexcept
|
|||||||
|
|
||||||
HeaderParserValue& HeaderParserValue::operator=(HeaderParserValue&& other) noexcept
|
HeaderParserValue& HeaderParserValue::operator=(HeaderParserValue&& other) noexcept
|
||||||
{
|
{
|
||||||
|
m_pos = other.m_pos;
|
||||||
m_type = other.m_type;
|
m_type = other.m_type;
|
||||||
m_value = other.m_value;
|
m_value = other.m_value;
|
||||||
other.m_value = ValueType();
|
other.m_value = ValueType();
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include "IParserLineStream.h"
|
||||||
#include "Parsing/IParserValue.h"
|
#include "Parsing/IParserValue.h"
|
||||||
|
|
||||||
template<typename TokenType>
|
template<typename TokenType>
|
||||||
@ -21,4 +22,5 @@ public:
|
|||||||
|
|
||||||
_NODISCARD virtual bool IsEof() = 0;
|
_NODISCARD virtual bool IsEof() = 0;
|
||||||
_NODISCARD virtual const TokenPos& GetPos() = 0;
|
_NODISCARD virtual const TokenPos& GetPos() = 0;
|
||||||
|
_NODISCARD virtual ParserLine GetLineForPos(const TokenPos& pos) const = 0;
|
||||||
};
|
};
|
||||||
|
@ -32,6 +32,7 @@ public:
|
|||||||
|
|
||||||
virtual ParserLine NextLine() = 0;
|
virtual ParserLine NextLine() = 0;
|
||||||
virtual bool IncludeFile(const std::string& filename) = 0;
|
virtual bool IncludeFile(const std::string& filename) = 0;
|
||||||
|
virtual void PopCurrentFile() = 0;
|
||||||
_NODISCARD virtual bool IsOpen() const = 0;
|
_NODISCARD virtual bool IsOpen() const = 0;
|
||||||
_NODISCARD virtual bool Eof() const = 0;
|
_NODISCARD virtual bool Eof() const = 0;
|
||||||
};
|
};
|
||||||
|
@ -16,19 +16,16 @@ class AbstractLexer : public ILexer<TokenType>
|
|||||||
|
|
||||||
protected:
|
protected:
|
||||||
std::deque<TokenType> m_token_cache;
|
std::deque<TokenType> m_token_cache;
|
||||||
|
std::deque<ParserLine> m_line_cache;
|
||||||
IParserLineStream* const m_stream;
|
IParserLineStream* const m_stream;
|
||||||
|
|
||||||
|
unsigned m_line_index;
|
||||||
unsigned m_current_line_offset;
|
unsigned m_current_line_offset;
|
||||||
bool m_peeked_next_line;
|
|
||||||
bool m_start;
|
|
||||||
ParserLine m_current_line;
|
|
||||||
ParserLine m_next_line;
|
|
||||||
|
|
||||||
explicit AbstractLexer(IParserLineStream* stream)
|
explicit AbstractLexer(IParserLineStream* stream)
|
||||||
: m_stream(stream),
|
: m_stream(stream),
|
||||||
m_current_line_offset(0u),
|
m_line_index(0u),
|
||||||
m_peeked_next_line(false),
|
m_current_line_offset(0u)
|
||||||
m_start(true)
|
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -36,60 +33,63 @@ protected:
|
|||||||
|
|
||||||
int NextChar()
|
int NextChar()
|
||||||
{
|
{
|
||||||
if (m_current_line.IsEof())
|
while (true)
|
||||||
{
|
{
|
||||||
if (m_start)
|
while (m_line_index >= m_line_cache.size())
|
||||||
m_start = false;
|
{
|
||||||
else
|
if (!m_line_cache.empty() && m_line_cache.back().IsEof())
|
||||||
return EOF;
|
return EOF;
|
||||||
|
|
||||||
|
m_line_cache.push_back(m_stream->NextLine());
|
||||||
}
|
}
|
||||||
|
|
||||||
while (m_current_line_offset >= m_current_line.m_line.size())
|
if (m_current_line_offset >= m_line_cache[m_line_index].m_line.size())
|
||||||
{
|
{
|
||||||
|
m_line_index++;
|
||||||
m_current_line_offset = 0;
|
m_current_line_offset = 0;
|
||||||
if (m_peeked_next_line)
|
|
||||||
{
|
|
||||||
m_current_line = m_next_line;
|
|
||||||
m_peeked_next_line = false;
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
break;
|
||||||
m_current_line = m_stream->NextLine();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (m_current_line.IsEof())
|
return m_line_cache[m_line_index].m_line[m_current_line_offset++];
|
||||||
return EOF;
|
|
||||||
}
|
|
||||||
|
|
||||||
return m_current_line.m_line[m_current_line_offset++];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int PeekChar()
|
int PeekChar()
|
||||||
{
|
{
|
||||||
if (m_current_line.IsEof())
|
auto peekLineOffset = m_current_line_offset;
|
||||||
|
auto peekLine = m_line_index;
|
||||||
|
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
while (peekLine >= m_line_cache.size())
|
||||||
|
{
|
||||||
|
if (m_line_cache.back().IsEof())
|
||||||
return EOF;
|
return EOF;
|
||||||
|
|
||||||
if (m_current_line_offset >= m_current_line.m_line.size())
|
m_line_cache.push_back(m_stream->NextLine());
|
||||||
{
|
|
||||||
m_peeked_next_line = true;
|
|
||||||
|
|
||||||
do
|
|
||||||
{
|
|
||||||
m_next_line = m_stream->NextLine();
|
|
||||||
if (m_next_line.IsEof())
|
|
||||||
return EOF;
|
|
||||||
}
|
|
||||||
while (m_next_line.m_line.empty());
|
|
||||||
|
|
||||||
return m_next_line.m_line[0];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return m_current_line.m_line[m_current_line_offset];
|
if (peekLineOffset >= m_line_cache[peekLine].m_line.size())
|
||||||
|
{
|
||||||
|
peekLine++;
|
||||||
|
peekLineOffset = 0;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return m_line_cache[peekLine].m_line[peekLineOffset];
|
||||||
|
}
|
||||||
|
|
||||||
|
_NODISCARD const ParserLine& CurrentLine() const
|
||||||
|
{
|
||||||
|
return m_line_cache[m_line_index];
|
||||||
}
|
}
|
||||||
|
|
||||||
_NODISCARD bool IsLineEnd() const
|
_NODISCARD bool IsLineEnd() const
|
||||||
{
|
{
|
||||||
return m_current_line_offset >= m_current_line.m_line.size();
|
return m_current_line_offset >= CurrentLine().m_line.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
_NODISCARD bool NextCharInLineIs(const char c)
|
_NODISCARD bool NextCharInLineIs(const char c)
|
||||||
@ -99,18 +99,20 @@ protected:
|
|||||||
|
|
||||||
_NODISCARD TokenPos GetPreviousCharacterPos() const
|
_NODISCARD TokenPos GetPreviousCharacterPos() const
|
||||||
{
|
{
|
||||||
return TokenPos(m_current_line.m_filename, m_current_line.m_line_number, m_current_line_offset);
|
const auto& currentLine = CurrentLine();
|
||||||
|
return TokenPos(currentLine.m_filename, currentLine.m_line_number, m_current_line_offset);
|
||||||
}
|
}
|
||||||
|
|
||||||
_NODISCARD TokenPos GetNextCharacterPos()
|
_NODISCARD TokenPos GetNextCharacterPos()
|
||||||
{
|
{
|
||||||
if (m_current_line_offset + 1 >= m_current_line.m_line.size())
|
const auto& currentLine = CurrentLine();
|
||||||
|
if (m_current_line_offset + 1 >= currentLine.m_line.size())
|
||||||
{
|
{
|
||||||
PeekChar();
|
PeekChar();
|
||||||
return TokenPos(m_next_line.m_filename, m_next_line.m_line_number, 1);
|
return TokenPos(m_line_cache.back().m_filename, m_line_cache.back().m_line_number, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
return TokenPos(m_current_line.m_filename, m_current_line.m_line_number, m_current_line_offset + 1);
|
return TokenPos(currentLine.m_filename, currentLine.m_line_number, m_current_line_offset + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -119,14 +121,15 @@ protected:
|
|||||||
*/
|
*/
|
||||||
std::string ReadIdentifier()
|
std::string ReadIdentifier()
|
||||||
{
|
{
|
||||||
|
const auto& currentLine = CurrentLine();
|
||||||
assert(m_current_line_offset >= 1);
|
assert(m_current_line_offset >= 1);
|
||||||
assert(isalpha(m_current_line.m_line[m_current_line_offset - 1]) || m_current_line.m_line[m_current_line_offset - 1] == '_');
|
assert(isalpha(currentLine.m_line[m_current_line_offset - 1]) || currentLine.m_line[m_current_line_offset - 1] == '_');
|
||||||
|
|
||||||
const auto startPos = m_current_line_offset - 1;
|
const auto startPos = m_current_line_offset - 1;
|
||||||
const auto lineSize = m_current_line.m_line.size();
|
const auto lineSize = currentLine.m_line.size();
|
||||||
while (m_current_line_offset < lineSize)
|
while (m_current_line_offset < lineSize)
|
||||||
{
|
{
|
||||||
const auto c = m_current_line.m_line[m_current_line_offset];
|
const auto c = currentLine.m_line[m_current_line_offset];
|
||||||
|
|
||||||
if (!isalnum(c) && c != '_')
|
if (!isalnum(c) && c != '_')
|
||||||
break;
|
break;
|
||||||
@ -134,7 +137,7 @@ protected:
|
|||||||
m_current_line_offset++;
|
m_current_line_offset++;
|
||||||
}
|
}
|
||||||
|
|
||||||
return std::string(m_current_line.m_line, startPos, m_current_line_offset - startPos);
|
return std::string(currentLine.m_line, startPos, m_current_line_offset - startPos);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -143,28 +146,30 @@ protected:
|
|||||||
*/
|
*/
|
||||||
std::string ReadString()
|
std::string ReadString()
|
||||||
{
|
{
|
||||||
|
const auto& currentLine = CurrentLine();
|
||||||
assert(m_current_line_offset >= 1);
|
assert(m_current_line_offset >= 1);
|
||||||
assert(m_current_line.m_line[m_current_line_offset - 1] == '"');
|
assert(currentLine.m_line[m_current_line_offset - 1] == '"');
|
||||||
|
|
||||||
const auto startPos = m_current_line_offset;
|
const auto startPos = m_current_line_offset;
|
||||||
const auto lineSize = m_current_line.m_line.size();
|
const auto lineSize = currentLine.m_line.size();
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
if (m_current_line_offset >= lineSize)
|
if (m_current_line_offset >= lineSize)
|
||||||
throw ParsingException(TokenPos(m_current_line.m_filename, m_current_line.m_line_number, m_current_line_offset), "Unclosed string");
|
throw ParsingException(TokenPos(currentLine.m_filename, currentLine.m_line_number, m_current_line_offset), "Unclosed string");
|
||||||
|
|
||||||
if (m_current_line.m_line[m_current_line_offset] == '\"')
|
if (currentLine.m_line[m_current_line_offset] == '\"')
|
||||||
break;
|
break;
|
||||||
|
|
||||||
m_current_line_offset++;
|
m_current_line_offset++;
|
||||||
}
|
}
|
||||||
|
|
||||||
return std::string(m_current_line.m_line, startPos, m_current_line_offset++ - startPos);
|
return std::string(currentLine.m_line, startPos, m_current_line_offset++ - startPos);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ReadHexNumber(int& integerValue)
|
void ReadHexNumber(int& integerValue)
|
||||||
{
|
{
|
||||||
const auto* start = &m_current_line.m_line.c_str()[m_current_line_offset - 1];
|
const auto& currentLine = CurrentLine();
|
||||||
|
const auto* start = ¤tLine.m_line.c_str()[m_current_line_offset - 1];
|
||||||
char* end;
|
char* end;
|
||||||
|
|
||||||
integerValue = static_cast<int>(std::strtoul(start, &end, 16));
|
integerValue = static_cast<int>(std::strtoul(start, &end, 16));
|
||||||
@ -177,7 +182,8 @@ protected:
|
|||||||
|
|
||||||
_NODISCARD bool IsIntegerNumber() const
|
_NODISCARD bool IsIntegerNumber() const
|
||||||
{
|
{
|
||||||
const auto* currentCharacter = &m_current_line.m_line.c_str()[m_current_line_offset - 1];
|
const auto& currentLine = CurrentLine();
|
||||||
|
const auto* currentCharacter = ¤tLine.m_line.c_str()[m_current_line_offset - 1];
|
||||||
auto isInteger = true;
|
auto isInteger = true;
|
||||||
auto dot = false;
|
auto dot = false;
|
||||||
auto exponent = false;
|
auto exponent = false;
|
||||||
@ -223,7 +229,8 @@ protected:
|
|||||||
|
|
||||||
int ReadInteger()
|
int ReadInteger()
|
||||||
{
|
{
|
||||||
const auto* start = &m_current_line.m_line.c_str()[m_current_line_offset - 1];
|
const auto& currentLine = CurrentLine();
|
||||||
|
const auto* start = ¤tLine.m_line.c_str()[m_current_line_offset - 1];
|
||||||
char* end;
|
char* end;
|
||||||
const auto integerValue = std::strtol(start, &end, 10);
|
const auto integerValue = std::strtol(start, &end, 10);
|
||||||
const auto numberLength = static_cast<unsigned>(end - start);
|
const auto numberLength = static_cast<unsigned>(end - start);
|
||||||
@ -238,7 +245,8 @@ protected:
|
|||||||
|
|
||||||
double ReadFloatingPoint()
|
double ReadFloatingPoint()
|
||||||
{
|
{
|
||||||
const auto* start = &m_current_line.m_line.c_str()[m_current_line_offset - 1];
|
const auto& currentLine = CurrentLine();
|
||||||
|
const auto* start = ¤tLine.m_line.c_str()[m_current_line_offset - 1];
|
||||||
char* end;
|
char* end;
|
||||||
const auto floatingPointValue = std::strtod(start, &end);
|
const auto floatingPointValue = std::strtod(start, &end);
|
||||||
const auto numberLength = static_cast<unsigned>(end - start);
|
const auto numberLength = static_cast<unsigned>(end - start);
|
||||||
@ -253,13 +261,14 @@ protected:
|
|||||||
|
|
||||||
void ReadNumber(bool& isFloatingPoint, double& floatingPointValue, int& integerValue)
|
void ReadNumber(bool& isFloatingPoint, double& floatingPointValue, int& integerValue)
|
||||||
{
|
{
|
||||||
|
const auto& currentLine = CurrentLine();
|
||||||
assert(m_current_line_offset >= 1);
|
assert(m_current_line_offset >= 1);
|
||||||
assert(isdigit(m_current_line.m_line[m_current_line_offset - 1]));
|
assert(isdigit(currentLine.m_line[m_current_line_offset - 1]));
|
||||||
|
|
||||||
const auto lineLength = m_current_line.m_line.size();
|
const auto lineLength = currentLine.m_line.size();
|
||||||
if (lineLength - m_current_line_offset >= 1
|
if (lineLength - m_current_line_offset >= 1
|
||||||
&& m_current_line.m_line[m_current_line_offset - 1] == '0'
|
&& currentLine.m_line[m_current_line_offset - 1] == '0'
|
||||||
&& m_current_line.m_line[m_current_line_offset] == 'x')
|
&& currentLine.m_line[m_current_line_offset] == 'x')
|
||||||
{
|
{
|
||||||
isFloatingPoint = false;
|
isFloatingPoint = false;
|
||||||
ReadHexNumber(integerValue);
|
ReadHexNumber(integerValue);
|
||||||
@ -287,8 +296,32 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
void PopTokens(int amount) override
|
void PopTokens(int amount) override
|
||||||
|
{
|
||||||
|
if (amount <= 0 || m_token_cache.empty())
|
||||||
|
return;
|
||||||
|
|
||||||
|
if (static_cast<int>(m_token_cache.size()) <= amount)
|
||||||
|
{
|
||||||
|
const auto& lastToken = m_token_cache.back();
|
||||||
|
while (m_line_cache.front().m_line_number != lastToken.GetPos().m_line
|
||||||
|
|| m_line_cache.front().m_filename.get() != lastToken.GetPos().m_filename.get())
|
||||||
|
{
|
||||||
|
m_line_cache.pop_front();
|
||||||
|
m_line_index--;
|
||||||
|
}
|
||||||
|
m_token_cache.clear();
|
||||||
|
}
|
||||||
|
else
|
||||||
{
|
{
|
||||||
m_token_cache.erase(m_token_cache.begin(), m_token_cache.begin() + amount);
|
m_token_cache.erase(m_token_cache.begin(), m_token_cache.begin() + amount);
|
||||||
|
const auto& firstToken = m_token_cache.front();
|
||||||
|
while (m_line_cache.front().m_line_number != firstToken.GetPos().m_line
|
||||||
|
|| m_line_cache.front().m_filename.get() != firstToken.GetPos().m_filename.get())
|
||||||
|
{
|
||||||
|
m_line_cache.pop_front();
|
||||||
|
m_line_index--;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_NODISCARD bool IsEof() override
|
_NODISCARD bool IsEof() override
|
||||||
@ -300,4 +333,16 @@ public:
|
|||||||
{
|
{
|
||||||
return GetToken(0).GetPos();
|
return GetToken(0).GetPos();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_NODISCARD ParserLine GetLineForPos(const TokenPos& pos) const override
|
||||||
|
{
|
||||||
|
for(const auto& line : m_line_cache)
|
||||||
|
{
|
||||||
|
if (line.m_filename.get() == pos.m_filename.get()
|
||||||
|
&& line.m_line_number == pos.m_line)
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
|
||||||
|
return ParserLine();
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
@ -60,7 +60,17 @@ public:
|
|||||||
if (!testSuccessful)
|
if (!testSuccessful)
|
||||||
{
|
{
|
||||||
const TokenPos& pos = m_lexer->GetPos();
|
const TokenPos& pos = m_lexer->GetPos();
|
||||||
std::cout << "Error: " << pos.m_filename << " L" << pos.m_line << ':' << pos.m_column << " Could not parse expression." << std::endl;
|
const ParserLine line = m_lexer->GetLineForPos(pos);
|
||||||
|
|
||||||
|
if (!line.IsEof())
|
||||||
|
{
|
||||||
|
std::cout << "Error: " << pos.m_filename.get() << " L" << pos.m_line << ':' << pos.m_column << " Could not parse expression:\n"
|
||||||
|
<< line.m_line.substr(pos.m_column - 1) << std::endl;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
std::cout << "Error: " << pos.m_filename.get() << " L" << pos.m_line << ':' << pos.m_column << " Could not parse expression." << std::endl;
|
||||||
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -63,6 +63,11 @@ bool CommentRemovingStreamProxy::IncludeFile(const std::string& filename)
|
|||||||
return m_stream->IncludeFile(filename);
|
return m_stream->IncludeFile(filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void CommentRemovingStreamProxy::PopCurrentFile()
|
||||||
|
{
|
||||||
|
m_stream->PopCurrentFile();
|
||||||
|
}
|
||||||
|
|
||||||
bool CommentRemovingStreamProxy::IsOpen() const
|
bool CommentRemovingStreamProxy::IsOpen() const
|
||||||
{
|
{
|
||||||
return m_stream->IsOpen();
|
return m_stream->IsOpen();
|
||||||
|
@ -13,6 +13,7 @@ public:
|
|||||||
|
|
||||||
ParserLine NextLine() override;
|
ParserLine NextLine() override;
|
||||||
bool IncludeFile(const std::string& filename) override;
|
bool IncludeFile(const std::string& filename) override;
|
||||||
|
void PopCurrentFile() override;
|
||||||
_NODISCARD bool IsOpen() const override;
|
_NODISCARD bool IsOpen() const override;
|
||||||
_NODISCARD bool Eof() const override;
|
_NODISCARD bool Eof() const override;
|
||||||
};
|
};
|
||||||
|
@ -290,6 +290,11 @@ bool DefinesStreamProxy::IncludeFile(const std::string& filename)
|
|||||||
return m_stream->IncludeFile(filename);
|
return m_stream->IncludeFile(filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void DefinesStreamProxy::PopCurrentFile()
|
||||||
|
{
|
||||||
|
m_stream->PopCurrentFile();
|
||||||
|
}
|
||||||
|
|
||||||
bool DefinesStreamProxy::IsOpen() const
|
bool DefinesStreamProxy::IsOpen() const
|
||||||
{
|
{
|
||||||
return m_stream->IsOpen();
|
return m_stream->IsOpen();
|
||||||
|
@ -38,6 +38,7 @@ public:
|
|||||||
|
|
||||||
ParserLine NextLine() override;
|
ParserLine NextLine() override;
|
||||||
bool IncludeFile(const std::string& filename) override;
|
bool IncludeFile(const std::string& filename) override;
|
||||||
|
void PopCurrentFile() override;
|
||||||
_NODISCARD bool IsOpen() const override;
|
_NODISCARD bool IsOpen() const override;
|
||||||
_NODISCARD bool Eof() const override;
|
_NODISCARD bool Eof() const override;
|
||||||
};
|
};
|
||||||
|
@ -1,38 +1,17 @@
|
|||||||
#include "IncludingStreamProxy.h"
|
#include "IncludingStreamProxy.h"
|
||||||
|
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
|
#include <filesystem>
|
||||||
|
|
||||||
#include "Parsing/ParsingException.h"
|
#include "Parsing/ParsingException.h"
|
||||||
|
|
||||||
|
namespace fs = std::filesystem;
|
||||||
|
|
||||||
IncludingStreamProxy::IncludingStreamProxy(IParserLineStream* stream)
|
IncludingStreamProxy::IncludingStreamProxy(IParserLineStream* stream)
|
||||||
: m_stream(stream)
|
: m_stream(stream)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
bool IncludingStreamProxy::FindIncludeDirective(const ParserLine& line, unsigned& includeDirectivePosition)
|
|
||||||
{
|
|
||||||
includeDirectivePosition = 0;
|
|
||||||
for (; includeDirectivePosition < line.m_line.size() - INCLUDE_DIRECTIVE_MINIMUM_TOTAL_LENGTH; includeDirectivePosition++)
|
|
||||||
{
|
|
||||||
const auto c = line.m_line[includeDirectivePosition];
|
|
||||||
|
|
||||||
if (isspace(c))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
if (c != '#')
|
|
||||||
return false;
|
|
||||||
|
|
||||||
if (line.m_line.compare(includeDirectivePosition + 1, INCLUDE_DIRECTIVE_LENGTH, INCLUDE_DIRECTIVE) != 0)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool IncludingStreamProxy::ExtractIncludeFilename(const ParserLine& line, const unsigned includeDirectivePosition, unsigned& filenameStartPosition, unsigned& filenameEndPosition)
|
bool IncludingStreamProxy::ExtractIncludeFilename(const ParserLine& line, const unsigned includeDirectivePosition, unsigned& filenameStartPosition, unsigned& filenameEndPosition)
|
||||||
{
|
{
|
||||||
auto currentPos = includeDirectivePosition;
|
auto currentPos = includeDirectivePosition;
|
||||||
@ -78,14 +57,13 @@ bool IncludingStreamProxy::ExtractIncludeFilename(const ParserLine& line, const
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool IncludingStreamProxy::MatchIncludeDirective(const ParserLine& line) const
|
bool IncludingStreamProxy::MatchIncludeDirective(const ParserLine& line, const unsigned directivePosition) const
|
||||||
{
|
{
|
||||||
unsigned includeDirectivePos;
|
constexpr auto directiveLength = std::char_traits<char>::length(INCLUDE_DIRECTIVE);
|
||||||
|
if (line.m_line.compare(directivePosition + 1, directiveLength, INCLUDE_DIRECTIVE) != 0)
|
||||||
if (!FindIncludeDirective(line, includeDirectivePos))
|
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
const auto currentPos = includeDirectivePos + INCLUDE_DIRECTIVE_LENGTH + 1;
|
const auto currentPos = directivePosition + directiveLength + 1;
|
||||||
unsigned filenameStart, filenameEnd;
|
unsigned filenameStart, filenameEnd;
|
||||||
|
|
||||||
if (!ExtractIncludeFilename(line, currentPos, filenameStart, filenameEnd))
|
if (!ExtractIncludeFilename(line, currentPos, filenameStart, filenameEnd))
|
||||||
@ -106,12 +84,57 @@ bool IncludingStreamProxy::MatchIncludeDirective(const ParserLine& line) const
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool IncludingStreamProxy::MatchPragmaOnceDirective(const ParserLine& line, const unsigned directivePosition)
|
||||||
|
{
|
||||||
|
constexpr auto directiveLength = std::char_traits<char>::length(PRAGMA_ONCE_DIRECTIVE);
|
||||||
|
if (line.m_line.compare(directivePosition + 1, directiveLength, PRAGMA_ONCE_DIRECTIVE) != 0)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
const auto absolutePath = absolute(fs::path(line.m_filename.get()));
|
||||||
|
const auto absolutePathStr = absolutePath.string();
|
||||||
|
|
||||||
|
const auto existingPath = m_included_files.find(absolutePathStr);
|
||||||
|
if (existingPath != m_included_files.end())
|
||||||
|
m_stream->PopCurrentFile();
|
||||||
|
else
|
||||||
|
m_included_files.emplace(absolutePathStr);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool IncludingStreamProxy::FindDirective(const ParserLine& line, unsigned& directivePosition)
|
||||||
|
{
|
||||||
|
directivePosition = 0;
|
||||||
|
for (; directivePosition < line.m_line.size(); directivePosition++)
|
||||||
|
{
|
||||||
|
const auto c = line.m_line[directivePosition];
|
||||||
|
|
||||||
|
if (isspace(c))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
return c == '#';
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool IncludingStreamProxy::MatchDirectives(const ParserLine& line)
|
||||||
|
{
|
||||||
|
unsigned directivePos;
|
||||||
|
|
||||||
|
if (!FindDirective(line, directivePos))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
return MatchIncludeDirective(line, directivePos)
|
||||||
|
|| MatchPragmaOnceDirective(line, directivePos);
|
||||||
|
}
|
||||||
|
|
||||||
ParserLine IncludingStreamProxy::NextLine()
|
ParserLine IncludingStreamProxy::NextLine()
|
||||||
{
|
{
|
||||||
auto line = m_stream->NextLine();
|
auto line = m_stream->NextLine();
|
||||||
|
|
||||||
if (MatchIncludeDirective(line))
|
while(MatchDirectives(line))
|
||||||
return m_stream->NextLine();
|
line = m_stream->NextLine();
|
||||||
|
|
||||||
return line;
|
return line;
|
||||||
}
|
}
|
||||||
@ -121,6 +144,11 @@ bool IncludingStreamProxy::IncludeFile(const std::string& filename)
|
|||||||
return m_stream->IncludeFile(filename);
|
return m_stream->IncludeFile(filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void IncludingStreamProxy::PopCurrentFile()
|
||||||
|
{
|
||||||
|
m_stream->PopCurrentFile();
|
||||||
|
}
|
||||||
|
|
||||||
bool IncludingStreamProxy::IsOpen() const
|
bool IncludingStreamProxy::IsOpen() const
|
||||||
{
|
{
|
||||||
return m_stream->IsOpen();
|
return m_stream->IsOpen();
|
||||||
|
@ -1,25 +1,31 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include <set>
|
||||||
|
|
||||||
#include "Parsing/IParserLineStream.h"
|
#include "Parsing/IParserLineStream.h"
|
||||||
|
|
||||||
class IncludingStreamProxy final : public IParserLineStream
|
class IncludingStreamProxy final : public IParserLineStream
|
||||||
{
|
{
|
||||||
static constexpr const char* INCLUDE_QUOTES_ERROR = "Invalid include directive. Expected \"\" or <>";
|
static constexpr const char* INCLUDE_QUOTES_ERROR = "Invalid include directive. Expected \"\" or <>";
|
||||||
static constexpr const char* INCLUDE_DIRECTIVE = "include ";
|
static constexpr const char* INCLUDE_DIRECTIVE = "include ";
|
||||||
static constexpr int INCLUDE_DIRECTIVE_LENGTH = std::char_traits<char>::length(INCLUDE_DIRECTIVE);
|
static constexpr const char* PRAGMA_ONCE_DIRECTIVE = "pragma once";
|
||||||
static constexpr int INCLUDE_DIRECTIVE_MINIMUM_TOTAL_LENGTH = INCLUDE_DIRECTIVE_LENGTH + 1 + 2; // #=+1 ""=+2
|
|
||||||
|
|
||||||
IParserLineStream* const m_stream;
|
IParserLineStream* const m_stream;
|
||||||
|
std::set<std::string> m_included_files;
|
||||||
|
|
||||||
|
|
||||||
_NODISCARD static bool FindIncludeDirective(const ParserLine& line, unsigned& includeDirectivePosition);
|
|
||||||
_NODISCARD static bool ExtractIncludeFilename(const ParserLine& line, unsigned includeDirectivePosition, unsigned& filenameStartPosition, unsigned& filenameEndPosition);
|
_NODISCARD static bool ExtractIncludeFilename(const ParserLine& line, unsigned includeDirectivePosition, unsigned& filenameStartPosition, unsigned& filenameEndPosition);
|
||||||
_NODISCARD bool MatchIncludeDirective(const ParserLine& line) const;
|
_NODISCARD bool MatchIncludeDirective(const ParserLine& line, unsigned directivePosition) const;
|
||||||
|
_NODISCARD bool MatchPragmaOnceDirective(const ParserLine& line, unsigned directivePosition);
|
||||||
|
_NODISCARD static bool FindDirective(const ParserLine& line, unsigned& directivePosition);
|
||||||
|
_NODISCARD bool MatchDirectives(const ParserLine& line);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
explicit IncludingStreamProxy(IParserLineStream* stream);
|
explicit IncludingStreamProxy(IParserLineStream* stream);
|
||||||
|
|
||||||
ParserLine NextLine() override;
|
ParserLine NextLine() override;
|
||||||
bool IncludeFile(const std::string& filename) override;
|
bool IncludeFile(const std::string& filename) override;
|
||||||
|
void PopCurrentFile() override;
|
||||||
_NODISCARD bool IsOpen() const override;
|
_NODISCARD bool IsOpen() const override;
|
||||||
_NODISCARD bool Eof() const override;
|
_NODISCARD bool Eof() const override;
|
||||||
};
|
};
|
||||||
|
@ -14,7 +14,8 @@ ParserFilesystemStream::FileInfo::FileInfo(std::string filePath)
|
|||||||
|
|
||||||
ParserFilesystemStream::ParserFilesystemStream(std::string path)
|
ParserFilesystemStream::ParserFilesystemStream(std::string path)
|
||||||
{
|
{
|
||||||
m_files.emplace(FileInfo(std::move(path)));
|
const auto absolutePath = absolute(fs::path(path));
|
||||||
|
m_files.emplace(FileInfo(absolutePath.string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ParserFilesystemStream::IsOpen() const
|
bool ParserFilesystemStream::IsOpen() const
|
||||||
@ -75,6 +76,7 @@ bool ParserFilesystemStream::IncludeFile(const std::string& filename)
|
|||||||
|
|
||||||
auto newFilePath = fs::path(m_files.top().m_file_path);
|
auto newFilePath = fs::path(m_files.top().m_file_path);
|
||||||
newFilePath.remove_filename().concat(filename);
|
newFilePath.remove_filename().concat(filename);
|
||||||
|
newFilePath = absolute(newFilePath);
|
||||||
|
|
||||||
FileInfo fileInfo(newFilePath.string());
|
FileInfo fileInfo(newFilePath.string());
|
||||||
|
|
||||||
@ -85,6 +87,12 @@ bool ParserFilesystemStream::IncludeFile(const std::string& filename)
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ParserFilesystemStream::PopCurrentFile()
|
||||||
|
{
|
||||||
|
if(!m_files.empty())
|
||||||
|
m_files.pop();
|
||||||
|
}
|
||||||
|
|
||||||
bool ParserFilesystemStream::Eof() const
|
bool ParserFilesystemStream::Eof() const
|
||||||
{
|
{
|
||||||
return m_files.empty()
|
return m_files.empty()
|
||||||
|
@ -23,6 +23,7 @@ public:
|
|||||||
|
|
||||||
ParserLine NextLine() override;
|
ParserLine NextLine() override;
|
||||||
bool IncludeFile(const std::string& filename) override;
|
bool IncludeFile(const std::string& filename) override;
|
||||||
|
void PopCurrentFile() override;
|
||||||
_NODISCARD bool IsOpen() const override;
|
_NODISCARD bool IsOpen() const override;
|
||||||
_NODISCARD bool Eof() const override;
|
_NODISCARD bool Eof() const override;
|
||||||
};
|
};
|
||||||
|
@ -7,7 +7,7 @@ ParsingException::ParsingException(const TokenPos position, std::string message)
|
|||||||
m_message(std::move(message))
|
m_message(std::move(message))
|
||||||
{
|
{
|
||||||
std::ostringstream str;
|
std::ostringstream str;
|
||||||
str << position.m_filename << " L" << m_pos.m_line << ':' << m_pos.m_column << ' ' << m_message;
|
str << position.m_filename.get() << " L" << m_pos.m_line << ':' << m_pos.m_column << ' ' << m_message;
|
||||||
m_full_message = str.str();
|
m_full_message = str.str();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -7,7 +7,7 @@ class TokenPos
|
|||||||
static const std::string EMPTY_FILENAME;
|
static const std::string EMPTY_FILENAME;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
const std::string& m_filename;
|
std::reference_wrapper<const std::string> m_filename;
|
||||||
int m_line;
|
int m_line;
|
||||||
int m_column;
|
int m_column;
|
||||||
|
|
||||||
|
@ -133,4 +133,49 @@ namespace test::parsing::impl::including_stream_proxy
|
|||||||
|
|
||||||
REQUIRE(proxy.Eof());
|
REQUIRE(proxy.Eof());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_CASE("IncludingStreamProxy: Ensure pragma once prevents including the same file more than once", "[parsing][parsingstream]")
|
||||||
|
{
|
||||||
|
const std::vector<std::string> lines
|
||||||
|
{
|
||||||
|
"Hello world",
|
||||||
|
"#include \"ASDF.txt\"",
|
||||||
|
"#include \"ASDF.txt\"",
|
||||||
|
"and bye"
|
||||||
|
};
|
||||||
|
|
||||||
|
const std::vector<std::string> asdf
|
||||||
|
{
|
||||||
|
"#pragma once",
|
||||||
|
"Hello galaxy"
|
||||||
|
};
|
||||||
|
|
||||||
|
MockParserLineStream mockStream(lines);
|
||||||
|
mockStream.AddIncludeLines("ASDF.txt", asdf);
|
||||||
|
|
||||||
|
IncludingStreamProxy proxy(&mockStream);
|
||||||
|
|
||||||
|
{
|
||||||
|
auto line = proxy.NextLine();
|
||||||
|
REQUIRE(line.m_line_number == 1);
|
||||||
|
REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME);
|
||||||
|
REQUIRE(line.m_line == "Hello world");
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
auto line = proxy.NextLine();
|
||||||
|
REQUIRE(line.m_line_number == 2);
|
||||||
|
REQUIRE(line.m_filename.get() == "ASDF.txt");
|
||||||
|
REQUIRE(line.m_line == "Hello galaxy");
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
auto line = proxy.NextLine();
|
||||||
|
REQUIRE(line.m_line_number == 4);
|
||||||
|
REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME);
|
||||||
|
REQUIRE(line.m_line == "and bye");
|
||||||
|
}
|
||||||
|
|
||||||
|
REQUIRE(proxy.Eof());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -50,6 +50,12 @@ bool MockParserLineStream::IncludeFile(const std::string& filename)
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void MockParserLineStream::PopCurrentFile()
|
||||||
|
{
|
||||||
|
if (!m_include_positions.empty())
|
||||||
|
m_include_positions.pop_back();
|
||||||
|
}
|
||||||
|
|
||||||
bool MockParserLineStream::IsOpen() const
|
bool MockParserLineStream::IsOpen() const
|
||||||
{
|
{
|
||||||
return true;
|
return true;
|
||||||
|
@ -30,6 +30,7 @@ public:
|
|||||||
void AddIncludeLines(const std::string& filename, const std::vector<std::string>& lines);
|
void AddIncludeLines(const std::string& filename, const std::vector<std::string>& lines);
|
||||||
ParserLine NextLine() override;
|
ParserLine NextLine() override;
|
||||||
bool IncludeFile(const std::string& filename) override;
|
bool IncludeFile(const std::string& filename) override;
|
||||||
|
void PopCurrentFile() override;
|
||||||
_NODISCARD bool IsOpen() const override;
|
_NODISCARD bool IsOpen() const override;
|
||||||
_NODISCARD bool Eof() const override;
|
_NODISCARD bool Eof() const override;
|
||||||
};
|
};
|
||||||
|
Loading…
x
Reference in New Issue
Block a user