From 216125739cfd56f3d2c3ef251e48bb9b00e06332 Mon Sep 17 00:00:00 2001 From: Jan Date: Sat, 13 Feb 2021 23:16:19 +0100 Subject: [PATCH] Cache lines in Lexer and show original input when running into an error --- .../Parsing/Header/Impl/HeaderLexer.cpp | 8 +- .../Parsing/Header/Impl/HeaderParserValue.cpp | 10 +- src/ZoneCodeGeneratorLib/Parsing/ILexer.h | 2 + .../Parsing/IParserLineStream.h | 1 + .../Parsing/Impl/AbstractLexer.h | 173 +++++++++++------- .../Parsing/Impl/AbstractParser.h | 12 +- .../Impl/CommentRemovingStreamProxy.cpp | 5 + .../Parsing/Impl/CommentRemovingStreamProxy.h | 1 + .../Parsing/Impl/DefinesStreamProxy.cpp | 5 + .../Parsing/Impl/DefinesStreamProxy.h | 1 + .../Parsing/Impl/IncludingStreamProxy.cpp | 100 ++++++---- .../Parsing/Impl/IncludingStreamProxy.h | 14 +- .../Parsing/Impl/ParserFilesystemStream.cpp | 10 +- .../Parsing/Impl/ParserFilesystemStream.h | 1 + .../Parsing/ParsingException.cpp | 2 +- src/ZoneCodeGeneratorLib/Parsing/TokenPos.h | 2 +- .../Parsing/Impl/IncludingStreamProxyTest.cpp | 45 +++++ .../Parsing/Mock/MockParserLineStream.cpp | 6 + .../Parsing/Mock/MockParserLineStream.h | 1 + 19 files changed, 283 insertions(+), 116 deletions(-) diff --git a/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.cpp b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.cpp index 36aac358..7258518a 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.cpp @@ -129,9 +129,9 @@ HeaderParserValue HeaderLexer::GetNextToken() if (isspace(c)) break; + const auto pos = GetPreviousCharacterPos(); if(isdigit(c)) { - const auto pos = GetPreviousCharacterPos(); bool isFloatingPointValue; double doubleValue; int integerValue; @@ -150,12 +150,12 @@ HeaderParserValue HeaderLexer::GetNextToken() const auto foundKeyword = m_keywords.find(identifier); if (foundKeyword != m_keywords.end()) - return HeaderParserValue::Keyword(GetPreviousCharacterPos(), foundKeyword->second); + return HeaderParserValue::Keyword(pos, foundKeyword->second); - return HeaderParserValue::Identifier(GetPreviousCharacterPos(), new std::string(std::move(identifier))); + return HeaderParserValue::Identifier(pos, new std::string(std::move(identifier))); } - return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast(c)); + return HeaderParserValue::Character(pos, static_cast(c)); } } diff --git a/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.cpp b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.cpp index 5eb1db58..c8906820 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.cpp @@ -111,8 +111,8 @@ HeaderParserValue HeaderParserValue::TypeName(const TokenPos pos, std::string* t HeaderParserValue::HeaderParserValue(const TokenPos pos, const HeaderParserValueType type) : m_pos(pos), - m_type(type), - m_value() + m_type(type), + m_value() { } @@ -134,14 +134,16 @@ HeaderParserValue::~HeaderParserValue() } HeaderParserValue::HeaderParserValue(HeaderParserValue&& other) noexcept - : m_type(other.m_type), - m_value(other.m_value) + : m_pos(other.m_pos), + m_type(other.m_type), + m_value(other.m_value) { other.m_value = ValueType(); } HeaderParserValue& HeaderParserValue::operator=(HeaderParserValue&& other) noexcept { + m_pos = other.m_pos; m_type = other.m_type; m_value = other.m_value; other.m_value = ValueType(); diff --git a/src/ZoneCodeGeneratorLib/Parsing/ILexer.h b/src/ZoneCodeGeneratorLib/Parsing/ILexer.h index da94842d..e84249fe 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/ILexer.h +++ b/src/ZoneCodeGeneratorLib/Parsing/ILexer.h @@ -1,5 +1,6 @@ #pragma once +#include "IParserLineStream.h" #include "Parsing/IParserValue.h" template @@ -21,4 +22,5 @@ public: _NODISCARD virtual bool IsEof() = 0; _NODISCARD virtual const TokenPos& GetPos() = 0; + _NODISCARD virtual ParserLine GetLineForPos(const TokenPos& pos) const = 0; }; diff --git a/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.h b/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.h index 1c1bb8d4..b53c4cba 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.h +++ b/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.h @@ -32,6 +32,7 @@ public: virtual ParserLine NextLine() = 0; virtual bool IncludeFile(const std::string& filename) = 0; + virtual void PopCurrentFile() = 0; _NODISCARD virtual bool IsOpen() const = 0; _NODISCARD virtual bool Eof() const = 0; }; diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/AbstractLexer.h b/src/ZoneCodeGeneratorLib/Parsing/Impl/AbstractLexer.h index a0c6eba8..798a8ba6 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/AbstractLexer.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/AbstractLexer.h @@ -16,19 +16,16 @@ class AbstractLexer : public ILexer protected: std::deque m_token_cache; + std::deque m_line_cache; IParserLineStream* const m_stream; + unsigned m_line_index; unsigned m_current_line_offset; - bool m_peeked_next_line; - bool m_start; - ParserLine m_current_line; - ParserLine m_next_line; explicit AbstractLexer(IParserLineStream* stream) : m_stream(stream), - m_current_line_offset(0u), - m_peeked_next_line(false), - m_start(true) + m_line_index(0u), + m_current_line_offset(0u) { } @@ -36,60 +33,63 @@ protected: int NextChar() { - if (m_current_line.IsEof()) + while (true) { - if (m_start) - m_start = false; - else - return EOF; - } + while (m_line_index >= m_line_cache.size()) + { + if (!m_line_cache.empty() && m_line_cache.back().IsEof()) + return EOF; - while (m_current_line_offset >= m_current_line.m_line.size()) - { - m_current_line_offset = 0; - if (m_peeked_next_line) - { - m_current_line = m_next_line; - m_peeked_next_line = false; - } - else - { - m_current_line = m_stream->NextLine(); + m_line_cache.push_back(m_stream->NextLine()); } - if (m_current_line.IsEof()) - return EOF; + if (m_current_line_offset >= m_line_cache[m_line_index].m_line.size()) + { + m_line_index++; + m_current_line_offset = 0; + } + else + break; } - return m_current_line.m_line[m_current_line_offset++]; + return m_line_cache[m_line_index].m_line[m_current_line_offset++]; } int PeekChar() { - if (m_current_line.IsEof()) - return EOF; + auto peekLineOffset = m_current_line_offset; + auto peekLine = m_line_index; - if (m_current_line_offset >= m_current_line.m_line.size()) + while (true) { - m_peeked_next_line = true; - - do + while (peekLine >= m_line_cache.size()) { - m_next_line = m_stream->NextLine(); - if (m_next_line.IsEof()) + if (m_line_cache.back().IsEof()) return EOF; - } - while (m_next_line.m_line.empty()); - return m_next_line.m_line[0]; + m_line_cache.push_back(m_stream->NextLine()); + } + + if (peekLineOffset >= m_line_cache[peekLine].m_line.size()) + { + peekLine++; + peekLineOffset = 0; + } + else + break; } - return m_current_line.m_line[m_current_line_offset]; + return m_line_cache[peekLine].m_line[peekLineOffset]; + } + + _NODISCARD const ParserLine& CurrentLine() const + { + return m_line_cache[m_line_index]; } _NODISCARD bool IsLineEnd() const { - return m_current_line_offset >= m_current_line.m_line.size(); + return m_current_line_offset >= CurrentLine().m_line.size(); } _NODISCARD bool NextCharInLineIs(const char c) @@ -99,18 +99,20 @@ protected: _NODISCARD TokenPos GetPreviousCharacterPos() const { - return TokenPos(m_current_line.m_filename, m_current_line.m_line_number, m_current_line_offset); + const auto& currentLine = CurrentLine(); + return TokenPos(currentLine.m_filename, currentLine.m_line_number, m_current_line_offset); } _NODISCARD TokenPos GetNextCharacterPos() { - if (m_current_line_offset + 1 >= m_current_line.m_line.size()) + const auto& currentLine = CurrentLine(); + if (m_current_line_offset + 1 >= currentLine.m_line.size()) { PeekChar(); - return TokenPos(m_next_line.m_filename, m_next_line.m_line_number, 1); + return TokenPos(m_line_cache.back().m_filename, m_line_cache.back().m_line_number, 1); } - return TokenPos(m_current_line.m_filename, m_current_line.m_line_number, m_current_line_offset + 1); + return TokenPos(currentLine.m_filename, currentLine.m_line_number, m_current_line_offset + 1); } /** @@ -119,14 +121,15 @@ protected: */ std::string ReadIdentifier() { + const auto& currentLine = CurrentLine(); assert(m_current_line_offset >= 1); - assert(isalpha(m_current_line.m_line[m_current_line_offset - 1]) || m_current_line.m_line[m_current_line_offset - 1] == '_'); + assert(isalpha(currentLine.m_line[m_current_line_offset - 1]) || currentLine.m_line[m_current_line_offset - 1] == '_'); const auto startPos = m_current_line_offset - 1; - const auto lineSize = m_current_line.m_line.size(); + const auto lineSize = currentLine.m_line.size(); while (m_current_line_offset < lineSize) { - const auto c = m_current_line.m_line[m_current_line_offset]; + const auto c = currentLine.m_line[m_current_line_offset]; if (!isalnum(c) && c != '_') break; @@ -134,7 +137,7 @@ protected: m_current_line_offset++; } - return std::string(m_current_line.m_line, startPos, m_current_line_offset - startPos); + return std::string(currentLine.m_line, startPos, m_current_line_offset - startPos); } /** @@ -143,28 +146,30 @@ protected: */ std::string ReadString() { + const auto& currentLine = CurrentLine(); assert(m_current_line_offset >= 1); - assert(m_current_line.m_line[m_current_line_offset - 1] == '"'); + assert(currentLine.m_line[m_current_line_offset - 1] == '"'); const auto startPos = m_current_line_offset; - const auto lineSize = m_current_line.m_line.size(); + const auto lineSize = currentLine.m_line.size(); while (true) { if (m_current_line_offset >= lineSize) - throw ParsingException(TokenPos(m_current_line.m_filename, m_current_line.m_line_number, m_current_line_offset), "Unclosed string"); + throw ParsingException(TokenPos(currentLine.m_filename, currentLine.m_line_number, m_current_line_offset), "Unclosed string"); - if (m_current_line.m_line[m_current_line_offset] == '\"') + if (currentLine.m_line[m_current_line_offset] == '\"') break; m_current_line_offset++; } - - return std::string(m_current_line.m_line, startPos, m_current_line_offset++ - startPos); + + return std::string(currentLine.m_line, startPos, m_current_line_offset++ - startPos); } void ReadHexNumber(int& integerValue) { - const auto* start = &m_current_line.m_line.c_str()[m_current_line_offset - 1]; + const auto& currentLine = CurrentLine(); + const auto* start = ¤tLine.m_line.c_str()[m_current_line_offset - 1]; char* end; integerValue = static_cast(std::strtoul(start, &end, 16)); @@ -177,7 +182,8 @@ protected: _NODISCARD bool IsIntegerNumber() const { - const auto* currentCharacter = &m_current_line.m_line.c_str()[m_current_line_offset - 1]; + const auto& currentLine = CurrentLine(); + const auto* currentCharacter = ¤tLine.m_line.c_str()[m_current_line_offset - 1]; auto isInteger = true; auto dot = false; auto exponent = false; @@ -223,12 +229,13 @@ protected: int ReadInteger() { - const auto* start = &m_current_line.m_line.c_str()[m_current_line_offset - 1]; + const auto& currentLine = CurrentLine(); + const auto* start = ¤tLine.m_line.c_str()[m_current_line_offset - 1]; char* end; const auto integerValue = std::strtol(start, &end, 10); const auto numberLength = static_cast(end - start); - if(numberLength == 0) + if (numberLength == 0) throw ParsingException(GetPreviousCharacterPos(), "Invalid number"); m_current_line_offset += numberLength - 1; @@ -238,7 +245,8 @@ protected: double ReadFloatingPoint() { - const auto* start = &m_current_line.m_line.c_str()[m_current_line_offset - 1]; + const auto& currentLine = CurrentLine(); + const auto* start = ¤tLine.m_line.c_str()[m_current_line_offset - 1]; char* end; const auto floatingPointValue = std::strtod(start, &end); const auto numberLength = static_cast(end - start); @@ -253,13 +261,14 @@ protected: void ReadNumber(bool& isFloatingPoint, double& floatingPointValue, int& integerValue) { + const auto& currentLine = CurrentLine(); assert(m_current_line_offset >= 1); - assert(isdigit(m_current_line.m_line[m_current_line_offset - 1])); + assert(isdigit(currentLine.m_line[m_current_line_offset - 1])); - const auto lineLength = m_current_line.m_line.size(); + const auto lineLength = currentLine.m_line.size(); if (lineLength - m_current_line_offset >= 1 - && m_current_line.m_line[m_current_line_offset - 1] == '0' - && m_current_line.m_line[m_current_line_offset] == 'x') + && currentLine.m_line[m_current_line_offset - 1] == '0' + && currentLine.m_line[m_current_line_offset] == 'x') { isFloatingPoint = false; ReadHexNumber(integerValue); @@ -288,7 +297,31 @@ public: void PopTokens(int amount) override { - m_token_cache.erase(m_token_cache.begin(), m_token_cache.begin() + amount); + if (amount <= 0 || m_token_cache.empty()) + return; + + if (static_cast(m_token_cache.size()) <= amount) + { + const auto& lastToken = m_token_cache.back(); + while (m_line_cache.front().m_line_number != lastToken.GetPos().m_line + || m_line_cache.front().m_filename.get() != lastToken.GetPos().m_filename.get()) + { + m_line_cache.pop_front(); + m_line_index--; + } + m_token_cache.clear(); + } + else + { + m_token_cache.erase(m_token_cache.begin(), m_token_cache.begin() + amount); + const auto& firstToken = m_token_cache.front(); + while (m_line_cache.front().m_line_number != firstToken.GetPos().m_line + || m_line_cache.front().m_filename.get() != firstToken.GetPos().m_filename.get()) + { + m_line_cache.pop_front(); + m_line_index--; + } + } } _NODISCARD bool IsEof() override @@ -300,4 +333,16 @@ public: { return GetToken(0).GetPos(); } + + _NODISCARD ParserLine GetLineForPos(const TokenPos& pos) const override + { + for(const auto& line : m_line_cache) + { + if (line.m_filename.get() == pos.m_filename.get() + && line.m_line_number == pos.m_line) + return line; + } + + return ParserLine(); + } }; diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/AbstractParser.h b/src/ZoneCodeGeneratorLib/Parsing/Impl/AbstractParser.h index 7fa3ff65..011a6b99 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/AbstractParser.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/AbstractParser.h @@ -60,7 +60,17 @@ public: if (!testSuccessful) { const TokenPos& pos = m_lexer->GetPos(); - std::cout << "Error: " << pos.m_filename << " L" << pos.m_line << ':' << pos.m_column << " Could not parse expression." << std::endl; + const ParserLine line = m_lexer->GetLineForPos(pos); + + if (!line.IsEof()) + { + std::cout << "Error: " << pos.m_filename.get() << " L" << pos.m_line << ':' << pos.m_column << " Could not parse expression:\n" + << line.m_line.substr(pos.m_column - 1) << std::endl; + } + else + { + std::cout << "Error: " << pos.m_filename.get() << " L" << pos.m_line << ':' << pos.m_column << " Could not parse expression." << std::endl; + } return false; } } diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/CommentRemovingStreamProxy.cpp b/src/ZoneCodeGeneratorLib/Parsing/Impl/CommentRemovingStreamProxy.cpp index f43772f0..9f222567 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/CommentRemovingStreamProxy.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/CommentRemovingStreamProxy.cpp @@ -63,6 +63,11 @@ bool CommentRemovingStreamProxy::IncludeFile(const std::string& filename) return m_stream->IncludeFile(filename); } +void CommentRemovingStreamProxy::PopCurrentFile() +{ + m_stream->PopCurrentFile(); +} + bool CommentRemovingStreamProxy::IsOpen() const { return m_stream->IsOpen(); diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/CommentRemovingStreamProxy.h b/src/ZoneCodeGeneratorLib/Parsing/Impl/CommentRemovingStreamProxy.h index bdbb5ad9..d9d9bfc5 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/CommentRemovingStreamProxy.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/CommentRemovingStreamProxy.h @@ -13,6 +13,7 @@ public: ParserLine NextLine() override; bool IncludeFile(const std::string& filename) override; + void PopCurrentFile() override; _NODISCARD bool IsOpen() const override; _NODISCARD bool Eof() const override; }; diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/DefinesStreamProxy.cpp b/src/ZoneCodeGeneratorLib/Parsing/Impl/DefinesStreamProxy.cpp index 0b75bd68..09b5912b 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/DefinesStreamProxy.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/DefinesStreamProxy.cpp @@ -290,6 +290,11 @@ bool DefinesStreamProxy::IncludeFile(const std::string& filename) return m_stream->IncludeFile(filename); } +void DefinesStreamProxy::PopCurrentFile() +{ + m_stream->PopCurrentFile(); +} + bool DefinesStreamProxy::IsOpen() const { return m_stream->IsOpen(); diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/DefinesStreamProxy.h b/src/ZoneCodeGeneratorLib/Parsing/Impl/DefinesStreamProxy.h index 64b5cf03..ce7bb2fc 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/DefinesStreamProxy.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/DefinesStreamProxy.h @@ -38,6 +38,7 @@ public: ParserLine NextLine() override; bool IncludeFile(const std::string& filename) override; + void PopCurrentFile() override; _NODISCARD bool IsOpen() const override; _NODISCARD bool Eof() const override; }; diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/IncludingStreamProxy.cpp b/src/ZoneCodeGeneratorLib/Parsing/Impl/IncludingStreamProxy.cpp index c141f65f..1f470745 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/IncludingStreamProxy.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/IncludingStreamProxy.cpp @@ -1,46 +1,25 @@ #include "IncludingStreamProxy.h" #include +#include #include "Parsing/ParsingException.h" +namespace fs = std::filesystem; + IncludingStreamProxy::IncludingStreamProxy(IParserLineStream* stream) : m_stream(stream) { } -bool IncludingStreamProxy::FindIncludeDirective(const ParserLine& line, unsigned& includeDirectivePosition) -{ - includeDirectivePosition = 0; - for (; includeDirectivePosition < line.m_line.size() - INCLUDE_DIRECTIVE_MINIMUM_TOTAL_LENGTH; includeDirectivePosition++) - { - const auto c = line.m_line[includeDirectivePosition]; - - if (isspace(c)) - continue; - - if (c != '#') - return false; - - if (line.m_line.compare(includeDirectivePosition + 1, INCLUDE_DIRECTIVE_LENGTH, INCLUDE_DIRECTIVE) != 0) - { - return false; - } - - return true; - } - - return false; -} - bool IncludingStreamProxy::ExtractIncludeFilename(const ParserLine& line, const unsigned includeDirectivePosition, unsigned& filenameStartPosition, unsigned& filenameEndPosition) { auto currentPos = includeDirectivePosition; bool isDoubleQuotes; - while(isspace(line.m_line[currentPos])) + while (isspace(line.m_line[currentPos])) { - if(currentPos++ >= line.m_line.size()) + if (currentPos++ >= line.m_line.size()) throw ParsingException(TokenPos(line.m_filename, line.m_line_number, currentPos), INCLUDE_QUOTES_ERROR); } @@ -78,25 +57,24 @@ bool IncludingStreamProxy::ExtractIncludeFilename(const ParserLine& line, const return false; } -bool IncludingStreamProxy::MatchIncludeDirective(const ParserLine& line) const +bool IncludingStreamProxy::MatchIncludeDirective(const ParserLine& line, const unsigned directivePosition) const { - unsigned includeDirectivePos; - - if (!FindIncludeDirective(line, includeDirectivePos)) + constexpr auto directiveLength = std::char_traits::length(INCLUDE_DIRECTIVE); + if (line.m_line.compare(directivePosition + 1, directiveLength, INCLUDE_DIRECTIVE) != 0) return false; - const auto currentPos = includeDirectivePos + INCLUDE_DIRECTIVE_LENGTH + 1; + const auto currentPos = directivePosition + directiveLength + 1; unsigned filenameStart, filenameEnd; - if(!ExtractIncludeFilename(line, currentPos, filenameStart, filenameEnd)) + if (!ExtractIncludeFilename(line, currentPos, filenameStart, filenameEnd)) throw ParsingException(TokenPos(line.m_filename, line.m_line_number, currentPos), INCLUDE_QUOTES_ERROR); - if(filenameEnd <= filenameStart) + if (filenameEnd <= filenameStart) throw ParsingException(TokenPos(line.m_filename, line.m_line_number, currentPos), "No filename specified"); const auto filename = line.m_line.substr(filenameStart, filenameEnd - filenameStart); - if(!m_stream->IncludeFile(filename)) + if (!m_stream->IncludeFile(filename)) { std::ostringstream errorStr; errorStr << "Could not include file \"" << filename << "\""; @@ -106,12 +84,57 @@ bool IncludingStreamProxy::MatchIncludeDirective(const ParserLine& line) const return true; } +bool IncludingStreamProxy::MatchPragmaOnceDirective(const ParserLine& line, const unsigned directivePosition) +{ + constexpr auto directiveLength = std::char_traits::length(PRAGMA_ONCE_DIRECTIVE); + if (line.m_line.compare(directivePosition + 1, directiveLength, PRAGMA_ONCE_DIRECTIVE) != 0) + return false; + + const auto absolutePath = absolute(fs::path(line.m_filename.get())); + const auto absolutePathStr = absolutePath.string(); + + const auto existingPath = m_included_files.find(absolutePathStr); + if (existingPath != m_included_files.end()) + m_stream->PopCurrentFile(); + else + m_included_files.emplace(absolutePathStr); + + return true; +} + +bool IncludingStreamProxy::FindDirective(const ParserLine& line, unsigned& directivePosition) +{ + directivePosition = 0; + for (; directivePosition < line.m_line.size(); directivePosition++) + { + const auto c = line.m_line[directivePosition]; + + if (isspace(c)) + continue; + + return c == '#'; + } + + return false; +} + +bool IncludingStreamProxy::MatchDirectives(const ParserLine& line) +{ + unsigned directivePos; + + if (!FindDirective(line, directivePos)) + return false; + + return MatchIncludeDirective(line, directivePos) + || MatchPragmaOnceDirective(line, directivePos); +} + ParserLine IncludingStreamProxy::NextLine() { auto line = m_stream->NextLine(); - if (MatchIncludeDirective(line)) - return m_stream->NextLine(); + while(MatchDirectives(line)) + line = m_stream->NextLine(); return line; } @@ -121,6 +144,11 @@ bool IncludingStreamProxy::IncludeFile(const std::string& filename) return m_stream->IncludeFile(filename); } +void IncludingStreamProxy::PopCurrentFile() +{ + m_stream->PopCurrentFile(); +} + bool IncludingStreamProxy::IsOpen() const { return m_stream->IsOpen(); diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/IncludingStreamProxy.h b/src/ZoneCodeGeneratorLib/Parsing/Impl/IncludingStreamProxy.h index ef921a05..2c253f63 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/IncludingStreamProxy.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/IncludingStreamProxy.h @@ -1,25 +1,31 @@ #pragma once +#include + #include "Parsing/IParserLineStream.h" class IncludingStreamProxy final : public IParserLineStream { static constexpr const char* INCLUDE_QUOTES_ERROR = "Invalid include directive. Expected \"\" or <>"; static constexpr const char* INCLUDE_DIRECTIVE = "include "; - static constexpr int INCLUDE_DIRECTIVE_LENGTH = std::char_traits::length(INCLUDE_DIRECTIVE); - static constexpr int INCLUDE_DIRECTIVE_MINIMUM_TOTAL_LENGTH = INCLUDE_DIRECTIVE_LENGTH + 1 + 2; // #=+1 ""=+2 + static constexpr const char* PRAGMA_ONCE_DIRECTIVE = "pragma once"; IParserLineStream* const m_stream; + std::set m_included_files; - _NODISCARD static bool FindIncludeDirective(const ParserLine& line, unsigned& includeDirectivePosition); + _NODISCARD static bool ExtractIncludeFilename(const ParserLine& line, unsigned includeDirectivePosition, unsigned& filenameStartPosition, unsigned& filenameEndPosition); - _NODISCARD bool MatchIncludeDirective(const ParserLine& line) const; + _NODISCARD bool MatchIncludeDirective(const ParserLine& line, unsigned directivePosition) const; + _NODISCARD bool MatchPragmaOnceDirective(const ParserLine& line, unsigned directivePosition); + _NODISCARD static bool FindDirective(const ParserLine& line, unsigned& directivePosition); + _NODISCARD bool MatchDirectives(const ParserLine& line); public: explicit IncludingStreamProxy(IParserLineStream* stream); ParserLine NextLine() override; bool IncludeFile(const std::string& filename) override; + void PopCurrentFile() override; _NODISCARD bool IsOpen() const override; _NODISCARD bool Eof() const override; }; diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.cpp b/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.cpp index d7bd2993..a7569dcf 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.cpp @@ -14,7 +14,8 @@ ParserFilesystemStream::FileInfo::FileInfo(std::string filePath) ParserFilesystemStream::ParserFilesystemStream(std::string path) { - m_files.emplace(FileInfo(std::move(path))); + const auto absolutePath = absolute(fs::path(path)); + m_files.emplace(FileInfo(absolutePath.string())); } bool ParserFilesystemStream::IsOpen() const @@ -75,6 +76,7 @@ bool ParserFilesystemStream::IncludeFile(const std::string& filename) auto newFilePath = fs::path(m_files.top().m_file_path); newFilePath.remove_filename().concat(filename); + newFilePath = absolute(newFilePath); FileInfo fileInfo(newFilePath.string()); @@ -85,6 +87,12 @@ bool ParserFilesystemStream::IncludeFile(const std::string& filename) return true; } +void ParserFilesystemStream::PopCurrentFile() +{ + if(!m_files.empty()) + m_files.pop(); +} + bool ParserFilesystemStream::Eof() const { return m_files.empty() diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.h b/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.h index b010cff5..bdca77e5 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.h @@ -23,6 +23,7 @@ public: ParserLine NextLine() override; bool IncludeFile(const std::string& filename) override; + void PopCurrentFile() override; _NODISCARD bool IsOpen() const override; _NODISCARD bool Eof() const override; }; diff --git a/src/ZoneCodeGeneratorLib/Parsing/ParsingException.cpp b/src/ZoneCodeGeneratorLib/Parsing/ParsingException.cpp index ccad2836..f48f6bf7 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/ParsingException.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/ParsingException.cpp @@ -7,7 +7,7 @@ ParsingException::ParsingException(const TokenPos position, std::string message) m_message(std::move(message)) { std::ostringstream str; - str << position.m_filename << " L" << m_pos.m_line << ':' << m_pos.m_column << ' ' << m_message; + str << position.m_filename.get() << " L" << m_pos.m_line << ':' << m_pos.m_column << ' ' << m_message; m_full_message = str.str(); } diff --git a/src/ZoneCodeGeneratorLib/Parsing/TokenPos.h b/src/ZoneCodeGeneratorLib/Parsing/TokenPos.h index 1abc47f7..cf513580 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/TokenPos.h +++ b/src/ZoneCodeGeneratorLib/Parsing/TokenPos.h @@ -7,7 +7,7 @@ class TokenPos static const std::string EMPTY_FILENAME; public: - const std::string& m_filename; + std::reference_wrapper m_filename; int m_line; int m_column; diff --git a/test/ZoneCodeGeneratorLibTests/Parsing/Impl/IncludingStreamProxyTest.cpp b/test/ZoneCodeGeneratorLibTests/Parsing/Impl/IncludingStreamProxyTest.cpp index aa86b1ae..b225b33c 100644 --- a/test/ZoneCodeGeneratorLibTests/Parsing/Impl/IncludingStreamProxyTest.cpp +++ b/test/ZoneCodeGeneratorLibTests/Parsing/Impl/IncludingStreamProxyTest.cpp @@ -133,4 +133,49 @@ namespace test::parsing::impl::including_stream_proxy REQUIRE(proxy.Eof()); } + + TEST_CASE("IncludingStreamProxy: Ensure pragma once prevents including the same file more than once", "[parsing][parsingstream]") + { + const std::vector lines + { + "Hello world", + "#include \"ASDF.txt\"", + "#include \"ASDF.txt\"", + "and bye" + }; + + const std::vector asdf + { + "#pragma once", + "Hello galaxy" + }; + + MockParserLineStream mockStream(lines); + mockStream.AddIncludeLines("ASDF.txt", asdf); + + IncludingStreamProxy proxy(&mockStream); + + { + auto line = proxy.NextLine(); + REQUIRE(line.m_line_number == 1); + REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME); + REQUIRE(line.m_line == "Hello world"); + } + + { + auto line = proxy.NextLine(); + REQUIRE(line.m_line_number == 2); + REQUIRE(line.m_filename.get() == "ASDF.txt"); + REQUIRE(line.m_line == "Hello galaxy"); + } + + { + auto line = proxy.NextLine(); + REQUIRE(line.m_line_number == 4); + REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME); + REQUIRE(line.m_line == "and bye"); + } + + REQUIRE(proxy.Eof()); + } } diff --git a/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.cpp b/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.cpp index ad373a5c..12d03a11 100644 --- a/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.cpp +++ b/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.cpp @@ -50,6 +50,12 @@ bool MockParserLineStream::IncludeFile(const std::string& filename) return true; } +void MockParserLineStream::PopCurrentFile() +{ + if (!m_include_positions.empty()) + m_include_positions.pop_back(); +} + bool MockParserLineStream::IsOpen() const { return true; diff --git a/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.h b/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.h index 70a7dc62..23ac1997 100644 --- a/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.h +++ b/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.h @@ -30,6 +30,7 @@ public: void AddIncludeLines(const std::string& filename, const std::vector& lines); ParserLine NextLine() override; bool IncludeFile(const std::string& filename) override; + void PopCurrentFile() override; _NODISCARD bool IsOpen() const override; _NODISCARD bool Eof() const override; };