From 87b7921c73c0cf4b3ea1c925d4855cdfaedb932d Mon Sep 17 00:00:00 2001 From: Jan Date: Fri, 12 Feb 2021 00:01:41 +0100 Subject: [PATCH] Add Header Lexer for ZCG cpp --- .../Parsing/AbstractLexer.h | 265 ++++++++- .../Parsing/Commands/CommandsFileReader.cpp | 50 +- .../Parsing/Commands/CommandsFileReader.h | 7 + .../Commands/Impl/CommandsParserValue.h | 3 +- .../Parsing/Header/HeaderFileReader.cpp | 85 ++- .../Parsing/Header/HeaderFileReader.h | 9 +- .../Parsing/Header/Impl/HeaderLexer.cpp | 144 +++++ .../Parsing/Header/Impl/HeaderLexer.h | 13 + .../Impl/HeaderParser.cpp} | 0 .../Parsing/Header/Impl/HeaderParser.h | 6 + .../Parsing/Header/Impl/HeaderParserValue.cpp | 181 ++++++ .../Parsing/Header/Impl/HeaderParserValue.h | 86 +++ .../Parsing/IParserLineStream.cpp | 8 + .../Parsing/IParserLineStream.h | 7 +- .../Parsing/Impl/ParserFilesystemStream.cpp | 6 +- .../Parsing/Impl/ParserFilesystemStream.h | 2 - .../Commands/Impl/CommandsLexerTests.cpp | 12 + .../Parsing/Header/Impl/HeaderLexerTests.cpp | 541 ++++++++++++++++++ .../Parsing/Impl/IncludingStreamProxyTest.cpp | 18 +- .../Parsing/Mock/MockParserLineStream.cpp | 2 +- 20 files changed, 1386 insertions(+), 59 deletions(-) create mode 100644 src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.cpp create mode 100644 src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.h rename src/ZoneCodeGeneratorLib/Parsing/{AbstractLexer.cpp => Header/Impl/HeaderParser.cpp} (100%) create mode 100644 src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParser.h create mode 100644 src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.cpp create mode 100644 src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.h create mode 100644 test/ZoneCodeGeneratorLibTests/Parsing/Commands/Impl/CommandsLexerTests.cpp create mode 100644 test/ZoneCodeGeneratorLibTests/Parsing/Header/Impl/HeaderLexerTests.cpp diff --git a/src/ZoneCodeGeneratorLib/Parsing/AbstractLexer.h b/src/ZoneCodeGeneratorLib/Parsing/AbstractLexer.h index 5cf9bd4b..a1d5413a 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/AbstractLexer.h +++ b/src/ZoneCodeGeneratorLib/Parsing/AbstractLexer.h @@ -3,29 +3,284 @@ #include #include +#include "Utils/ClassUtils.h" #include "ILexer.h" #include "IParserLineStream.h" +#include "ParsingException.h" template class AbstractLexer : public ILexer { +protected: std::deque m_token_cache; IParserLineStream* const m_stream; -protected: + unsigned m_current_line_offset; + bool m_peeked_next_line; + bool m_start; + ParserLine m_current_line; + ParserLine m_next_line; + explicit AbstractLexer(IParserLineStream* stream) - : m_stream(stream) + : m_stream(stream), + m_current_line_offset(0u), + m_peeked_next_line(false), + m_start(true) { } virtual TokenType GetNextToken() = 0; + int NextChar() + { + if (m_current_line.IsEof()) + { + if (m_start) + m_start = false; + else + return EOF; + } + + while (m_current_line_offset >= m_current_line.m_line.size()) + { + m_current_line_offset = 0; + if (m_peeked_next_line) + { + m_current_line = m_next_line; + m_peeked_next_line = false; + } + else + { + m_current_line = m_stream->NextLine(); + } + + if (m_current_line.IsEof()) + return EOF; + } + + return m_current_line.m_line[m_current_line_offset++]; + } + + int PeekChar() + { + if (m_current_line.IsEof()) + return EOF; + + if (m_current_line_offset >= m_current_line.m_line.size()) + { + m_peeked_next_line = true; + + do + { + m_next_line = m_stream->NextLine(); + if (m_next_line.IsEof()) + return EOF; + } + while (m_next_line.m_line.empty()); + + return m_next_line.m_line[0]; + } + + return m_current_line.m_line[m_current_line_offset]; + } + + _NODISCARD bool IsLineEnd() const + { + return m_current_line_offset >= m_current_line.m_line.size(); + } + + _NODISCARD bool NextCharInLineIs(const char c) + { + return !IsLineEnd() && PeekChar() == c; + } + + _NODISCARD TokenPos GetPreviousCharacterPos() const + { + return TokenPos(m_current_line.m_filename, m_current_line.m_line_number, m_current_line_offset); + } + + _NODISCARD TokenPos GetNextCharacterPos() + { + if (m_current_line_offset + 1 >= m_current_line.m_line.size()) + { + PeekChar(); + return TokenPos(m_next_line.m_filename, m_next_line.m_line_number, 1); + } + + return TokenPos(m_current_line.m_filename, m_current_line.m_line_number, m_current_line_offset + 1); + } + + /** + * \brief Reads an identifier from the current position + * \return The value of the read identifier + */ + std::string ReadIdentifier() + { + assert(m_current_line_offset >= 1); + assert(isalpha(m_current_line.m_line[m_current_line_offset - 1]) || m_current_line.m_line[m_current_line_offset - 1] == '_'); + + const auto startPos = m_current_line_offset - 1; + const auto lineSize = m_current_line.m_line.size(); + while (m_current_line_offset < lineSize) + { + const auto c = m_current_line.m_line[m_current_line_offset]; + + if (!isalnum(c) && c != '_') + break; + + m_current_line_offset++; + } + + return std::string(m_current_line.m_line, startPos, m_current_line_offset - startPos); + } + + /** + * \brief Reads an identifier from the current position + * \return The value of the read identifier + */ + std::string ReadString() + { + assert(m_current_line_offset >= 1); + assert(m_current_line.m_line[m_current_line_offset - 1] == '"'); + + const auto startPos = m_current_line_offset; + const auto lineSize = m_current_line.m_line.size(); + while (true) + { + if (m_current_line_offset >= lineSize) + throw ParsingException(TokenPos(m_current_line.m_filename, m_current_line.m_line_number, m_current_line_offset), "Unclosed string"); + + if (m_current_line.m_line[m_current_line_offset] == '\"') + break; + + m_current_line_offset++; + } + + return std::string(m_current_line.m_line, startPos, m_current_line_offset++ - startPos); + } + + void ReadHexNumber(int& integerValue) + { + const auto* start = &m_current_line.m_line.c_str()[m_current_line_offset - 1]; + char* end; + + integerValue = std::strtoul(start, &end, 16); + const auto numberLength = static_cast(end - start); + if (numberLength == 0 || isalnum(*end) || *end == '_') + throw ParsingException(GetPreviousCharacterPos(), "Invalid hex number"); + + m_current_line_offset += numberLength; + } + + _NODISCARD bool IsIntegerNumber() const + { + const auto* currentCharacter = &m_current_line.m_line.c_str()[m_current_line_offset - 1]; + auto isInteger = true; + auto dot = false; + auto exponent = false; + + if (*currentCharacter == '-') + currentCharacter++; + else if (*currentCharacter == '+') + currentCharacter++; + + while (*currentCharacter) + { + if (isdigit(*currentCharacter)) + { + } + else if (*currentCharacter == '.') + { + if (dot) + throw ParsingException(GetPreviousCharacterPos(), "Invalid number"); + dot = true; + isInteger = false; + } + else if (*currentCharacter == 'e' || *currentCharacter == 'E') + { + if (exponent) + throw ParsingException(GetPreviousCharacterPos(), "Invalid number"); + if (currentCharacter[1] == '-') + currentCharacter++; + exponent = true; + isInteger = false; + } + else if (isalpha(*currentCharacter)) + { + throw ParsingException(GetPreviousCharacterPos(), "Invalid number"); + } + else + { + break; + } + + currentCharacter++; + } + + return isInteger; + } + + int ReadInteger() + { + const auto* start = &m_current_line.m_line.c_str()[m_current_line_offset - 1]; + char* end; + const auto integerValue = std::strtol(start, &end, 10); + const auto numberLength = static_cast(end - start); + + if(numberLength == 0) + throw ParsingException(GetPreviousCharacterPos(), "Invalid number"); + + m_current_line_offset += numberLength - 1; + + return integerValue; + } + + double ReadFloatingPoint() + { + const auto* start = &m_current_line.m_line.c_str()[m_current_line_offset - 1]; + char* end; + const auto floatingPointValue = std::strtod(start, &end); + const auto numberLength = static_cast(end - start); + + if (numberLength == 0) + throw ParsingException(GetPreviousCharacterPos(), "Invalid number"); + + m_current_line_offset += numberLength - 1; + + return floatingPointValue; + } + + void ReadNumber(bool& isFloatingPoint, double& floatingPointValue, int& integerValue) + { + assert(m_current_line_offset >= 1); + assert(isdigit(m_current_line.m_line[m_current_line_offset - 1])); + + const auto lineLength = m_current_line.m_line.size(); + if (lineLength - m_current_line_offset >= 1 + && m_current_line.m_line[m_current_line_offset - 1] == '0' + && m_current_line.m_line[m_current_line_offset] == 'x') + { + isFloatingPoint = false; + ReadHexNumber(integerValue); + } + else if (IsIntegerNumber()) + { + isFloatingPoint = false; + integerValue = ReadInteger(); + } + else + { + isFloatingPoint = true; + floatingPointValue = ReadFloatingPoint(); + } + } + public: - const TokenType& GetToken(int index) + const TokenType& GetToken(unsigned index) { assert(index >= 0); - while (index <= m_token_cache.size()) - m_token_cache.emplace_back(std::move(GetNextToken())); + while (index >= m_token_cache.size()) + m_token_cache.emplace_back(GetNextToken()); return m_token_cache[index]; } diff --git a/src/ZoneCodeGeneratorLib/Parsing/Commands/CommandsFileReader.cpp b/src/ZoneCodeGeneratorLib/Parsing/Commands/CommandsFileReader.cpp index 22df4dc2..acf4dda4 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Commands/CommandsFileReader.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/Commands/CommandsFileReader.cpp @@ -2,6 +2,7 @@ #include +#include "Impl/CommandsLexer.h" #include "Parsing/ParsingException.h" #include "Parsing/Impl/CommentRemovingStreamProxy.h" #include "Parsing/Impl/DefinesStreamProxy.h" @@ -10,43 +11,60 @@ CommandsFileReader::CommandsFileReader(const ZoneCodeGeneratorArguments* args, std::string filename) : m_args(args), - m_filename(std::move(filename)) + m_filename(std::move(filename)), + m_stream(nullptr) { } -bool CommandsFileReader::ReadCommandsFile(IDataRepository* repository) +bool CommandsFileReader::OpenBaseStream() { - std::cout << "Reading commands file: " << m_filename << std::endl; - - ParserFilesystemStream stream(m_filename); - if (!stream.IsOpen()) + auto stream = std::make_unique(m_filename); + if (!stream->IsOpen()) { std::cout << "Could not open commands file" << std::endl; return false; } - IParserLineStream* lineStream = &stream; + m_stream = stream.get(); + m_open_streams.emplace_back(std::move(stream)); + return true; +} - CommentRemovingStreamProxy commentProxy(lineStream); - lineStream = &commentProxy; +void CommandsFileReader::SetupStreamProxies() +{ + auto commentProxy = std::make_unique(m_stream); + auto includeProxy = std::make_unique(commentProxy.get()); + auto definesProxy = std::make_unique(includeProxy.get()); + definesProxy->AddDefine(ZONE_CODE_GENERATOR_DEFINE_NAME, ZONE_CODE_GENERATOR_DEFINE_VALUE); - IncludingStreamProxy includeProxy(lineStream); - lineStream = &includeProxy; + m_stream = definesProxy.get(); - DefinesStreamProxy definesProxy(lineStream); - definesProxy.AddDefine(ZONE_CODE_GENERATOR_DEFINE_NAME, ZONE_CODE_GENERATOR_DEFINE_VALUE); - lineStream = &definesProxy; + m_open_streams.emplace_back(std::move(commentProxy)); + m_open_streams.emplace_back(std::move(includeProxy)); + m_open_streams.emplace_back(std::move(definesProxy)); +} + +bool CommandsFileReader::ReadCommandsFile(IDataRepository* repository) +{ + std::cout << "Reading commands file: " << m_filename << std::endl; + + if (!OpenBaseStream()) + return false; + + SetupStreamProxies(); + + auto lexer = std::make_unique(m_stream); try { while (true) { - auto line = lineStream->NextLine(); + auto line = m_stream->NextLine(); if (line.IsEof()) break; - std::cout << "Line " << line.m_filename << ":" << line.m_line_number << ": " << line.m_line << "\n"; + std::cout << "Line " << line.m_filename.get() << ":" << line.m_line_number << ": " << line.m_line << "\n"; } } catch (const ParsingException& e) diff --git a/src/ZoneCodeGeneratorLib/Parsing/Commands/CommandsFileReader.h b/src/ZoneCodeGeneratorLib/Parsing/Commands/CommandsFileReader.h index 5b8ac207..5eb8be11 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Commands/CommandsFileReader.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Commands/CommandsFileReader.h @@ -3,6 +3,7 @@ #include #include "ZoneCodeGeneratorArguments.h" +#include "Parsing/IParserLineStream.h" #include "Persistence/IDataRepository.h" class CommandsFileReader @@ -13,6 +14,12 @@ class CommandsFileReader const ZoneCodeGeneratorArguments* m_args; std::string m_filename; + std::vector> m_open_streams; + IParserLineStream* m_stream; + + bool OpenBaseStream(); + void SetupStreamProxies(); + public: explicit CommandsFileReader(const ZoneCodeGeneratorArguments* args, std::string filename); diff --git a/src/ZoneCodeGeneratorLib/Parsing/Commands/Impl/CommandsParserValue.h b/src/ZoneCodeGeneratorLib/Parsing/Commands/Impl/CommandsParserValue.h index b0bf36f9..5529b96f 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Commands/Impl/CommandsParserValue.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Commands/Impl/CommandsParserValue.h @@ -2,12 +2,13 @@ #include -#include "Parsing/TokenPos.h" #include "Utils/ClassUtils.h" +#include "Parsing/TokenPos.h" class CommandsParserValueType { CommandsParserValueType() = default; + public: enum { diff --git a/src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.cpp b/src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.cpp index ec7d70d9..a0d05732 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.cpp @@ -2,7 +2,7 @@ #include - +#include "Impl/HeaderLexer.h" #include "Parsing/ParsingException.h" #include "Parsing/Impl/CommentRemovingStreamProxy.h" #include "Parsing/Impl/DefinesStreamProxy.h" @@ -11,43 +11,92 @@ HeaderFileReader::HeaderFileReader(const ZoneCodeGeneratorArguments* args, std::string filename) : m_args(args), - m_filename(std::move(filename)) + m_filename(std::move(filename)), + m_stream(nullptr) { } -bool HeaderFileReader::ReadHeaderFile(IDataRepository* repository) const +bool HeaderFileReader::OpenBaseStream() { - std::cout << "Reading header file: " << m_filename << std::endl; - - ParserFilesystemStream stream(m_filename); - if (!stream.IsOpen()) + auto stream = std::make_unique(m_filename); + if (!stream->IsOpen()) { std::cout << "Could not open header file" << std::endl; return false; } - IParserLineStream* lineStream = &stream; + m_stream = stream.get(); + m_open_streams.emplace_back(std::move(stream)); + return true; +} - CommentRemovingStreamProxy commentProxy(lineStream); - lineStream = &commentProxy; +void HeaderFileReader::SetupStreamProxies() +{ + auto commentProxy = std::make_unique(m_stream); + auto includeProxy = std::make_unique(commentProxy.get()); + auto definesProxy = std::make_unique(includeProxy.get()); + definesProxy->AddDefine(ZONE_CODE_GENERATOR_DEFINE_NAME, ZONE_CODE_GENERATOR_DEFINE_VALUE); - IncludingStreamProxy includeProxy(lineStream); - lineStream = &includeProxy; + m_stream = definesProxy.get(); - DefinesStreamProxy definesProxy(lineStream); - definesProxy.AddDefine(ZONE_CODE_GENERATOR_DEFINE_NAME, ZONE_CODE_GENERATOR_DEFINE_VALUE); - lineStream = &definesProxy; + m_open_streams.emplace_back(std::move(commentProxy)); + m_open_streams.emplace_back(std::move(includeProxy)); + m_open_streams.emplace_back(std::move(definesProxy)); +} + +bool HeaderFileReader::ReadHeaderFile(IDataRepository* repository) +{ + std::cout << "Reading header file: " << m_filename << std::endl; + + if (!OpenBaseStream()) + return false; + + SetupStreamProxies(); + + auto lexer = std::make_unique(m_stream); try { - while (true) + /*while (true) { - auto line = lineStream->NextLine(); + auto line = m_stream->NextLine(); if (line.IsEof()) break; - std::cout << "Line " << line.m_filename << ":" << line.m_line_number << ": " << line.m_line << "\n"; + std::cout << "Line " << line.m_filename.get() << ":" << line.m_line_number << ": " << line.m_line << "\n"; + }*/ + + auto eof = false; + while (!eof) + { + const auto& token = lexer->GetToken(0); + + switch (token.m_type) + { + case HeaderParserValueType::END_OF_FILE: + case HeaderParserValueType::INVALID: + eof = true; + break; + + case HeaderParserValueType::CHARACTER: + std::cout << "Token " << token.CharacterValue() << "\n"; + break; + + case HeaderParserValueType::IDENTIFIER: + std::cout << "Token IDENTIFIER \"" << token.IdentifierValue() << "\"\n"; + break; + + case HeaderParserValueType::STRING: + std::cout << "Token STRING \"" << token.StringValue() << "\"\n"; + break; + + default: + std::cout << "Token UNKNOWN\n"; + break; + } + + lexer->PopTokens(1); } } catch (const ParsingException& e) diff --git a/src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.h b/src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.h index fd5c50d6..7a8f5d1f 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.h @@ -3,6 +3,7 @@ #include #include "ZoneCodeGeneratorArguments.h" +#include "Parsing/IParserLineStream.h" #include "Persistence/IDataRepository.h" class HeaderFileReader @@ -13,8 +14,14 @@ class HeaderFileReader const ZoneCodeGeneratorArguments* m_args; std::string m_filename; + std::vector> m_open_streams; + IParserLineStream* m_stream; + + bool OpenBaseStream(); + void SetupStreamProxies(); + public: HeaderFileReader(const ZoneCodeGeneratorArguments* args, std::string filename); - bool ReadHeaderFile(IDataRepository* repository) const; + bool ReadHeaderFile(IDataRepository* repository); }; diff --git a/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.cpp b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.cpp new file mode 100644 index 00000000..a5697294 --- /dev/null +++ b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.cpp @@ -0,0 +1,144 @@ +#include "HeaderLexer.h" + +HeaderLexer::HeaderLexer(IParserLineStream* stream) + : AbstractLexer(stream) +{ +} + +HeaderParserValue HeaderLexer::GetNextToken() +{ + auto c = NextChar(); + + while (c != EOF) + { + switch (c) + { + case '\"': + { + return HeaderParserValue::String(GetPreviousCharacterPos(), new std::string(ReadString())); + } + + case '<': + { + if (!IsLineEnd()) + { + const auto pos = GetPreviousCharacterPos(); + const auto nextChar = PeekChar(); + + if (nextChar == '=') + { + NextChar(); + return HeaderParserValue::LessEqual(pos); + } + if (nextChar == '<') + { + NextChar(); + return HeaderParserValue::ShiftLeft(pos); + } + } + + return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast(c)); + } + + case '>': + { + if (!IsLineEnd()) + { + const auto pos = GetPreviousCharacterPos(); + const auto nextChar = PeekChar(); + + if (nextChar == '=') + { + NextChar(); + return HeaderParserValue::GreaterEqual(pos); + } + if (nextChar == '>') + { + NextChar(); + return HeaderParserValue::ShiftRight(pos); + } + } + + return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast(c)); + } + + case '=': + { + if (NextCharInLineIs('=')) + { + const auto pos = GetPreviousCharacterPos(); + NextChar(); + return HeaderParserValue::Equals(pos); + } + + return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast(c)); + } + + case '&': + { + if (NextCharInLineIs('&')) + { + const auto pos = GetPreviousCharacterPos(); + NextChar(); + return HeaderParserValue::LogicalAnd(pos); + } + + return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast(c)); + } + + case '|': + { + if (NextCharInLineIs('|')) + { + const auto pos = GetPreviousCharacterPos(); + NextChar(); + return HeaderParserValue::LogicalOr(pos); + } + + return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast(c)); + } + + case '!': + { + if (NextCharInLineIs('=')) + { + const auto pos = GetPreviousCharacterPos(); + NextChar(); + return HeaderParserValue::NotEqual(pos); + } + + return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast(c)); + } + + default: + { + if (isspace(c)) + break; + + if(isdigit(c)) + { + const auto pos = GetPreviousCharacterPos(); + bool isFloatingPointValue; + double doubleValue; + int integerValue; + + ReadNumber(isFloatingPointValue, doubleValue, integerValue); + + if (isFloatingPointValue) + return HeaderParserValue::FloatingPoint(pos, doubleValue); + + return HeaderParserValue::Integer(pos, integerValue); + } + + if (isalpha(c) || c == '_') + return HeaderParserValue::Identifier(GetPreviousCharacterPos(), new std::string(ReadIdentifier())); + + return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast(c)); + } + } + + c = NextChar(); + } + + return HeaderParserValue::EndOfFile(TokenPos()); +} diff --git a/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.h b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.h new file mode 100644 index 00000000..54d6f3aa --- /dev/null +++ b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.h @@ -0,0 +1,13 @@ +#pragma once + +#include "HeaderParserValue.h" +#include "Parsing/AbstractLexer.h" + +class HeaderLexer final : public AbstractLexer +{ +protected: + HeaderParserValue GetNextToken() override; + +public: + explicit HeaderLexer(IParserLineStream* stream); +}; diff --git a/src/ZoneCodeGeneratorLib/Parsing/AbstractLexer.cpp b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParser.cpp similarity index 100% rename from src/ZoneCodeGeneratorLib/Parsing/AbstractLexer.cpp rename to src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParser.cpp diff --git a/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParser.h b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParser.h new file mode 100644 index 00000000..c7a02df4 --- /dev/null +++ b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParser.h @@ -0,0 +1,6 @@ +#pragma once + +class HeaderParser +{ + +}; \ No newline at end of file diff --git a/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.cpp b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.cpp new file mode 100644 index 00000000..8d1b23bd --- /dev/null +++ b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.cpp @@ -0,0 +1,181 @@ +#include "HeaderParserValue.h" + +#include + +HeaderParserValue HeaderParserValue::Invalid(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::INVALID); + return pv; +} + +HeaderParserValue HeaderParserValue::EndOfFile(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::END_OF_FILE); + return pv; +} + +HeaderParserValue HeaderParserValue::Character(const TokenPos pos, const char c) +{ + HeaderParserValue pv(pos, HeaderParserValueType::CHARACTER); + pv.m_value.char_value = c; + return pv; +} + +HeaderParserValue HeaderParserValue::ShiftLeft(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::SHIFT_LEFT); + return pv; +} + +HeaderParserValue HeaderParserValue::ShiftRight(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::SHIFT_RIGHT); + return pv; +} + +HeaderParserValue HeaderParserValue::Equals(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::EQUALS); + return pv; +} + +HeaderParserValue HeaderParserValue::NotEqual(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::NOT_EQUAL); + return pv; +} + +HeaderParserValue HeaderParserValue::GreaterEqual(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::GREATER_EQUAL); + return pv; +} + +HeaderParserValue HeaderParserValue::LessEqual(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::LESS_EQUAL); + return pv; +} + +HeaderParserValue HeaderParserValue::LogicalAnd(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::LOGICAL_AND); + return pv; +} + +HeaderParserValue HeaderParserValue::LogicalOr(const TokenPos pos) +{ + HeaderParserValue pv(pos, HeaderParserValueType::LOGICAL_OR); + return pv; +} + +HeaderParserValue HeaderParserValue::Integer(const TokenPos pos, const int value) +{ + HeaderParserValue pv(pos, HeaderParserValueType::INTEGER); + pv.m_value.int_value = value; + return pv; +} + +HeaderParserValue HeaderParserValue::FloatingPoint(const TokenPos pos, const double value) +{ + HeaderParserValue pv(pos, HeaderParserValueType::FLOATING_POINT); + pv.m_value.double_value = value; + return pv; +} + +HeaderParserValue HeaderParserValue::String(const TokenPos pos, std::string* stringValue) +{ + HeaderParserValue pv(pos, HeaderParserValueType::STRING); + pv.m_value.string_value = stringValue; + return pv; +} + +HeaderParserValue HeaderParserValue::Identifier(const TokenPos pos, std::string* identifier) +{ + HeaderParserValue pv(pos, HeaderParserValueType::IDENTIFIER); + pv.m_value.string_value = identifier; + return pv; +} + +HeaderParserValue HeaderParserValue::TypeName(const TokenPos pos, std::string* typeName) +{ + HeaderParserValue pv(pos, HeaderParserValueType::TYPE_NAME); + pv.m_value.string_value = typeName; + return pv; +} + +HeaderParserValue::HeaderParserValue(const TokenPos pos, const HeaderParserValueType type) + : m_pos(pos), + m_type(type), + m_value() +{ +} + +HeaderParserValue::~HeaderParserValue() +{ + switch (m_type) + { + case HeaderParserValueType::STRING: + case HeaderParserValueType::IDENTIFIER: + case HeaderParserValueType::TYPE_NAME: + delete m_value.string_value; + break; + + default: + break; + } + + m_value = ValueType(); +} + +HeaderParserValue::HeaderParserValue(HeaderParserValue&& other) noexcept + : m_type(other.m_type), + m_value(other.m_value) +{ + other.m_value = ValueType(); +} + +HeaderParserValue& HeaderParserValue::operator=(HeaderParserValue&& other) noexcept +{ + m_type = other.m_type; + m_value = other.m_value; + other.m_value = ValueType(); + + return *this; +} + +char HeaderParserValue::CharacterValue() const +{ + assert(m_type == HeaderParserValueType::CHARACTER); + return m_value.char_value; +} + +int HeaderParserValue::IntegerValue() const +{ + assert(m_type == HeaderParserValueType::INTEGER); + return m_value.int_value; +} + +double HeaderParserValue::FloatingPointValue() const +{ + assert(m_type == HeaderParserValueType::FLOATING_POINT); + return m_value.double_value; +} + +std::string& HeaderParserValue::StringValue() const +{ + assert(m_type == HeaderParserValueType::STRING); + return *m_value.string_value; +} + +std::string& HeaderParserValue::IdentifierValue() const +{ + assert(m_type == HeaderParserValueType::IDENTIFIER); + return *m_value.string_value; +} + +std::string& HeaderParserValue::TypeNameValue() const +{ + assert(m_type == HeaderParserValueType::TYPE_NAME); + return *m_value.string_value; +} diff --git a/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.h b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.h new file mode 100644 index 00000000..dcef1f24 --- /dev/null +++ b/src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.h @@ -0,0 +1,86 @@ +#pragma once + +#include + +#include "Utils/ClassUtils.h" +#include "Parsing/TokenPos.h" + +enum class HeaderParserValueType +{ + // Meta tokens + INVALID, + END_OF_FILE, + + // Single character + CHARACTER, + + // Symbol tokens + SHIFT_LEFT, + SHIFT_RIGHT, + EQUALS, + NOT_EQUAL, + GREATER_EQUAL, + LESS_EQUAL, + LOGICAL_AND, + LOGICAL_OR, + + // Generic token types + INTEGER, + FLOATING_POINT, + STRING, + IDENTIFIER, + + // Parser created + TYPE_NAME, + + // End + MAX +}; + +class HeaderParserValue +{ +public: + TokenPos m_pos; + HeaderParserValueType m_type; + union ValueType + { + char char_value; + int int_value; + double double_value; + std::string* string_value; + } m_value; + + static HeaderParserValue Invalid(TokenPos pos); + static HeaderParserValue EndOfFile(TokenPos pos); + static HeaderParserValue Character(TokenPos pos, char c); + static HeaderParserValue ShiftLeft(TokenPos pos); + static HeaderParserValue ShiftRight(TokenPos pos); + static HeaderParserValue Equals(TokenPos pos); + static HeaderParserValue NotEqual(TokenPos pos); + static HeaderParserValue GreaterEqual(TokenPos pos); + static HeaderParserValue LessEqual(TokenPos pos); + static HeaderParserValue LogicalAnd(TokenPos pos); + static HeaderParserValue LogicalOr(TokenPos pos); + static HeaderParserValue Integer(TokenPos pos, int value); + static HeaderParserValue FloatingPoint(TokenPos pos, double value); + static HeaderParserValue String(TokenPos pos, std::string* stringValue); + static HeaderParserValue Identifier(TokenPos pos, std::string* identifier); + static HeaderParserValue TypeName(TokenPos pos, std::string* typeName); + +private: + HeaderParserValue(TokenPos pos, HeaderParserValueType type); + +public: + ~HeaderParserValue(); + HeaderParserValue(const HeaderParserValue& other) = delete; + HeaderParserValue(HeaderParserValue&& other) noexcept; + HeaderParserValue& operator=(const HeaderParserValue& other) = delete; + HeaderParserValue& operator=(HeaderParserValue&& other) noexcept; + + _NODISCARD char CharacterValue() const; + _NODISCARD int IntegerValue() const; + _NODISCARD double FloatingPointValue() const; + _NODISCARD std::string& StringValue() const; + _NODISCARD std::string& IdentifierValue() const; + _NODISCARD std::string& TypeNameValue() const; +}; \ No newline at end of file diff --git a/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.cpp b/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.cpp index 1460adf6..332e6302 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.cpp @@ -1,5 +1,13 @@ #include "IParserLineStream.h" +const std::string ParserLine::EMPTY_STRING; + +ParserLine::ParserLine() + : m_filename(EMPTY_STRING), + m_line_number(0) +{ +} + ParserLine::ParserLine(const std::string& filename, const int lineNumber, std::string line) : m_filename(filename), m_line_number(lineNumber), diff --git a/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.h b/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.h index 1594bd56..1c1bb8d4 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.h +++ b/src/ZoneCodeGeneratorLib/Parsing/IParserLineStream.h @@ -6,11 +6,14 @@ class ParserLine { + static const std::string EMPTY_STRING; + public: - const std::string& m_filename; - const int m_line_number; + std::reference_wrapper m_filename; + int m_line_number; std::string m_line; + ParserLine(); ParserLine(const std::string& filename, int lineNumber, std::string line); _NODISCARD bool IsEof() const; diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.cpp b/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.cpp index 05f3395d..d7bd2993 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.cpp +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.cpp @@ -5,8 +5,6 @@ namespace fs = std::filesystem; -const std::string ParserFilesystemStream::EMPTY_FILE_NAME; - ParserFilesystemStream::FileInfo::FileInfo(std::string filePath) : m_file_path(std::move(filePath)), m_stream(m_file_path), @@ -31,7 +29,7 @@ ParserLine ParserFilesystemStream::NextLine() auto hasLength = false; if (m_files.empty()) - return ParserLine(EMPTY_FILE_NAME, 0, std::string()); + return ParserLine(); while(!m_files.empty()) { @@ -67,7 +65,7 @@ ParserLine ParserFilesystemStream::NextLine() m_files.pop(); } - return ParserLine(EMPTY_FILE_NAME, 0, std::string()); + return ParserLine(); } bool ParserFilesystemStream::IncludeFile(const std::string& filename) diff --git a/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.h b/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.h index 385d5556..b010cff5 100644 --- a/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.h +++ b/src/ZoneCodeGeneratorLib/Parsing/Impl/ParserFilesystemStream.h @@ -7,8 +7,6 @@ class ParserFilesystemStream final : public IParserLineStream { - static const std::string EMPTY_FILE_NAME; - class FileInfo { public: diff --git a/test/ZoneCodeGeneratorLibTests/Parsing/Commands/Impl/CommandsLexerTests.cpp b/test/ZoneCodeGeneratorLibTests/Parsing/Commands/Impl/CommandsLexerTests.cpp new file mode 100644 index 00000000..1ce0d4aa --- /dev/null +++ b/test/ZoneCodeGeneratorLibTests/Parsing/Commands/Impl/CommandsLexerTests.cpp @@ -0,0 +1,12 @@ +#include + +#include "Parsing/Commands/Impl/CommandsLexer.h" +#include "Parsing/Mock/MockParserLineStream.h" + +namespace test::parsing::commands +{ + TEST_CASE("CommandsLexer: ", "[parsing][commands]") + { + + } +} diff --git a/test/ZoneCodeGeneratorLibTests/Parsing/Header/Impl/HeaderLexerTests.cpp b/test/ZoneCodeGeneratorLibTests/Parsing/Header/Impl/HeaderLexerTests.cpp new file mode 100644 index 00000000..bb580032 --- /dev/null +++ b/test/ZoneCodeGeneratorLibTests/Parsing/Header/Impl/HeaderLexerTests.cpp @@ -0,0 +1,541 @@ +#include + +#include "Parsing/Header/Impl/HeaderLexer.h" +#include "Parsing/Mock/MockParserLineStream.h" + +namespace test::parsing::header +{ + void ExpectCharacterToken(HeaderLexer& lexer, char c) + { + REQUIRE(lexer.GetToken(0).m_type == HeaderParserValueType::CHARACTER); + REQUIRE(lexer.GetToken(0).CharacterValue() == c); + lexer.PopTokens(1); + } + + void ExpectIntegerToken(HeaderLexer& lexer, int number) + { + REQUIRE(lexer.GetToken(0).m_type == HeaderParserValueType::INTEGER); + REQUIRE(lexer.GetToken(0).IntegerValue() == number); + lexer.PopTokens(1); + } + + void ExpectFloatingPointToken(HeaderLexer& lexer, double number) + { + REQUIRE(lexer.GetToken(0).m_type == HeaderParserValueType::FLOATING_POINT); + REQUIRE(lexer.GetToken(0).FloatingPointValue() == Approx(number)); + lexer.PopTokens(1); + } + + void ExpectIdentifierToken(HeaderLexer& lexer, const std::string& identifier) + { + REQUIRE(lexer.GetToken(0).m_type == HeaderParserValueType::IDENTIFIER); + REQUIRE(lexer.GetToken(0).IdentifierValue() == identifier); + lexer.PopTokens(1); + } + + void ExpectStringToken(HeaderLexer& lexer, const std::string& c) + { + REQUIRE(lexer.GetToken(0).m_type == HeaderParserValueType::STRING); + REQUIRE(lexer.GetToken(0).StringValue() == c); + lexer.PopTokens(1); + } + + void ExpectTokenWithType(HeaderLexer& lexer, HeaderParserValueType type) + { + REQUIRE(lexer.GetToken(0).m_type == type); + lexer.PopTokens(1); + } + + TEST_CASE("HeaderLexer: Ensure can parse simple hex numbers", "[parsing][header]") + { + const std::vector lines + { + "0x1A4", + " 0xABC ", + " 0xAAA", + "0xBBB " + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectIntegerToken(lexer, 0x1a4); + ExpectIntegerToken(lexer, 0xabc); + ExpectIntegerToken(lexer, 0xaaa); + ExpectIntegerToken(lexer, 0xbbb); + + REQUIRE(lexer.GetToken(0).m_type == HeaderParserValueType::END_OF_FILE); + } + + TEST_CASE("HeaderLexer: Ensure can parse simple hex numbers surrounded by identifiers", "[parsing][header]") + { + const std::vector lines + { + "abc 0xABC cba", + "aaa 0xAAA", + "0xBBB bbb" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectIdentifierToken(lexer, "abc"); + ExpectIntegerToken(lexer, 0xabc); + ExpectIdentifierToken(lexer, "cba"); + + ExpectIdentifierToken(lexer, "aaa"); + ExpectIntegerToken(lexer, 0xaaa); + + ExpectIntegerToken(lexer, 0xbbb); + ExpectIdentifierToken(lexer, "bbb"); + + REQUIRE(lexer.GetToken(0).m_type == HeaderParserValueType::END_OF_FILE); + } + + TEST_CASE("HeaderLexer: Ensure throws exception when parsing incomplete hex number", "[parsing][header]") + { + const std::vector lines + { + "0x" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + REQUIRE_THROWS_AS(lexer.GetToken(0), ParsingException); + } + + TEST_CASE("HeaderLexer: Ensure throws exception when parsing invalid hex number", "[parsing][header]") + { + const std::vector lines + { + "0xGEGE" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + REQUIRE_THROWS_AS(lexer.GetToken(0), ParsingException); + } + + TEST_CASE("HeaderLexer: Ensure throws exception when parsing invalid hex number that starts with a valid letter", "[parsing][header]") + { + const std::vector lines + { + "0xEGEG" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + REQUIRE_THROWS_AS(lexer.GetToken(0), ParsingException); + } + + TEST_CASE("HeaderLexer: Ensure can parse simple integers", "[parsing][header]") + { + const std::vector lines + { + " 524 ", + "4221111 1337 ", + "0 420" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectIntegerToken(lexer, 524); + ExpectIntegerToken(lexer, 4221111); + ExpectIntegerToken(lexer, 1337); + ExpectIntegerToken(lexer, 0); + ExpectIntegerToken(lexer, 420); + } + + TEST_CASE("HeaderLexer: Ensure can parse integers surrounded by identifiers", "[parsing][header]") + { + const std::vector lines + { + "aa 6 bb", + "123456789 ccc", + "0 d 420" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectIdentifierToken(lexer, "aa"); + ExpectIntegerToken(lexer, 6); + ExpectIdentifierToken(lexer, "bb"); + ExpectIntegerToken(lexer, 123456789); + ExpectIdentifierToken(lexer, "ccc"); + ExpectIntegerToken(lexer, 0); + ExpectIdentifierToken(lexer, "d"); + ExpectIntegerToken(lexer, 420); + } + + TEST_CASE("HeaderLexer: Ensure parses negative numbers as character and number", "[parsing][header]") + { + const std::vector lines + { + "-1337" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectCharacterToken(lexer, '-'); + ExpectIntegerToken(lexer, 1337); + } + + TEST_CASE("HeaderLexer: Ensure recognizes numbers surrounded by characters", "[parsing][header]") + { + const std::vector lines + { + "(1337)" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectCharacterToken(lexer, '('); + ExpectIntegerToken(lexer, 1337); + ExpectCharacterToken(lexer, ')'); + } + + TEST_CASE("HeaderLexer: Ensure parses simple floating point numbers", "[parsing][header]") + { + const std::vector lines + { + "420.1337" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectFloatingPointToken(lexer, 420.1337); + } + + TEST_CASE("HeaderLexer: Ensure parses simple floating point numbers surrounded by identifiers", "[parsing][header]") + { + const std::vector lines + { + "aa 50.24 a", + "b 36.999", + "59595.2414 c" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectIdentifierToken(lexer, "aa"); + ExpectFloatingPointToken(lexer, 50.24); + ExpectIdentifierToken(lexer, "a"); + ExpectIdentifierToken(lexer, "b"); + ExpectFloatingPointToken(lexer, 36.999); + ExpectFloatingPointToken(lexer, 59595.2412); + ExpectIdentifierToken(lexer, "c"); + } + + TEST_CASE("HeaderLexer: Ensure recognizes floating point numbers surrounded by characters", "[parsing][header]") + { + const std::vector lines + { + "(1337.420)" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectCharacterToken(lexer, '('); + ExpectFloatingPointToken(lexer, 1337.420); + ExpectCharacterToken(lexer, ')'); + } + + TEST_CASE("HeaderLexer: Ensure can separate identifiers with symbols", "[parsing][header]") + { + const std::vector lines + { + "hello|world hello_+universe" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectIdentifierToken(lexer, "hello"); + ExpectCharacterToken(lexer, '|'); + ExpectIdentifierToken(lexer, "world"); + ExpectIdentifierToken(lexer, "hello_"); + ExpectCharacterToken(lexer, '+'); + ExpectIdentifierToken(lexer, "universe"); + } + + TEST_CASE("HeaderLexer: Can recognize shift left", "[parsing][header]") + { + const std::vector lines + { + "< lines + { + "<=hello<=world<=" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectTokenWithType(lexer, HeaderParserValueType::LESS_EQUAL); + ExpectIdentifierToken(lexer, "hello"); + ExpectTokenWithType(lexer, HeaderParserValueType::LESS_EQUAL); + ExpectIdentifierToken(lexer, "world"); + ExpectTokenWithType(lexer, HeaderParserValueType::LESS_EQUAL); + } + + TEST_CASE("HeaderLexer: Can recognize less", "[parsing][header]") + { + const std::vector lines + { + "<%hello lines + { + ">>hello>>world>>" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectTokenWithType(lexer, HeaderParserValueType::SHIFT_RIGHT); + ExpectIdentifierToken(lexer, "hello"); + ExpectTokenWithType(lexer, HeaderParserValueType::SHIFT_RIGHT); + ExpectIdentifierToken(lexer, "world"); + ExpectTokenWithType(lexer, HeaderParserValueType::SHIFT_RIGHT); + } + + TEST_CASE("HeaderLexer: Can recognize greater equals", "[parsing][header]") + { + const std::vector lines + { + ">=hello>=world>=" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectTokenWithType(lexer, HeaderParserValueType::GREATER_EQUAL); + ExpectIdentifierToken(lexer, "hello"); + ExpectTokenWithType(lexer, HeaderParserValueType::GREATER_EQUAL); + ExpectIdentifierToken(lexer, "world"); + ExpectTokenWithType(lexer, HeaderParserValueType::GREATER_EQUAL); + } + + TEST_CASE("HeaderLexer: Can recognize greater", "[parsing][header]") + { + const std::vector lines + { + ">%hello>world>&" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectCharacterToken(lexer, '>'); + ExpectCharacterToken(lexer, '%'); + ExpectIdentifierToken(lexer, "hello"); + ExpectCharacterToken(lexer, '>'); + ExpectIdentifierToken(lexer, "world"); + ExpectCharacterToken(lexer, '>'); + ExpectCharacterToken(lexer, '&'); + } + + TEST_CASE("HeaderLexer: Can recognize equals", "[parsing][header]") + { + const std::vector lines + { + "==hello==world==" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectTokenWithType(lexer, HeaderParserValueType::EQUALS); + ExpectIdentifierToken(lexer, "hello"); + ExpectTokenWithType(lexer, HeaderParserValueType::EQUALS); + ExpectIdentifierToken(lexer, "world"); + ExpectTokenWithType(lexer, HeaderParserValueType::EQUALS); + } + + TEST_CASE("HeaderLexer: Can recognize assign", "[parsing][header]") + { + const std::vector lines + { + "=%hello=world=&" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectCharacterToken(lexer, '='); + ExpectCharacterToken(lexer, '%'); + ExpectIdentifierToken(lexer, "hello"); + ExpectCharacterToken(lexer, '='); + ExpectIdentifierToken(lexer, "world"); + ExpectCharacterToken(lexer, '='); + ExpectCharacterToken(lexer, '&'); + } + + TEST_CASE("HeaderLexer: Can recognize logical and", "[parsing][header]") + { + const std::vector lines + { + "&&hello&&world&&" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectTokenWithType(lexer, HeaderParserValueType::LOGICAL_AND); + ExpectIdentifierToken(lexer, "hello"); + ExpectTokenWithType(lexer, HeaderParserValueType::LOGICAL_AND); + ExpectIdentifierToken(lexer, "world"); + ExpectTokenWithType(lexer, HeaderParserValueType::LOGICAL_AND); + } + + TEST_CASE("HeaderLexer: Can recognize ampersand", "[parsing][header]") + { + const std::vector lines + { + "&%hello&world&+" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectCharacterToken(lexer, '&'); + ExpectCharacterToken(lexer, '%'); + ExpectIdentifierToken(lexer, "hello"); + ExpectCharacterToken(lexer, '&'); + ExpectIdentifierToken(lexer, "world"); + ExpectCharacterToken(lexer, '&'); + ExpectCharacterToken(lexer, '+'); + } + + TEST_CASE("HeaderLexer: Can recognize logical or", "[parsing][header]") + { + const std::vector lines + { + "||hello||world||" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectTokenWithType(lexer, HeaderParserValueType::LOGICAL_OR); + ExpectIdentifierToken(lexer, "hello"); + ExpectTokenWithType(lexer, HeaderParserValueType::LOGICAL_OR); + ExpectIdentifierToken(lexer, "world"); + ExpectTokenWithType(lexer, HeaderParserValueType::LOGICAL_OR); + } + + TEST_CASE("HeaderLexer: Can recognize pipe", "[parsing][header]") + { + const std::vector lines + { + "|%hello|world|&" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectCharacterToken(lexer, '|'); + ExpectCharacterToken(lexer, '%'); + ExpectIdentifierToken(lexer, "hello"); + ExpectCharacterToken(lexer, '|'); + ExpectIdentifierToken(lexer, "world"); + ExpectCharacterToken(lexer, '|'); + ExpectCharacterToken(lexer, '&'); + } + + TEST_CASE("HeaderLexer: Can recognize not equals", "[parsing][header]") + { + const std::vector lines + { + "!=hello!=world!=" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectTokenWithType(lexer, HeaderParserValueType::NOT_EQUAL); + ExpectIdentifierToken(lexer, "hello"); + ExpectTokenWithType(lexer, HeaderParserValueType::NOT_EQUAL); + ExpectIdentifierToken(lexer, "world"); + ExpectTokenWithType(lexer, HeaderParserValueType::NOT_EQUAL); + } + + TEST_CASE("HeaderLexer: Can recognize exclamation mark", "[parsing][header]") + { + const std::vector lines + { + "!%hello!world!&" + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectCharacterToken(lexer, '!'); + ExpectCharacterToken(lexer, '%'); + ExpectIdentifierToken(lexer, "hello"); + ExpectCharacterToken(lexer, '!'); + ExpectIdentifierToken(lexer, "world"); + ExpectCharacterToken(lexer, '!'); + ExpectCharacterToken(lexer, '&'); + } + + TEST_CASE("HeaderLexer: Can recognize strings", "[parsing][header]") + { + const std::vector lines + { + "\"hello world\"", + "a\"hi there\"bbb", + " \"nice\"", + "\"meme\" " + }; + + MockParserLineStream mockStream(lines); + HeaderLexer lexer(&mockStream); + + ExpectStringToken(lexer, "hello world"); + ExpectIdentifierToken(lexer, "a"); + ExpectStringToken(lexer, "hi there"); + ExpectIdentifierToken(lexer, "bbb"); + ExpectStringToken(lexer, "nice"); + ExpectStringToken(lexer, "meme"); + } +} diff --git a/test/ZoneCodeGeneratorLibTests/Parsing/Impl/IncludingStreamProxyTest.cpp b/test/ZoneCodeGeneratorLibTests/Parsing/Impl/IncludingStreamProxyTest.cpp index f551a78b..556070d2 100644 --- a/test/ZoneCodeGeneratorLibTests/Parsing/Impl/IncludingStreamProxyTest.cpp +++ b/test/ZoneCodeGeneratorLibTests/Parsing/Impl/IncludingStreamProxyTest.cpp @@ -27,21 +27,21 @@ namespace test::parsing { auto line = proxy.NextLine(); REQUIRE(line.m_line_number == 1); - REQUIRE(line.m_filename == MockParserLineStream::MOCK_FILENAME); + REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME); REQUIRE(line.m_line == "Hello world"); } { auto line = proxy.NextLine(); REQUIRE(line.m_line_number == 1); - REQUIRE(line.m_filename == "ASDF.txt"); + REQUIRE(line.m_filename.get() == "ASDF.txt"); REQUIRE(line.m_line == "Hello galaxy"); } { auto line = proxy.NextLine(); REQUIRE(line.m_line_number == 3); - REQUIRE(line.m_filename == MockParserLineStream::MOCK_FILENAME); + REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME); REQUIRE(line.m_line == "and bye"); } @@ -70,21 +70,21 @@ namespace test::parsing { auto line = proxy.NextLine(); REQUIRE(line.m_line_number == 1); - REQUIRE(line.m_filename == MockParserLineStream::MOCK_FILENAME); + REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME); REQUIRE(line.m_line == "Hello world"); } { auto line = proxy.NextLine(); REQUIRE(line.m_line_number == 1); - REQUIRE(line.m_filename == "ASDF.txt"); + REQUIRE(line.m_filename.get() == "ASDF.txt"); REQUIRE(line.m_line == "Hello galaxy"); } { auto line = proxy.NextLine(); REQUIRE(line.m_line_number == 3); - REQUIRE(line.m_filename == MockParserLineStream::MOCK_FILENAME); + REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME); REQUIRE(line.m_line == "and bye"); } @@ -113,21 +113,21 @@ namespace test::parsing { auto line = proxy.NextLine(); REQUIRE(line.m_line_number == 1); - REQUIRE(line.m_filename == MockParserLineStream::MOCK_FILENAME); + REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME); REQUIRE(line.m_line == "Hello world"); } { auto line = proxy.NextLine(); REQUIRE(line.m_line_number == 1); - REQUIRE(line.m_filename == "ASDF.txt"); + REQUIRE(line.m_filename.get() == "ASDF.txt"); REQUIRE(line.m_line == "Hello galaxy"); } { auto line = proxy.NextLine(); REQUIRE(line.m_line_number == 3); - REQUIRE(line.m_filename == MockParserLineStream::MOCK_FILENAME); + REQUIRE(line.m_filename.get() == MockParserLineStream::MOCK_FILENAME); REQUIRE(line.m_line == "and bye"); } diff --git a/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.cpp b/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.cpp index d297374f..ad373a5c 100644 --- a/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.cpp +++ b/test/ZoneCodeGeneratorLibTests/Parsing/Mock/MockParserLineStream.cpp @@ -37,7 +37,7 @@ ParserLine MockParserLineStream::NextLine() m_include_positions.pop_back(); } - return ParserLine(MOCK_FILENAME, 0, std::string()); + return ParserLine(); } bool MockParserLineStream::IncludeFile(const std::string& filename)