Add Header Lexer for ZCG cpp

src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.cpp

@@ -2,7 +2,7 @@
 
 #include <iostream>
 
-
+#include "Impl/HeaderLexer.h"
 #include "Parsing/ParsingException.h"
 #include "Parsing/Impl/CommentRemovingStreamProxy.h"
 #include "Parsing/Impl/DefinesStreamProxy.h"
@@ -11,43 +11,92 @@
 
 HeaderFileReader::HeaderFileReader(const ZoneCodeGeneratorArguments* args, std::string filename)
     : m_args(args),
-      m_filename(std::move(filename))
+      m_filename(std::move(filename)),
+      m_stream(nullptr)
 {
 }
 
-bool HeaderFileReader::ReadHeaderFile(IDataRepository* repository) const
+bool HeaderFileReader::OpenBaseStream()
 {
-    std::cout << "Reading header file: " << m_filename << std::endl;
-
-    ParserFilesystemStream stream(m_filename);
-    if (!stream.IsOpen())
+    auto stream = std::make_unique<ParserFilesystemStream>(m_filename);
+    if (!stream->IsOpen())
     {
         std::cout << "Could not open header file" << std::endl;
         return false;
     }
 
-    IParserLineStream* lineStream = &stream;
+    m_stream = stream.get();
+    m_open_streams.emplace_back(std::move(stream));
+    return true;
+}
 
-    CommentRemovingStreamProxy commentProxy(lineStream);
-    lineStream = &commentProxy;
+void HeaderFileReader::SetupStreamProxies()
+{
+    auto commentProxy = std::make_unique<CommentRemovingStreamProxy>(m_stream);
+    auto includeProxy = std::make_unique<IncludingStreamProxy>(commentProxy.get());
+    auto definesProxy = std::make_unique<DefinesStreamProxy>(includeProxy.get());
+    definesProxy->AddDefine(ZONE_CODE_GENERATOR_DEFINE_NAME, ZONE_CODE_GENERATOR_DEFINE_VALUE);
 
-    IncludingStreamProxy includeProxy(lineStream);
-    lineStream = &includeProxy;
+    m_stream = definesProxy.get();
 
-    DefinesStreamProxy definesProxy(lineStream);
-    definesProxy.AddDefine(ZONE_CODE_GENERATOR_DEFINE_NAME, ZONE_CODE_GENERATOR_DEFINE_VALUE);
-    lineStream = &definesProxy;
+    m_open_streams.emplace_back(std::move(commentProxy));
+    m_open_streams.emplace_back(std::move(includeProxy));
+    m_open_streams.emplace_back(std::move(definesProxy));
+}
+
+bool HeaderFileReader::ReadHeaderFile(IDataRepository* repository)
+{
+    std::cout << "Reading header file: " << m_filename << std::endl;
+
+    if (!OpenBaseStream())
+        return false;
+
+    SetupStreamProxies();
+
+    auto lexer = std::make_unique<HeaderLexer>(m_stream);
 
     try
     {
-        while (true)
+        /*while (true)
         {
-            auto line = lineStream->NextLine();
+            auto line = m_stream->NextLine();
 
             if (line.IsEof())
                 break;
 
-            std::cout << "Line " << line.m_filename << ":" << line.m_line_number << ": " << line.m_line << "\n";
+            std::cout << "Line " << line.m_filename.get() << ":" << line.m_line_number << ": " << line.m_line << "\n";
+        }*/
+
+        auto eof = false;
+        while (!eof)
+        {
+            const auto& token = lexer->GetToken(0);
+
+            switch (token.m_type)
+            {
+            case HeaderParserValueType::END_OF_FILE:
+            case HeaderParserValueType::INVALID:
+                eof = true;
+                break;
+
+            case HeaderParserValueType::CHARACTER:
+                std::cout << "Token " << token.CharacterValue() << "\n";
+                break;
+
+            case HeaderParserValueType::IDENTIFIER:
+                std::cout << "Token IDENTIFIER \"" << token.IdentifierValue() << "\"\n";
+                break;
+
+            case HeaderParserValueType::STRING:
+                std::cout << "Token STRING \"" << token.StringValue() << "\"\n";
+                break;
+
+            default:
+                std::cout << "Token UNKNOWN\n";
+                break;
+            }
+
+            lexer->PopTokens(1);
         }
     }
     catch (const ParsingException& e)

src/ZoneCodeGeneratorLib/Parsing/Header/HeaderFileReader.h

@@ -3,6 +3,7 @@
 #include <string>
 
 #include "ZoneCodeGeneratorArguments.h"
+#include "Parsing/IParserLineStream.h"
 #include "Persistence/IDataRepository.h"
 
 class HeaderFileReader
@@ -13,8 +14,14 @@ class HeaderFileReader
     const ZoneCodeGeneratorArguments* m_args;
     std::string m_filename;
 
+    std::vector<std::unique_ptr<IParserLineStream>> m_open_streams;
+    IParserLineStream* m_stream;
+
+    bool OpenBaseStream();
+    void SetupStreamProxies();
+
 public:
     HeaderFileReader(const ZoneCodeGeneratorArguments* args, std::string filename);
 
-    bool ReadHeaderFile(IDataRepository* repository) const;
+    bool ReadHeaderFile(IDataRepository* repository);
 };

src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.cpp

@@ -0,0 +1,144 @@
+#include "HeaderLexer.h"
+
+HeaderLexer::HeaderLexer(IParserLineStream* stream)
+    : AbstractLexer(stream)
+{
+}
+
+HeaderParserValue HeaderLexer::GetNextToken()
+{
+    auto c = NextChar();
+
+    while (c != EOF)
+    {
+        switch (c)
+        {
+        case '\"':
+            {
+                return HeaderParserValue::String(GetPreviousCharacterPos(), new std::string(ReadString()));
+            }
+
+        case '<':
+            {
+                if (!IsLineEnd())
+                {
+                    const auto pos = GetPreviousCharacterPos();
+                    const auto nextChar = PeekChar();
+
+                    if (nextChar == '=')
+                    {
+                        NextChar();
+                        return HeaderParserValue::LessEqual(pos);
+                    }
+                    if (nextChar == '<')
+                    {
+                        NextChar();
+                        return HeaderParserValue::ShiftLeft(pos);
+                    }
+                }
+
+                return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast<char>(c));
+            }
+
+        case '>':
+            {
+                if (!IsLineEnd())
+                {
+                    const auto pos = GetPreviousCharacterPos();
+                    const auto nextChar = PeekChar();
+
+                    if (nextChar == '=')
+                    {
+                        NextChar();
+                        return HeaderParserValue::GreaterEqual(pos);
+                    }
+                    if (nextChar == '>')
+                    {
+                        NextChar();
+                        return HeaderParserValue::ShiftRight(pos);
+                    }
+                }
+
+                return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast<char>(c));
+            }
+
+        case '=':
+            {
+                if (NextCharInLineIs('='))
+                {
+                    const auto pos = GetPreviousCharacterPos();
+                    NextChar();
+                    return HeaderParserValue::Equals(pos);
+                }
+
+                return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast<char>(c));
+            }
+
+        case '&':
+            {
+                if (NextCharInLineIs('&'))
+                {
+                    const auto pos = GetPreviousCharacterPos();
+                    NextChar();
+                    return HeaderParserValue::LogicalAnd(pos);
+                }
+
+                return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast<char>(c));
+            }
+
+        case '|':
+            {
+                if (NextCharInLineIs('|'))
+                {
+                    const auto pos = GetPreviousCharacterPos();
+                    NextChar();
+                    return HeaderParserValue::LogicalOr(pos);
+                }
+
+                return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast<char>(c));
+            }
+
+        case '!':
+            {
+                if (NextCharInLineIs('='))
+                {
+                    const auto pos = GetPreviousCharacterPos();
+                    NextChar();
+                    return HeaderParserValue::NotEqual(pos);
+                }
+
+                return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast<char>(c));
+            }
+
+        default:
+            {
+                if (isspace(c))
+                    break;
+
+                if(isdigit(c))
+                {
+                    const auto pos = GetPreviousCharacterPos();
+                    bool isFloatingPointValue;
+                    double doubleValue;
+                    int integerValue;
+
+                    ReadNumber(isFloatingPointValue, doubleValue, integerValue);
+
+                    if (isFloatingPointValue)
+                        return HeaderParserValue::FloatingPoint(pos, doubleValue);
+
+                    return HeaderParserValue::Integer(pos, integerValue);
+                }
+
+                if (isalpha(c) || c == '_')
+                    return HeaderParserValue::Identifier(GetPreviousCharacterPos(), new std::string(ReadIdentifier()));
+
+                return HeaderParserValue::Character(GetPreviousCharacterPos(), static_cast<char>(c));
+            }
+        }
+
+        c = NextChar();
+    }
+
+    return HeaderParserValue::EndOfFile(TokenPos());
+}

src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderLexer.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "HeaderParserValue.h"
+#include "Parsing/AbstractLexer.h"
+
+class HeaderLexer final : public AbstractLexer<HeaderParserValue>
+{
+protected:
+    HeaderParserValue GetNextToken() override;
+
+public:
+    explicit HeaderLexer(IParserLineStream* stream);
+};

src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParser.h

@@ -0,0 +1,6 @@
+#pragma once
+
+class HeaderParser
+{
+
+};

src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.cpp

@@ -0,0 +1,181 @@
+#include "HeaderParserValue.h"
+
+#include <cassert>
+
+HeaderParserValue HeaderParserValue::Invalid(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::INVALID);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::EndOfFile(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::END_OF_FILE);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::Character(const TokenPos pos, const char c)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::CHARACTER);
+    pv.m_value.char_value = c;
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::ShiftLeft(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::SHIFT_LEFT);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::ShiftRight(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::SHIFT_RIGHT);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::Equals(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::EQUALS);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::NotEqual(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::NOT_EQUAL);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::GreaterEqual(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::GREATER_EQUAL);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::LessEqual(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::LESS_EQUAL);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::LogicalAnd(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::LOGICAL_AND);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::LogicalOr(const TokenPos pos)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::LOGICAL_OR);
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::Integer(const TokenPos pos, const int value)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::INTEGER);
+    pv.m_value.int_value = value;
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::FloatingPoint(const TokenPos pos, const double value)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::FLOATING_POINT);
+    pv.m_value.double_value = value;
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::String(const TokenPos pos, std::string* stringValue)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::STRING);
+    pv.m_value.string_value = stringValue;
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::Identifier(const TokenPos pos, std::string* identifier)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::IDENTIFIER);
+    pv.m_value.string_value = identifier;
+    return pv;
+}
+
+HeaderParserValue HeaderParserValue::TypeName(const TokenPos pos, std::string* typeName)
+{
+    HeaderParserValue pv(pos, HeaderParserValueType::TYPE_NAME);
+    pv.m_value.string_value = typeName;
+    return pv;
+}
+
+HeaderParserValue::HeaderParserValue(const TokenPos pos, const HeaderParserValueType type)
+    : m_pos(pos),
+    m_type(type),
+    m_value()
+{
+}
+
+HeaderParserValue::~HeaderParserValue()
+{
+    switch (m_type)
+    {
+    case HeaderParserValueType::STRING:
+    case HeaderParserValueType::IDENTIFIER:
+    case HeaderParserValueType::TYPE_NAME:
+        delete m_value.string_value;
+        break;
+
+    default:
+        break;
+    }
+
+    m_value = ValueType();
+}
+
+HeaderParserValue::HeaderParserValue(HeaderParserValue&& other) noexcept
+    : m_type(other.m_type),
+    m_value(other.m_value)
+{
+    other.m_value = ValueType();
+}
+
+HeaderParserValue& HeaderParserValue::operator=(HeaderParserValue&& other) noexcept
+{
+    m_type = other.m_type;
+    m_value = other.m_value;
+    other.m_value = ValueType();
+
+    return *this;
+}
+
+char HeaderParserValue::CharacterValue() const
+{
+    assert(m_type == HeaderParserValueType::CHARACTER);
+    return m_value.char_value;
+}
+
+int HeaderParserValue::IntegerValue() const
+{
+    assert(m_type == HeaderParserValueType::INTEGER);
+    return m_value.int_value;
+}
+
+double HeaderParserValue::FloatingPointValue() const
+{
+    assert(m_type == HeaderParserValueType::FLOATING_POINT);
+    return m_value.double_value;
+}
+
+std::string& HeaderParserValue::StringValue() const
+{
+    assert(m_type == HeaderParserValueType::STRING);
+    return *m_value.string_value;
+}
+
+std::string& HeaderParserValue::IdentifierValue() const
+{
+    assert(m_type == HeaderParserValueType::IDENTIFIER);
+    return *m_value.string_value;
+}
+
+std::string& HeaderParserValue::TypeNameValue() const
+{
+    assert(m_type == HeaderParserValueType::TYPE_NAME);
+    return *m_value.string_value;
+}

src/ZoneCodeGeneratorLib/Parsing/Header/Impl/HeaderParserValue.h

@@ -0,0 +1,86 @@
+#pragma once
+
+#include <string>
+
+#include "Utils/ClassUtils.h"
+#include "Parsing/TokenPos.h"
+
+enum class HeaderParserValueType
+{
+    // Meta tokens
+    INVALID,
+    END_OF_FILE,
+
+    // Single character
+    CHARACTER,
+
+    // Symbol tokens
+    SHIFT_LEFT,
+    SHIFT_RIGHT,
+    EQUALS,
+    NOT_EQUAL,
+    GREATER_EQUAL,
+    LESS_EQUAL,
+    LOGICAL_AND,
+    LOGICAL_OR,
+
+    // Generic token types
+    INTEGER,
+    FLOATING_POINT,
+    STRING,
+    IDENTIFIER,
+
+    // Parser created
+    TYPE_NAME,
+
+    // End
+    MAX
+};
+
+class HeaderParserValue
+{
+public:
+    TokenPos m_pos;
+    HeaderParserValueType m_type;
+    union ValueType
+    {
+        char char_value;
+        int int_value;
+        double double_value;
+        std::string* string_value;
+    } m_value;
+
+    static HeaderParserValue Invalid(TokenPos pos);
+    static HeaderParserValue EndOfFile(TokenPos pos);
+    static HeaderParserValue Character(TokenPos pos, char c);
+    static HeaderParserValue ShiftLeft(TokenPos pos);
+    static HeaderParserValue ShiftRight(TokenPos pos);
+    static HeaderParserValue Equals(TokenPos pos);
+    static HeaderParserValue NotEqual(TokenPos pos);
+    static HeaderParserValue GreaterEqual(TokenPos pos);
+    static HeaderParserValue LessEqual(TokenPos pos);
+    static HeaderParserValue LogicalAnd(TokenPos pos);
+    static HeaderParserValue LogicalOr(TokenPos pos);
+    static HeaderParserValue Integer(TokenPos pos, int value);
+    static HeaderParserValue FloatingPoint(TokenPos pos, double value);
+    static HeaderParserValue String(TokenPos pos, std::string* stringValue);
+    static HeaderParserValue Identifier(TokenPos pos, std::string* identifier);
+    static HeaderParserValue TypeName(TokenPos pos, std::string* typeName);
+
+private:
+    HeaderParserValue(TokenPos pos, HeaderParserValueType type);
+
+public:
+    ~HeaderParserValue();
+    HeaderParserValue(const HeaderParserValue& other) = delete;
+    HeaderParserValue(HeaderParserValue&& other) noexcept;
+    HeaderParserValue& operator=(const HeaderParserValue& other) = delete;
+    HeaderParserValue& operator=(HeaderParserValue&& other) noexcept;
+
+    _NODISCARD char CharacterValue() const;
+    _NODISCARD int IntegerValue() const;
+    _NODISCARD double FloatingPointValue() const;
+    _NODISCARD std::string& StringValue() const;
+    _NODISCARD std::string& IdentifierValue() const;
+    _NODISCARD std::string& TypeNameValue() const;
+};