Use custom parser values instead of simple parser values for parsing Zone definitions

src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.cpp

@@ -1,6 +1,5 @@
 #include "SimpleMatcherFactory.h"
 
-
 #include "SimpleMatcherAnyCharacterBesides.h"
 #include "SimpleMatcherCharacter.h"
 #include "SimpleMatcherKeyword.h"

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.cpp

@@ -1,61 +0,0 @@
-#include "ZoneDefinitionCommonMatchers.h"
-
-#include <sstream>
-
-#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
-
-std::unique_ptr<ZoneDefinitionCommonMatchers::matcher_t> ZoneDefinitionCommonMatchers::AssetName(const supplier_t* labelSupplier)
-{
-    const SimpleMatcherFactory create(labelSupplier);
-
-    return create.And({
-        create.Loop(create.Or({
-            create.Identifier(),
-            create.AnyCharBesides({',', '<', '>', '"', '\\', '*', '?', '|', ':'})
-        })),
-        create.Or({
-            create.Type(SimpleParserValueType::NEW_LINE),
-            create.Type(SimpleParserValueType::END_OF_FILE)
-        }).NoConsume()
-    }).Transform([](SimpleMatcherFactory::token_list_t& tokens)
-    {
-        std::ostringstream str;
-        auto previousType = SimpleParserValueType::INVALID;
-        auto previousCharacterWasSlash = false;
-
-        for (const auto& token : tokens)
-        {
-            switch (token.get().m_type)
-            {
-            case SimpleParserValueType::IDENTIFIER:
-                if (previousType == SimpleParserValueType::IDENTIFIER)
-                    str << " ";
-                str << token.get().IdentifierValue();
-                break;
-
-            case SimpleParserValueType::CHARACTER:
-                if (token.get().CharacterValue() == '/')
-                {
-                    if (previousType != SimpleParserValueType::CHARACTER || !previousCharacterWasSlash)
-                    {
-                        str << "/";
-                        previousCharacterWasSlash = true;
-                    }
-                }
-                else
-                {
-                    str << token.get().CharacterValue();
-                    previousCharacterWasSlash = false;
-                }
-                break;
-
-            default:
-                break;
-            }
-
-            previousType = token.get().m_type;
-        }
-
-        return SimpleParserValue::Identifier(tokens.front().get().GetPos(), new std::string(str.str()));
-    });
-}

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h

@@ -1,19 +0,0 @@
-#pragma once
-
-#include <limits>
-#include <memory>
-
-#include "Parsing/Simple/SimpleParserValue.h"
-#include "Parsing/Matcher/AbstractMatcher.h"
-#include "Parsing/Matcher/MatcherLabel.h"
-
-class ZoneDefinitionCommonMatchers
-{
-public:
-    typedef AbstractMatcher<SimpleParserValue> matcher_t;
-    typedef IMatcherForLabelSupplier<SimpleParserValue> supplier_t;
-
-    static constexpr int LABEL_ASSET_NAME = std::numeric_limits<int>::max() - 1;
-    
-    static std::unique_ptr<matcher_t> AssetName(const supplier_t* labelSupplier);
-};

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherCharacter.cpp

@@ -0,0 +1,14 @@
+#include "ZoneDefinitionMatcherCharacter.h"
+
+ZoneDefinitionMatcherCharacter::ZoneDefinitionMatcherCharacter(const char c)
+    : m_char(c)
+{
+}
+
+MatcherResult<ZoneDefinitionParserValue> ZoneDefinitionMatcherCharacter::CanMatch(ILexer<ZoneDefinitionParserValue>* lexer, const unsigned tokenOffset)
+{
+    const auto& token = lexer->GetToken(tokenOffset);
+    return token.m_type == ZoneDefinitionParserValueType::CHARACTER && token.CharacterValue() == m_char
+        ? MatcherResult<ZoneDefinitionParserValue>::Match(1)
+        : MatcherResult<ZoneDefinitionParserValue>::NoMatch();
+}

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherCharacter.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "Parsing/Matcher/AbstractMatcher.h"
+#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
+
+class ZoneDefinitionMatcherCharacter final : public AbstractMatcher<ZoneDefinitionParserValue>
+{
+    char m_char;
+
+protected:
+    MatcherResult<ZoneDefinitionParserValue> CanMatch(ILexer<ZoneDefinitionParserValue>* lexer, unsigned tokenOffset) override;
+
+public:
+    explicit ZoneDefinitionMatcherCharacter(char c);
+};

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.cpp

@@ -0,0 +1,35 @@
+#include "ZoneDefinitionMatcherFactory.h"
+
+#include "ZoneDefinitionMatcherCharacter.h"
+#include "ZoneDefinitionMatcherKeyword.h"
+#include "ZoneDefinitionMatcherValueType.h"
+
+ZoneDefinitionMatcherFactory::ZoneDefinitionMatcherFactory(const IMatcherForLabelSupplier<ZoneDefinitionParserValue>* labelSupplier)
+    : AbstractMatcherFactory(labelSupplier)
+{
+}
+
+MatcherFactoryWrapper<ZoneDefinitionParserValue> ZoneDefinitionMatcherFactory::Type(ZoneDefinitionParserValueType type) const
+{
+    return MatcherFactoryWrapper<ZoneDefinitionParserValue>(std::make_unique<ZoneDefinitionMatcherValueType>(type));
+}
+
+MatcherFactoryWrapper<ZoneDefinitionParserValue> ZoneDefinitionMatcherFactory::Keyword(std::string keyword) const
+{
+    return MatcherFactoryWrapper<ZoneDefinitionParserValue>(std::make_unique<ZoneDefinitionMatcherKeyword>(std::move(keyword)));
+}
+
+MatcherFactoryWrapper<ZoneDefinitionParserValue> ZoneDefinitionMatcherFactory::Field() const
+{
+    return MatcherFactoryWrapper<ZoneDefinitionParserValue>(std::make_unique<ZoneDefinitionMatcherValueType>(ZoneDefinitionParserValueType::FIELD));
+}
+
+MatcherFactoryWrapper<ZoneDefinitionParserValue> ZoneDefinitionMatcherFactory::String() const
+{
+    return MatcherFactoryWrapper<ZoneDefinitionParserValue>(std::make_unique<ZoneDefinitionMatcherValueType>(ZoneDefinitionParserValueType::STRING));
+}
+
+MatcherFactoryWrapper<ZoneDefinitionParserValue> ZoneDefinitionMatcherFactory::Char(char c) const
+{
+    return MatcherFactoryWrapper<ZoneDefinitionParserValue>(std::make_unique<ZoneDefinitionMatcherCharacter>(c));
+}

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <string>
+
+#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
+#include "Parsing/Matcher/AbstractMatcherFactory.h"
+
+class ZoneDefinitionMatcherFactory final : public AbstractMatcherFactory<ZoneDefinitionParserValue>
+{
+public:
+    explicit ZoneDefinitionMatcherFactory(const IMatcherForLabelSupplier<ZoneDefinitionParserValue>* labelSupplier);
+
+    _NODISCARD MatcherFactoryWrapper<ZoneDefinitionParserValue> Type(ZoneDefinitionParserValueType type) const;
+    _NODISCARD MatcherFactoryWrapper<ZoneDefinitionParserValue> Keyword(std::string keyword) const;
+    _NODISCARD MatcherFactoryWrapper<ZoneDefinitionParserValue> Field() const;
+    _NODISCARD MatcherFactoryWrapper<ZoneDefinitionParserValue> String() const;
+    _NODISCARD MatcherFactoryWrapper<ZoneDefinitionParserValue> Char(char c) const;
+};

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherKeyword.cpp

@@ -0,0 +1,16 @@
+#include "ZoneDefinitionMatcherKeyword.h"
+
+ZoneDefinitionMatcherKeyword::ZoneDefinitionMatcherKeyword(std::string value)
+    : m_value(std::move(value))
+{
+    const std::hash<std::string> hash;
+    m_hash = hash(m_value);
+}
+
+MatcherResult<ZoneDefinitionParserValue> ZoneDefinitionMatcherKeyword::CanMatch(ILexer<ZoneDefinitionParserValue>* lexer, const unsigned tokenOffset)
+{
+    const auto& token = lexer->GetToken(tokenOffset);
+    return token.m_type == ZoneDefinitionParserValueType::FIELD && token.FieldHash() == m_hash && token.FieldValue() == m_value
+        ? MatcherResult<ZoneDefinitionParserValue>::Match(1)
+        : MatcherResult<ZoneDefinitionParserValue>::NoMatch();
+}

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherKeyword.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <string>
+
+#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
+#include "Parsing/Matcher/AbstractMatcher.h"
+
+class ZoneDefinitionMatcherKeyword final : public AbstractMatcher<ZoneDefinitionParserValue>
+{
+    size_t m_hash;
+    std::string m_value;
+
+protected:
+    MatcherResult<ZoneDefinitionParserValue> CanMatch(ILexer<ZoneDefinitionParserValue>* lexer, unsigned tokenOffset) override;
+
+public:
+    explicit ZoneDefinitionMatcherKeyword(std::string value);
+};

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherValueType.cpp

@@ -0,0 +1,13 @@
+#include "ZoneDefinitionMatcherValueType.h"
+
+ZoneDefinitionMatcherValueType::ZoneDefinitionMatcherValueType(const ZoneDefinitionParserValueType type)
+    : m_type(type)
+{
+}
+
+MatcherResult<ZoneDefinitionParserValue> ZoneDefinitionMatcherValueType::CanMatch(ILexer<ZoneDefinitionParserValue>* lexer, const unsigned tokenOffset)
+{
+    return lexer->GetToken(tokenOffset).m_type == m_type
+        ? MatcherResult<ZoneDefinitionParserValue>::Match(1)
+        : MatcherResult<ZoneDefinitionParserValue>::NoMatch();
+}

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherValueType.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
+#include "Parsing/Matcher/AbstractMatcher.h"
+
+class ZoneDefinitionMatcherValueType final : public AbstractMatcher<ZoneDefinitionParserValue>
+{
+    ZoneDefinitionParserValueType m_type;
+
+protected:
+    MatcherResult<ZoneDefinitionParserValue> CanMatch(ILexer<ZoneDefinitionParserValue>* lexer, unsigned tokenOffset) override;
+
+public:
+    explicit ZoneDefinitionMatcherValueType(ZoneDefinitionParserValueType type);
+};

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.cpp

@@ -1,14 +0,0 @@
-#include "SequenceZoneDefinitionConsumeEmptyLines.h"
-
-#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
-
-SequenceZoneDefinitionConsumeEmptyLines::SequenceZoneDefinitionConsumeEmptyLines()
-{
-    const SimpleMatcherFactory create(this);
-
-    AddMatchers(create.Loop(create.Type(SimpleParserValueType::NEW_LINE)));
-}
-
-void SequenceZoneDefinitionConsumeEmptyLines::ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const
-{
-}

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.h

@@ -1,12 +0,0 @@
-#pragma once
-
-#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
-
-class SequenceZoneDefinitionConsumeEmptyLines final : public ZoneDefinitionParser::sequence_t
-{
-protected:
-    void ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const override;
-
-public:
-    SequenceZoneDefinitionConsumeEmptyLines();
-};

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionEntry.cpp

@@ -1,44 +1,38 @@
 #include "SequenceZoneDefinitionEntry.h"
 
-#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h"
+#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
-#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
 
 SequenceZoneDefinitionEntry::SequenceZoneDefinitionEntry()
 {
-    const SimpleMatcherFactory create(this);
+    const ZoneDefinitionMatcherFactory create(this);
     
-    AddLabeledMatchers(ZoneDefinitionCommonMatchers::AssetName(this), ZoneDefinitionCommonMatchers::LABEL_ASSET_NAME);
     AddMatchers({
-        create.Identifier().Capture(CAPTURE_TYPE_NAME),
+        create.Field().Capture(CAPTURE_TYPE_NAME),
         create.Char(','),
         create.Optional(create.Char(',').Tag(TAG_REFERENCE)),
         create.Or({
-            create.Type(SimpleParserValueType::STRING),
+            create.String(),
-            create.Label(ZoneDefinitionCommonMatchers::LABEL_ASSET_NAME)
+            create.Field()
-        }).Capture(CAPTURE_ASSET_NAME),
+        }).Capture(CAPTURE_ASSET_NAME)
-        create.Or({
-            create.Type(SimpleParserValueType::NEW_LINE),
-            create.Type(SimpleParserValueType::END_OF_FILE)
-        })
     });
 }
 
-void SequenceZoneDefinitionEntry::ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const
+void SequenceZoneDefinitionEntry::ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const
 {
     const auto& assetNameCapture = result.NextCapture(CAPTURE_ASSET_NAME);
 
     std::string assetName;
-    if (assetNameCapture.m_type == SimpleParserValueType::STRING)
+    if (assetNameCapture.m_type == ZoneDefinitionParserValueType::STRING)
     {
         assetName = assetNameCapture.StringValue();
     }
     else
     {
-        assetName = assetNameCapture.IdentifierValue();
+        assetName = assetNameCapture.FieldValue();
     }
 
     state->m_assets.emplace_back(
-        result.NextCapture(CAPTURE_TYPE_NAME).IdentifierValue(),
+        result.NextCapture(CAPTURE_TYPE_NAME).FieldValue(),
         assetName,
         result.NextTag() == TAG_REFERENCE);
 }

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionEntry.h

@@ -10,7 +10,7 @@ class SequenceZoneDefinitionEntry final : public ZoneDefinitionParser::sequence_
     static constexpr auto CAPTURE_ASSET_NAME = 2;
 
 protected:
-    void ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const override;
+    void ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const override;
 
 public:
     SequenceZoneDefinitionEntry();

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionIgnore.cpp

@@ -1,23 +1,19 @@
 #include "SequenceZoneDefinitionIgnore.h"
 
-#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
 
 SequenceZoneDefinitionIgnore::SequenceZoneDefinitionIgnore()
 {
-    const SimpleMatcherFactory create(this);
+    const ZoneDefinitionMatcherFactory create(this);
 
     AddMatchers({
         create.Keyword("ignore"),
         create.Char(','),
-        create.Identifier().Capture(CAPTURE_IGNORE_NAME),
+        create.Field().Capture(CAPTURE_IGNORE_NAME)
-        create.Or({
-            create.Type(SimpleParserValueType::NEW_LINE),
-            create.Type(SimpleParserValueType::END_OF_FILE)
-        })
     });
 }
 
-void SequenceZoneDefinitionIgnore::ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const
+void SequenceZoneDefinitionIgnore::ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const
 {
-    state->m_ignores.emplace_back(result.NextCapture(CAPTURE_IGNORE_NAME).IdentifierValue());
+    state->m_ignores.emplace_back(result.NextCapture(CAPTURE_IGNORE_NAME).FieldValue());
 }

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionIgnore.h

@@ -7,7 +7,7 @@ class SequenceZoneDefinitionIgnore final : public ZoneDefinitionParser::sequence
     static constexpr auto CAPTURE_IGNORE_NAME = 1;
 
 protected:
-    void ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const override;
+    void ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const override;
 
 public:
     SequenceZoneDefinitionIgnore();

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionInclude.cpp

@@ -1,23 +1,19 @@
 #include "SequenceZoneDefinitionInclude.h"
 
-#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
 
 SequenceZoneDefinitionInclude::SequenceZoneDefinitionInclude()
 {
-    const SimpleMatcherFactory create(this);
+    const ZoneDefinitionMatcherFactory create(this);
 
     AddMatchers({
         create.Keyword("include"),
         create.Char(','),
-        create.Identifier().Capture(CAPTURE_INCLUDE_NAME),
+        create.Field().Capture(CAPTURE_INCLUDE_NAME)
-        create.Or({
-            create.Type(SimpleParserValueType::NEW_LINE),
-            create.Type(SimpleParserValueType::END_OF_FILE)
-        })
     });
 }
 
-void SequenceZoneDefinitionInclude::ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const
+void SequenceZoneDefinitionInclude::ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const
 {
-    state->m_includes.emplace_back(result.NextCapture(CAPTURE_INCLUDE_NAME).IdentifierValue());
+    state->m_includes.emplace_back(result.NextCapture(CAPTURE_INCLUDE_NAME).FieldValue());
 }

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionInclude.h

@@ -7,7 +7,7 @@ class SequenceZoneDefinitionInclude final : public ZoneDefinitionParser::sequenc
     static constexpr auto CAPTURE_INCLUDE_NAME = 1;
 
 protected:
-    void ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const override;
+    void ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const override;
 
 public:
     SequenceZoneDefinitionInclude();

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionMetaData.cpp

@@ -1,24 +1,20 @@
 #include "SequenceZoneDefinitionMetaData.h"
 
-#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
 
 SequenceZoneDefinitionMetaData::SequenceZoneDefinitionMetaData()
 {
-    const SimpleMatcherFactory create(this);
+    const ZoneDefinitionMatcherFactory create(this);
 
     AddMatchers({
         create.Char('>'),
-        create.Identifier().Capture(CAPTURE_KEY),
+        create.Field().Capture(CAPTURE_KEY),
         create.Char(','),
-        create.Identifier().Capture(CAPTURE_VALUE),
+        create.Field().Capture(CAPTURE_VALUE)
-        create.Or({
-            create.Type(SimpleParserValueType::NEW_LINE),
-            create.Type(SimpleParserValueType::END_OF_FILE)
-        })
     });
 }
 
-void SequenceZoneDefinitionMetaData::ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const
+void SequenceZoneDefinitionMetaData::ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const
 {
-    state->m_metadata.insert(std::make_pair(result.NextCapture(CAPTURE_KEY).IdentifierValue(), result.NextCapture(CAPTURE_VALUE).IdentifierValue()));
+    state->m_metadata.insert(std::make_pair(result.NextCapture(CAPTURE_KEY).FieldValue(), result.NextCapture(CAPTURE_VALUE).FieldValue()));
 }

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionMetaData.h

@@ -8,7 +8,7 @@ class SequenceZoneDefinitionMetaData final : public ZoneDefinitionParser::sequen
     static constexpr auto CAPTURE_VALUE = 2;
 
 protected:
-    void ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const override;
+    void ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const override;
 
 public:
     SequenceZoneDefinitionMetaData();

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.cpp

@@ -1,9 +1,71 @@
 #include "ZoneDefinitionLexer.h"
 
+#include <sstream>
+
 ZoneDefinitionLexer::ZoneDefinitionLexer(IParserLineStream* stream)
-    : SimpleLexer(stream)
+    : AbstractLexer(stream)
 {
-    SetShouldEmitNewLineTokens(true);
+}
-    SetShouldReadStrings(true);
+
-    SetShouldReadNumbers(false);
+std::string ZoneDefinitionLexer::ReadField()
+{
+    const auto& currentLine = CurrentLine();
+    assert(m_current_line_offset >= 1);
+
+    const auto startPos = m_current_line_offset - 1;
+    const auto lineSize = currentLine.m_line.size();
+    while (m_current_line_offset < lineSize)
+    {
+        const auto c = currentLine.m_line[m_current_line_offset];
+
+        bool isNonFieldCharacter;
+        switch (c)
+        {
+        case '\"':
+        case '>':
+        case '<':
+        case ',':
+            isNonFieldCharacter = true;
+            break;
+        default:
+            isNonFieldCharacter = false;
+            break;
+        }
+
+        if (isNonFieldCharacter)
+            break;
+
+        m_current_line_offset++;
+    }
+
+    return std::string(currentLine.m_line, startPos, m_current_line_offset - startPos);
+}
+
+ZoneDefinitionParserValue ZoneDefinitionLexer::GetNextToken()
+{
+    auto c = NextChar();
+
+    while (c != EOF)
+    {
+        switch (c)
+        {
+        case '\"':
+            return ZoneDefinitionParserValue::String(GetPreviousCharacterPos(), new std::string(ReadString()));
+
+        case '>':
+        case '<':
+        case ',':
+            return ZoneDefinitionParserValue::Character(GetPreviousCharacterPos(), static_cast<char>(c));
+
+        default:
+            if (isspace(c))
+                break;
+
+            return ZoneDefinitionParserValue::Field(GetPreviousCharacterPos(), new std::string(ReadField()));
+        }
+
+        c = NextChar();
+    }
+
+    return ZoneDefinitionParserValue::EndOfFile(TokenPos());
 }

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.h

@@ -1,9 +1,15 @@
 #pragma once
 
-#include "Parsing/Simple/SimpleLexer.h"
+#include "ZoneDefinitionParserValue.h"
+#include "Parsing/Impl/AbstractLexer.h"
 
-class ZoneDefinitionLexer final : public SimpleLexer
+class ZoneDefinitionLexer final : public AbstractLexer<ZoneDefinitionParserValue>
 {
+    std::string ReadField();
+
+protected:
+    ZoneDefinitionParserValue GetNextToken() override;
+
 public:
     explicit ZoneDefinitionLexer(IParserLineStream* stream);
 };

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp

@@ -1,25 +1,22 @@
 #include "ZoneDefinitionParser.h"
 
-
-#include "Sequence/SequenceZoneDefinitionConsumeEmptyLines.h"
 #include "Sequence/SequenceZoneDefinitionEntry.h"
 #include "Sequence/SequenceZoneDefinitionIgnore.h"
 #include "Sequence/SequenceZoneDefinitionInclude.h"
 #include "Sequence/SequenceZoneDefinitionMetaData.h"
 
-ZoneDefinitionParser::ZoneDefinitionParser(SimpleLexer* lexer)
+ZoneDefinitionParser::ZoneDefinitionParser(ZoneDefinitionLexer* lexer)
     : AbstractParser(lexer, std::make_unique<ZoneDefinition>())
 {
 }
 
-const std::vector<AbstractParser<SimpleParserValue, ZoneDefinition>::sequence_t*>& ZoneDefinitionParser::GetTestsForState()
+const std::vector<AbstractParser<ZoneDefinitionParserValue, ZoneDefinition>::sequence_t*>& ZoneDefinitionParser::GetTestsForState()
 {
     static std::vector<sequence_t*> tests({
         new SequenceZoneDefinitionMetaData(),
         new SequenceZoneDefinitionInclude(),
         new SequenceZoneDefinitionIgnore(),
-        new SequenceZoneDefinitionEntry(),
+        new SequenceZoneDefinitionEntry()
-        new SequenceZoneDefinitionConsumeEmptyLines()
     });
 
     return tests;

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.h

@@ -1,16 +1,16 @@
 #pragma once
 
-#include "Parsing/Simple/SimpleLexer.h"
+#include "ZoneDefinitionLexer.h"
-#include "Parsing/Simple/SimpleParserValue.h"
+#include "ZoneDefinitionParserValue.h"
 #include "Parsing/Impl/AbstractParser.h"
 #include "Zone/Definition/ZoneDefinition.h"
 
-class ZoneDefinitionParser final : public AbstractParser<SimpleParserValue, ZoneDefinition>
+class ZoneDefinitionParser final : public AbstractParser<ZoneDefinitionParserValue, ZoneDefinition>
 {
 protected:
     const std::vector<sequence_t*>& GetTestsForState() override;
 
 public:
-    explicit ZoneDefinitionParser(SimpleLexer* lexer);
+    explicit ZoneDefinitionParser(ZoneDefinitionLexer* lexer);
     std::unique_ptr<ZoneDefinition> GetParsedValue();
 };

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParserValue.cpp

@@ -0,0 +1,115 @@
+#include "ZoneDefinitionParserValue.h"
+
+#include <cassert>
+
+ZoneDefinitionParserValue ZoneDefinitionParserValue::Invalid(const TokenPos pos)
+{
+    ZoneDefinitionParserValue pv(pos, ZoneDefinitionParserValueType::INVALID);
+    return pv;
+}
+
+ZoneDefinitionParserValue ZoneDefinitionParserValue::EndOfFile(const TokenPos pos)
+{
+    ZoneDefinitionParserValue pv(pos, ZoneDefinitionParserValueType::END_OF_FILE);
+    return pv;
+}
+
+ZoneDefinitionParserValue ZoneDefinitionParserValue::Character(const TokenPos pos, const char c)
+{
+    ZoneDefinitionParserValue pv(pos, ZoneDefinitionParserValueType::CHARACTER);
+    pv.m_value.char_value = c;
+    return pv;
+}
+
+ZoneDefinitionParserValue ZoneDefinitionParserValue::String(const TokenPos pos, std::string* str)
+{
+    ZoneDefinitionParserValue pv(pos, ZoneDefinitionParserValueType::STRING);
+    pv.m_value.string_value = str;
+    return pv;
+}
+
+ZoneDefinitionParserValue ZoneDefinitionParserValue::Field(const TokenPos pos, std::string* field)
+{
+    ZoneDefinitionParserValue pv(pos, ZoneDefinitionParserValueType::FIELD);
+    pv.m_value.string_value = field;
+    pv.m_hash = std::hash<std::string>()(*field);
+    return pv;
+}
+
+ZoneDefinitionParserValue::ZoneDefinitionParserValue(const TokenPos pos, const ZoneDefinitionParserValueType type)
+    : m_pos(pos),
+    m_type(type),
+    m_hash(0),
+    m_value{}
+{
+}
+
+ZoneDefinitionParserValue::~ZoneDefinitionParserValue()
+{
+    switch (m_type)
+    {
+    case ZoneDefinitionParserValueType::STRING:
+    case ZoneDefinitionParserValueType::FIELD:
+        delete m_value.string_value;
+        break;
+
+    default:
+        break;
+    }
+
+    m_value = ValueType();
+}
+
+ZoneDefinitionParserValue::ZoneDefinitionParserValue(ZoneDefinitionParserValue&& other) noexcept
+    : m_pos(other.m_pos),
+    m_type(other.m_type),
+    m_hash(other.m_hash),
+    m_value(other.m_value)
+{
+    other.m_value = ValueType();
+}
+
+ZoneDefinitionParserValue& ZoneDefinitionParserValue::operator=(ZoneDefinitionParserValue&& other) noexcept
+{
+    m_pos = other.m_pos;
+    m_type = other.m_type;
+    m_value = other.m_value;
+    m_hash = other.m_hash;
+    other.m_value = ValueType();
+
+    return *this;
+}
+
+bool ZoneDefinitionParserValue::IsEof() const
+{
+    return m_type == ZoneDefinitionParserValueType::END_OF_FILE;
+}
+
+const TokenPos& ZoneDefinitionParserValue::GetPos() const
+{
+    return m_pos;
+}
+
+char ZoneDefinitionParserValue::CharacterValue() const
+{
+    assert(m_type == ZoneDefinitionParserValueType::CHARACTER);
+    return m_value.char_value;
+}
+
+std::string& ZoneDefinitionParserValue::StringValue() const
+{
+    assert(m_type == ZoneDefinitionParserValueType::STRING);
+    return *m_value.string_value;
+}
+
+std::string& ZoneDefinitionParserValue::FieldValue() const
+{
+    assert(m_type == ZoneDefinitionParserValueType::FIELD);
+    return *m_value.string_value;
+}
+
+size_t ZoneDefinitionParserValue::FieldHash() const
+{
+    assert(m_type == ZoneDefinitionParserValueType::FIELD);
+    return m_hash;
+}

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParserValue.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <string>
+
+#include "Parsing/IParserValue.h"
+#include "Utils/ClassUtils.h"
+#include "Parsing/TokenPos.h"
+
+enum class ZoneDefinitionParserValueType
+{
+    // Meta tokens
+    INVALID,
+    END_OF_FILE,
+    
+    CHARACTER,
+    STRING,
+    FIELD,
+
+    // End
+    MAX
+};
+
+class ZoneDefinitionParserValue final : public IParserValue
+{
+public:
+    TokenPos m_pos;
+    ZoneDefinitionParserValueType m_type;
+    size_t m_hash;
+    union ValueType
+    {
+        char char_value;
+        std::string* string_value;
+    } m_value;
+
+    static ZoneDefinitionParserValue Invalid(TokenPos pos);
+    static ZoneDefinitionParserValue EndOfFile(TokenPos pos);
+    static ZoneDefinitionParserValue Character(TokenPos pos, char c);
+    static ZoneDefinitionParserValue String(TokenPos pos, std::string* str);
+    static ZoneDefinitionParserValue Field(TokenPos pos, std::string* field);
+
+private:
+    ZoneDefinitionParserValue(TokenPos pos, ZoneDefinitionParserValueType type);
+
+public:
+    ~ZoneDefinitionParserValue() override;
+    ZoneDefinitionParserValue(const ZoneDefinitionParserValue& other) = delete;
+    ZoneDefinitionParserValue(ZoneDefinitionParserValue&& other) noexcept;
+    ZoneDefinitionParserValue& operator=(const ZoneDefinitionParserValue& other) = delete;
+    ZoneDefinitionParserValue& operator=(ZoneDefinitionParserValue&& other) noexcept;
+
+    _NODISCARD bool IsEof() const override;
+    _NODISCARD const TokenPos& GetPos() const override;
+
+    _NODISCARD char CharacterValue() const;
+    _NODISCARD std::string& StringValue() const;
+    _NODISCARD std::string& FieldValue() const;
+    _NODISCARD size_t FieldHash() const;
+};