Add ZoneDefinition Lexer

src/Parser/Parsing/Impl/AbstractLexer.h

@@ -64,7 +64,7 @@ protected:
         {
             while (peekLine >= m_line_cache.size())
             {
-                if (m_line_cache.back().IsEof())
+                if (!m_line_cache.empty() && m_line_cache.back().IsEof())
                     return EOF;
 
                 m_line_cache.push_back(m_stream->NextLine());

src/Parser/Parsing/Simple/Matcher/SimpleMatcherAnyCharacterBesides.cpp

@@ -0,0 +1,14 @@
+#include "SimpleMatcherAnyCharacterBesides.h"
+
+SimpleMatcherAnyCharacterBesides::SimpleMatcherAnyCharacterBesides(std::vector<char> chars)
+    : m_chars(std::move(chars))
+{
+}
+
+MatcherResult<SimpleParserValue> SimpleMatcherAnyCharacterBesides::CanMatch(ILexer<SimpleParserValue>* lexer, const unsigned tokenOffset)
+{
+    const auto& token = lexer->GetToken(tokenOffset);
+    return token.m_type == SimpleParserValueType::CHARACTER && std::find(m_chars.begin(), m_chars.end(), token.CharacterValue()) == m_chars.end()
+               ? MatcherResult<SimpleParserValue>::Match(1)
+               : MatcherResult<SimpleParserValue>::NoMatch();
+}

src/Parser/Parsing/Simple/Matcher/SimpleMatcherAnyCharacterBesides.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <vector>
+
+#include "Parsing/Simple/SimpleParserValue.h"
+#include "Parsing/Matcher/AbstractMatcher.h"
+
+class SimpleMatcherAnyCharacterBesides final : public AbstractMatcher<SimpleParserValue>
+{
+    std::vector<char> m_chars;
+
+protected:
+    MatcherResult<SimpleParserValue> CanMatch(ILexer<SimpleParserValue>* lexer, unsigned tokenOffset) override;
+
+public:
+    explicit SimpleMatcherAnyCharacterBesides(std::vector<char> chars);
+};

src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.cpp

@@ -1,5 +1,7 @@
 #include "SimpleMatcherFactory.h"
 
+
+#include "SimpleMatcherAnyCharacterBesides.h"
 #include "SimpleMatcherCharacter.h"
 #include "SimpleMatcherKeyword.h"
 #include "SimpleMatcherValueType.h"
@@ -38,3 +40,8 @@ MatcherFactoryWrapper<SimpleParserValue> SimpleMatcherFactory::Char(char c) cons
 {
     return MatcherFactoryWrapper<SimpleParserValue>(std::make_unique<SimpleMatcherCharacter>(c));
 }
+
+MatcherFactoryWrapper<SimpleParserValue> SimpleMatcherFactory::AnyCharBesides(std::vector<char> chars) const
+{
+    return MatcherFactoryWrapper<SimpleParserValue>(std::make_unique<SimpleMatcherAnyCharacterBesides>(std::move(chars)));
+}

src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.h

@@ -16,4 +16,5 @@ public:
     _NODISCARD MatcherFactoryWrapper<SimpleParserValue> Integer() const;
     _NODISCARD MatcherFactoryWrapper<SimpleParserValue> FloatingPoint() const;
     _NODISCARD MatcherFactoryWrapper<SimpleParserValue> Char(char c) const;
+    _NODISCARD MatcherFactoryWrapper<SimpleParserValue> AnyCharBesides(std::vector<char> chars) const;
 };

src/Parser/Parsing/Simple/SimpleLexer.cpp

@@ -1,25 +1,56 @@
 #include "SimpleLexer.h"
 
 SimpleLexer::SimpleLexer(IParserLineStream* stream)
-    : AbstractLexer(stream)
+    : AbstractLexer(stream),
+      m_emit_new_line_tokens(false),
+      m_read_strings(true),
+      m_last_line(1)
 {
 }
 
+void SimpleLexer::SetShouldEmitNewLineTokens(const bool value)
+{
+    m_emit_new_line_tokens = value;
+}
+
+void SimpleLexer::SetShouldReadStrings(const bool value)
+{
+    m_read_strings = value;
+}
+
+void SimpleLexer::SetShouldReadNumbers(const bool value)
+{
+    m_read_numbers = value;
+}
+
 SimpleParserValue SimpleLexer::GetNextToken()
 {
+    PeekChar();
+    const auto nextCharPos = GetNextCharacterPos();
+    if(m_emit_new_line_tokens && nextCharPos.m_line > m_last_line)
+    {
+        m_last_line++;
+        return SimpleParserValue::NewLine(GetPreviousCharacterPos());
+    }
+
     auto c = NextChar();
 
     while (isspace(c))
-        c = NextChar();
+    {
+        if (m_emit_new_line_tokens && c == '\n')
+            return SimpleParserValue::NewLine(GetPreviousCharacterPos());
 
-    if(c == EOF)
+        c = NextChar();
+    }
+
+    if (c == EOF)
         return SimpleParserValue::EndOfFile(TokenPos());
 
-    if (c == '\"')
+    if (m_read_strings && c == '\"')
         return SimpleParserValue::String(GetPreviousCharacterPos(), new std::string(ReadString()));
 
     const auto pos = GetPreviousCharacterPos();
-    if (isdigit(c))
+    if (m_read_numbers && isdigit(c))
     {
         bool isFloatingPointValue;
         double doubleValue;

src/Parser/Parsing/Simple/SimpleLexer.h

@@ -3,11 +3,20 @@
 #include "SimpleParserValue.h"
 #include "Parsing/Impl/AbstractLexer.h"
 
-class SimpleLexer final : public AbstractLexer<SimpleParserValue>
+class SimpleLexer : public AbstractLexer<SimpleParserValue>
 {
+    bool m_emit_new_line_tokens;
+    bool m_read_strings;
+    bool m_read_numbers;
+    int m_last_line;
+
 protected:
     SimpleParserValue GetNextToken() override;
 
+    void SetShouldEmitNewLineTokens(bool value);
+    void SetShouldReadStrings(bool value);
+    void SetShouldReadNumbers(bool value);
+
 public:
     explicit SimpleLexer(IParserLineStream* stream);
 };

src/Parser/Parsing/Simple/SimpleParserValue.cpp

@@ -14,6 +14,12 @@ SimpleParserValue SimpleParserValue::EndOfFile(const TokenPos pos)
     return pv;
 }
 
+SimpleParserValue SimpleParserValue::NewLine(const TokenPos pos)
+{
+    SimpleParserValue pv(pos, SimpleParserValueType::NEW_LINE);
+    return pv;
+}
+
 SimpleParserValue SimpleParserValue::Character(const TokenPos pos, const char c)
 {
     SimpleParserValue pv(pos, SimpleParserValueType::CHARACTER);

src/Parser/Parsing/Simple/SimpleParserValue.h

@@ -11,6 +11,7 @@ enum class SimpleParserValueType
     // Meta tokens
     INVALID,
     END_OF_FILE,
+    NEW_LINE,
 
     // Single character
     CHARACTER,
@@ -41,6 +42,7 @@ public:
 
     static SimpleParserValue Invalid(TokenPos pos);
     static SimpleParserValue EndOfFile(TokenPos pos);
+    static SimpleParserValue NewLine(TokenPos pos);
     static SimpleParserValue Character(TokenPos pos, char c);
     static SimpleParserValue Integer(TokenPos pos, int value);
     static SimpleParserValue FloatingPoint(TokenPos pos, double value);

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.cpp

@@ -0,0 +1,58 @@
+#include "ZoneDefinitionCommonMatchers.h"
+
+#include <sstream>
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+std::unique_ptr<ZoneDefinitionCommonMatchers::matcher_t> ZoneDefinitionCommonMatchers::AssetName(const supplier_t* labelSupplier)
+{
+    const SimpleMatcherFactory create(labelSupplier);
+
+    return create.And({
+        create.Loop(create.Or({
+            create.Identifier(),
+            create.AnyCharBesides({',', '<', '>', '"', '\\', '*', '?', '|', ':'})
+        })),
+        create.Type(SimpleParserValueType::NEW_LINE).NoConsume()
+    }).Transform([](SimpleMatcherFactory::token_list_t& tokens)
+    {
+        std::ostringstream str;
+        auto previousType = SimpleParserValueType::INVALID;
+        auto previousCharacterWasSlash = false;
+
+        for (const auto& token : tokens)
+        {
+            switch (token.get().m_type)
+            {
+            case SimpleParserValueType::IDENTIFIER:
+                if (previousType == SimpleParserValueType::IDENTIFIER)
+                    str << " ";
+                str << token.get().IdentifierValue();
+                break;
+
+            case SimpleParserValueType::CHARACTER:
+                if (token.get().CharacterValue() == '/')
+                {
+                    if (previousType == SimpleParserValueType::CHARACTER && !previousCharacterWasSlash)
+                    {
+                        str << "/";
+                        previousCharacterWasSlash = true;
+                    }
+                }
+                else
+                {
+                    str << token.get().CharacterValue();
+                    previousCharacterWasSlash = false;
+                }
+                break;
+
+            default:
+                break;
+            }
+
+            previousType = token.get().m_type;
+        }
+
+        return SimpleParserValue::Identifier(tokens.front().get().GetPos(), new std::string(str.str()));
+    });
+}

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <limits>
+#include <memory>
+
+#include "Parsing/Simple/SimpleParserValue.h"
+#include "Parsing/Matcher/AbstractMatcher.h"
+#include "Parsing/Matcher/MatcherLabel.h"
+
+class ZoneDefinitionCommonMatchers
+{
+public:
+    typedef AbstractMatcher<SimpleParserValue> matcher_t;
+    typedef IMatcherForLabelSupplier<SimpleParserValue> supplier_t;
+
+    static constexpr int LABEL_ASSET_NAME = std::numeric_limits<int>::max() - 1;
+    
+    static std::unique_ptr<matcher_t> AssetName(const supplier_t* labelSupplier);
+};

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.cpp

@@ -0,0 +1,14 @@
+#include "SequenceZoneDefinitionConsumeEmptyLines.h"
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+SequenceZoneDefinitionConsumeEmptyLines::SequenceZoneDefinitionConsumeEmptyLines()
+{
+    const SimpleMatcherFactory create(this);
+
+    AddMatchers(create.Loop(create.Type(SimpleParserValueType::NEW_LINE)));
+}
+
+void SequenceZoneDefinitionConsumeEmptyLines::ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const
+{
+}

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
+
+class SequenceZoneDefinitionConsumeEmptyLines final : public ZoneDefinitionParser::sequence_t
+{
+protected:
+    void ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const override;
+
+public:
+    SequenceZoneDefinitionConsumeEmptyLines();
+};

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionEntry.cpp

@@ -1,16 +1,19 @@
 #include "SequenceZoneDefinitionEntry.h"
 
+#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h"
 #include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
 
 SequenceZoneDefinitionEntry::SequenceZoneDefinitionEntry()
 {
     const SimpleMatcherFactory create(this);
 
+    AddLabeledMatchers(ZoneDefinitionCommonMatchers::AssetName(this), ZoneDefinitionCommonMatchers::LABEL_ASSET_NAME);
     AddMatchers({
         create.Identifier().Capture(CAPTURE_TYPE_NAME),
         create.Char(','),
         create.Optional(create.Char(',').Tag(TAG_REFERENCE)),
-        create.Identifier().Capture(CAPTURE_ASSET_NAME)
+        create.Label(ZoneDefinitionCommonMatchers::LABEL_ASSET_NAME).Capture(CAPTURE_ASSET_NAME),
+        create.Type(SimpleParserValueType::NEW_LINE)
     });
 }
 

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionIgnore.cpp

@@ -9,8 +9,9 @@ SequenceZoneDefinitionIgnore::SequenceZoneDefinitionIgnore()
     AddMatchers({
         create.Keyword("ignore"),
         create.Char(','),
-        create.Identifier().Capture(CAPTURE_IGNORE_NAME)
-        });
+        create.Identifier().Capture(CAPTURE_IGNORE_NAME),
+        create.Type(SimpleParserValueType::NEW_LINE)
+    });
 }
 
 void SequenceZoneDefinitionIgnore::ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionInclude.cpp

@@ -9,7 +9,8 @@ SequenceZoneDefinitionInclude::SequenceZoneDefinitionInclude()
     AddMatchers({
         create.Keyword("include"),
         create.Char(','),
-        create.Identifier().Capture(CAPTURE_INCLUDE_NAME)
+        create.Identifier().Capture(CAPTURE_INCLUDE_NAME),
+        create.Type(SimpleParserValueType::NEW_LINE)
     });
 }
 

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionMetaData.cpp

@@ -10,7 +10,8 @@ SequenceZoneDefinitionMetaData::SequenceZoneDefinitionMetaData()
         create.Char('>'),
         create.Identifier().Capture(CAPTURE_KEY),
         create.Char(','),
-        create.Identifier().Capture(CAPTURE_VALUE)
+        create.Identifier().Capture(CAPTURE_VALUE),
+        create.Type(SimpleParserValueType::NEW_LINE)
     });
 }
 

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.cpp

@@ -0,0 +1,9 @@
+#include "ZoneDefinitionLexer.h"
+
+ZoneDefinitionLexer::ZoneDefinitionLexer(IParserLineStream* stream)
+    : SimpleLexer(stream)
+{
+    SetShouldEmitNewLineTokens(true);
+    SetShouldReadStrings(true);
+    SetShouldReadNumbers(false);
+}

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "Parsing/Simple/SimpleLexer.h"
+
+class ZoneDefinitionLexer final : public SimpleLexer
+{
+public:
+    explicit ZoneDefinitionLexer(IParserLineStream* stream);
+};

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp

@@ -1,5 +1,7 @@
 #include "ZoneDefinitionParser.h"
 
+
+#include "Sequence/SequenceZoneDefinitionConsumeEmptyLines.h"
 #include "Sequence/SequenceZoneDefinitionEntry.h"
 #include "Sequence/SequenceZoneDefinitionIgnore.h"
 #include "Sequence/SequenceZoneDefinitionInclude.h"
@@ -16,7 +18,8 @@ const std::vector<AbstractParser<SimpleParserValue, ZoneDefinition>::sequence_t*
         new SequenceZoneDefinitionMetaData(),
         new SequenceZoneDefinitionInclude(),
         new SequenceZoneDefinitionIgnore(),
-        new SequenceZoneDefinitionEntry()
+        new SequenceZoneDefinitionEntry(),
+        new SequenceZoneDefinitionConsumeEmptyLines()
     });
 
     return tests;

src/ZoneCommon/Zone/Definition/ZoneDefinitionStream.cpp

@@ -7,6 +7,7 @@
 
 #include "Parsing/Impl/DefinesStreamProxy.h"
 #include "Parsing/Simple/SimpleLexer.h"
+#include "Parsing/ZoneDefinition/ZoneDefinitionLexer.h"
 #include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
 
 ZoneDefinitionInputStream::ZoneDefinitionInputStream(std::istream& stream, std::string fileName, bool verbose)
@@ -42,7 +43,7 @@ std::unique_ptr<ZoneDefinition> ZoneDefinitionInputStream::ReadDefinition()
         std::cout << "Reading zone definition file: " << m_file_name << std::endl;
     }
 
-    const auto lexer = std::make_unique<SimpleLexer>(m_stream);
+    const auto lexer = std::make_unique<ZoneDefinitionLexer>(m_stream);
     const auto parser = std::make_unique<ZoneDefinitionParser>(lexer.get());
 
     const auto start = std::chrono::steady_clock::now();