diff --git a/src/Parser/Parsing/Impl/AbstractLexer.h b/src/Parser/Parsing/Impl/AbstractLexer.h index c77df139..37fc15d4 100644 --- a/src/Parser/Parsing/Impl/AbstractLexer.h +++ b/src/Parser/Parsing/Impl/AbstractLexer.h @@ -64,7 +64,7 @@ protected: { while (peekLine >= m_line_cache.size()) { - if (m_line_cache.back().IsEof()) + if (!m_line_cache.empty() && m_line_cache.back().IsEof()) return EOF; m_line_cache.push_back(m_stream->NextLine()); diff --git a/src/Parser/Parsing/Simple/Matcher/SimpleMatcherAnyCharacterBesides.cpp b/src/Parser/Parsing/Simple/Matcher/SimpleMatcherAnyCharacterBesides.cpp new file mode 100644 index 00000000..0208ffcf --- /dev/null +++ b/src/Parser/Parsing/Simple/Matcher/SimpleMatcherAnyCharacterBesides.cpp @@ -0,0 +1,14 @@ +#include "SimpleMatcherAnyCharacterBesides.h" + +SimpleMatcherAnyCharacterBesides::SimpleMatcherAnyCharacterBesides(std::vector chars) + : m_chars(std::move(chars)) +{ +} + +MatcherResult SimpleMatcherAnyCharacterBesides::CanMatch(ILexer* lexer, const unsigned tokenOffset) +{ + const auto& token = lexer->GetToken(tokenOffset); + return token.m_type == SimpleParserValueType::CHARACTER && std::find(m_chars.begin(), m_chars.end(), token.CharacterValue()) == m_chars.end() + ? MatcherResult::Match(1) + : MatcherResult::NoMatch(); +} diff --git a/src/Parser/Parsing/Simple/Matcher/SimpleMatcherAnyCharacterBesides.h b/src/Parser/Parsing/Simple/Matcher/SimpleMatcherAnyCharacterBesides.h new file mode 100644 index 00000000..46e25408 --- /dev/null +++ b/src/Parser/Parsing/Simple/Matcher/SimpleMatcherAnyCharacterBesides.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +#include "Parsing/Simple/SimpleParserValue.h" +#include "Parsing/Matcher/AbstractMatcher.h" + +class SimpleMatcherAnyCharacterBesides final : public AbstractMatcher +{ + std::vector m_chars; + +protected: + MatcherResult CanMatch(ILexer* lexer, unsigned tokenOffset) override; + +public: + explicit SimpleMatcherAnyCharacterBesides(std::vector chars); +}; diff --git a/src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.cpp b/src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.cpp index e8b3692d..58417e40 100644 --- a/src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.cpp +++ b/src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.cpp @@ -1,5 +1,7 @@ #include "SimpleMatcherFactory.h" + +#include "SimpleMatcherAnyCharacterBesides.h" #include "SimpleMatcherCharacter.h" #include "SimpleMatcherKeyword.h" #include "SimpleMatcherValueType.h" @@ -38,3 +40,8 @@ MatcherFactoryWrapper SimpleMatcherFactory::Char(char c) cons { return MatcherFactoryWrapper(std::make_unique(c)); } + +MatcherFactoryWrapper SimpleMatcherFactory::AnyCharBesides(std::vector chars) const +{ + return MatcherFactoryWrapper(std::make_unique(std::move(chars))); +} diff --git a/src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.h b/src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.h index 7c9c26ee..abbec723 100644 --- a/src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.h +++ b/src/Parser/Parsing/Simple/Matcher/SimpleMatcherFactory.h @@ -16,4 +16,5 @@ public: _NODISCARD MatcherFactoryWrapper Integer() const; _NODISCARD MatcherFactoryWrapper FloatingPoint() const; _NODISCARD MatcherFactoryWrapper Char(char c) const; + _NODISCARD MatcherFactoryWrapper AnyCharBesides(std::vector chars) const; }; diff --git a/src/Parser/Parsing/Simple/SimpleLexer.cpp b/src/Parser/Parsing/Simple/SimpleLexer.cpp index f0105785..52199490 100644 --- a/src/Parser/Parsing/Simple/SimpleLexer.cpp +++ b/src/Parser/Parsing/Simple/SimpleLexer.cpp @@ -1,25 +1,56 @@ #include "SimpleLexer.h" SimpleLexer::SimpleLexer(IParserLineStream* stream) - : AbstractLexer(stream) + : AbstractLexer(stream), + m_emit_new_line_tokens(false), + m_read_strings(true), + m_last_line(1) { } +void SimpleLexer::SetShouldEmitNewLineTokens(const bool value) +{ + m_emit_new_line_tokens = value; +} + +void SimpleLexer::SetShouldReadStrings(const bool value) +{ + m_read_strings = value; +} + +void SimpleLexer::SetShouldReadNumbers(const bool value) +{ + m_read_numbers = value; +} + SimpleParserValue SimpleLexer::GetNextToken() { + PeekChar(); + const auto nextCharPos = GetNextCharacterPos(); + if(m_emit_new_line_tokens && nextCharPos.m_line > m_last_line) + { + m_last_line++; + return SimpleParserValue::NewLine(GetPreviousCharacterPos()); + } + auto c = NextChar(); while (isspace(c)) - c = NextChar(); + { + if (m_emit_new_line_tokens && c == '\n') + return SimpleParserValue::NewLine(GetPreviousCharacterPos()); - if(c == EOF) + c = NextChar(); + } + + if (c == EOF) return SimpleParserValue::EndOfFile(TokenPos()); - if (c == '\"') + if (m_read_strings && c == '\"') return SimpleParserValue::String(GetPreviousCharacterPos(), new std::string(ReadString())); const auto pos = GetPreviousCharacterPos(); - if (isdigit(c)) + if (m_read_numbers && isdigit(c)) { bool isFloatingPointValue; double doubleValue; diff --git a/src/Parser/Parsing/Simple/SimpleLexer.h b/src/Parser/Parsing/Simple/SimpleLexer.h index e0dc3c58..1c5d4d61 100644 --- a/src/Parser/Parsing/Simple/SimpleLexer.h +++ b/src/Parser/Parsing/Simple/SimpleLexer.h @@ -3,11 +3,20 @@ #include "SimpleParserValue.h" #include "Parsing/Impl/AbstractLexer.h" -class SimpleLexer final : public AbstractLexer +class SimpleLexer : public AbstractLexer { + bool m_emit_new_line_tokens; + bool m_read_strings; + bool m_read_numbers; + int m_last_line; + protected: SimpleParserValue GetNextToken() override; + void SetShouldEmitNewLineTokens(bool value); + void SetShouldReadStrings(bool value); + void SetShouldReadNumbers(bool value); + public: explicit SimpleLexer(IParserLineStream* stream); }; \ No newline at end of file diff --git a/src/Parser/Parsing/Simple/SimpleParserValue.cpp b/src/Parser/Parsing/Simple/SimpleParserValue.cpp index e06c7951..d79149e9 100644 --- a/src/Parser/Parsing/Simple/SimpleParserValue.cpp +++ b/src/Parser/Parsing/Simple/SimpleParserValue.cpp @@ -14,6 +14,12 @@ SimpleParserValue SimpleParserValue::EndOfFile(const TokenPos pos) return pv; } +SimpleParserValue SimpleParserValue::NewLine(const TokenPos pos) +{ + SimpleParserValue pv(pos, SimpleParserValueType::NEW_LINE); + return pv; +} + SimpleParserValue SimpleParserValue::Character(const TokenPos pos, const char c) { SimpleParserValue pv(pos, SimpleParserValueType::CHARACTER); diff --git a/src/Parser/Parsing/Simple/SimpleParserValue.h b/src/Parser/Parsing/Simple/SimpleParserValue.h index 8a4b3ec6..eca0ae4e 100644 --- a/src/Parser/Parsing/Simple/SimpleParserValue.h +++ b/src/Parser/Parsing/Simple/SimpleParserValue.h @@ -11,6 +11,7 @@ enum class SimpleParserValueType // Meta tokens INVALID, END_OF_FILE, + NEW_LINE, // Single character CHARACTER, @@ -41,6 +42,7 @@ public: static SimpleParserValue Invalid(TokenPos pos); static SimpleParserValue EndOfFile(TokenPos pos); + static SimpleParserValue NewLine(TokenPos pos); static SimpleParserValue Character(TokenPos pos, char c); static SimpleParserValue Integer(TokenPos pos, int value); static SimpleParserValue FloatingPoint(TokenPos pos, double value); diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.cpp new file mode 100644 index 00000000..57632889 --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.cpp @@ -0,0 +1,58 @@ +#include "ZoneDefinitionCommonMatchers.h" + +#include + +#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h" + +std::unique_ptr ZoneDefinitionCommonMatchers::AssetName(const supplier_t* labelSupplier) +{ + const SimpleMatcherFactory create(labelSupplier); + + return create.And({ + create.Loop(create.Or({ + create.Identifier(), + create.AnyCharBesides({',', '<', '>', '"', '\\', '*', '?', '|', ':'}) + })), + create.Type(SimpleParserValueType::NEW_LINE).NoConsume() + }).Transform([](SimpleMatcherFactory::token_list_t& tokens) + { + std::ostringstream str; + auto previousType = SimpleParserValueType::INVALID; + auto previousCharacterWasSlash = false; + + for (const auto& token : tokens) + { + switch (token.get().m_type) + { + case SimpleParserValueType::IDENTIFIER: + if (previousType == SimpleParserValueType::IDENTIFIER) + str << " "; + str << token.get().IdentifierValue(); + break; + + case SimpleParserValueType::CHARACTER: + if (token.get().CharacterValue() == '/') + { + if (previousType == SimpleParserValueType::CHARACTER && !previousCharacterWasSlash) + { + str << "/"; + previousCharacterWasSlash = true; + } + } + else + { + str << token.get().CharacterValue(); + previousCharacterWasSlash = false; + } + break; + + default: + break; + } + + previousType = token.get().m_type; + } + + return SimpleParserValue::Identifier(tokens.front().get().GetPos(), new std::string(str.str())); + }); +} diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h b/src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h new file mode 100644 index 00000000..cc75e639 --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +#include "Parsing/Simple/SimpleParserValue.h" +#include "Parsing/Matcher/AbstractMatcher.h" +#include "Parsing/Matcher/MatcherLabel.h" + +class ZoneDefinitionCommonMatchers +{ +public: + typedef AbstractMatcher matcher_t; + typedef IMatcherForLabelSupplier supplier_t; + + static constexpr int LABEL_ASSET_NAME = std::numeric_limits::max() - 1; + + static std::unique_ptr AssetName(const supplier_t* labelSupplier); +}; diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.cpp new file mode 100644 index 00000000..bbba85e9 --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.cpp @@ -0,0 +1,14 @@ +#include "SequenceZoneDefinitionConsumeEmptyLines.h" + +#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h" + +SequenceZoneDefinitionConsumeEmptyLines::SequenceZoneDefinitionConsumeEmptyLines() +{ + const SimpleMatcherFactory create(this); + + AddMatchers(create.Loop(create.Type(SimpleParserValueType::NEW_LINE))); +} + +void SequenceZoneDefinitionConsumeEmptyLines::ProcessMatch(ZoneDefinition* state, SequenceResult& result) const +{ +} diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.h b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.h new file mode 100644 index 00000000..d2b6f7d7 --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.h @@ -0,0 +1,12 @@ +#pragma once + +#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h" + +class SequenceZoneDefinitionConsumeEmptyLines final : public ZoneDefinitionParser::sequence_t +{ +protected: + void ProcessMatch(ZoneDefinition* state, SequenceResult& result) const override; + +public: + SequenceZoneDefinitionConsumeEmptyLines(); +}; diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionEntry.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionEntry.cpp index 6bc257c9..7c773e9f 100644 --- a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionEntry.cpp +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionEntry.cpp @@ -1,16 +1,19 @@ #include "SequenceZoneDefinitionEntry.h" +#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h" #include "Parsing/Simple/Matcher/SimpleMatcherFactory.h" SequenceZoneDefinitionEntry::SequenceZoneDefinitionEntry() { const SimpleMatcherFactory create(this); + AddLabeledMatchers(ZoneDefinitionCommonMatchers::AssetName(this), ZoneDefinitionCommonMatchers::LABEL_ASSET_NAME); AddMatchers({ create.Identifier().Capture(CAPTURE_TYPE_NAME), create.Char(','), create.Optional(create.Char(',').Tag(TAG_REFERENCE)), - create.Identifier().Capture(CAPTURE_ASSET_NAME) + create.Label(ZoneDefinitionCommonMatchers::LABEL_ASSET_NAME).Capture(CAPTURE_ASSET_NAME), + create.Type(SimpleParserValueType::NEW_LINE) }); } diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionIgnore.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionIgnore.cpp index ad81092d..b1337279 100644 --- a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionIgnore.cpp +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionIgnore.cpp @@ -9,8 +9,9 @@ SequenceZoneDefinitionIgnore::SequenceZoneDefinitionIgnore() AddMatchers({ create.Keyword("ignore"), create.Char(','), - create.Identifier().Capture(CAPTURE_IGNORE_NAME) - }); + create.Identifier().Capture(CAPTURE_IGNORE_NAME), + create.Type(SimpleParserValueType::NEW_LINE) + }); } void SequenceZoneDefinitionIgnore::ProcessMatch(ZoneDefinition* state, SequenceResult& result) const diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionInclude.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionInclude.cpp index 32f77f20..c41f99ec 100644 --- a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionInclude.cpp +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionInclude.cpp @@ -9,7 +9,8 @@ SequenceZoneDefinitionInclude::SequenceZoneDefinitionInclude() AddMatchers({ create.Keyword("include"), create.Char(','), - create.Identifier().Capture(CAPTURE_INCLUDE_NAME) + create.Identifier().Capture(CAPTURE_INCLUDE_NAME), + create.Type(SimpleParserValueType::NEW_LINE) }); } diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionMetaData.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionMetaData.cpp index e4e296bd..5809d378 100644 --- a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionMetaData.cpp +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionMetaData.cpp @@ -10,7 +10,8 @@ SequenceZoneDefinitionMetaData::SequenceZoneDefinitionMetaData() create.Char('>'), create.Identifier().Capture(CAPTURE_KEY), create.Char(','), - create.Identifier().Capture(CAPTURE_VALUE) + create.Identifier().Capture(CAPTURE_VALUE), + create.Type(SimpleParserValueType::NEW_LINE) }); } diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.cpp new file mode 100644 index 00000000..8b4fc28d --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.cpp @@ -0,0 +1,9 @@ +#include "ZoneDefinitionLexer.h" + +ZoneDefinitionLexer::ZoneDefinitionLexer(IParserLineStream* stream) + : SimpleLexer(stream) +{ + SetShouldEmitNewLineTokens(true); + SetShouldReadStrings(true); + SetShouldReadNumbers(false); +} \ No newline at end of file diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.h b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.h new file mode 100644 index 00000000..e14ec839 --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.h @@ -0,0 +1,9 @@ +#pragma once + +#include "Parsing/Simple/SimpleLexer.h" + +class ZoneDefinitionLexer final : public SimpleLexer +{ +public: + explicit ZoneDefinitionLexer(IParserLineStream* stream); +}; \ No newline at end of file diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp index c08e88b8..33916eab 100644 --- a/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp +++ b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp @@ -1,5 +1,7 @@ #include "ZoneDefinitionParser.h" + +#include "Sequence/SequenceZoneDefinitionConsumeEmptyLines.h" #include "Sequence/SequenceZoneDefinitionEntry.h" #include "Sequence/SequenceZoneDefinitionIgnore.h" #include "Sequence/SequenceZoneDefinitionInclude.h" @@ -16,7 +18,8 @@ const std::vector::sequence_t* new SequenceZoneDefinitionMetaData(), new SequenceZoneDefinitionInclude(), new SequenceZoneDefinitionIgnore(), - new SequenceZoneDefinitionEntry() + new SequenceZoneDefinitionEntry(), + new SequenceZoneDefinitionConsumeEmptyLines() }); return tests; diff --git a/src/ZoneCommon/Zone/Definition/ZoneDefinitionStream.cpp b/src/ZoneCommon/Zone/Definition/ZoneDefinitionStream.cpp index dfae4e99..45e902b8 100644 --- a/src/ZoneCommon/Zone/Definition/ZoneDefinitionStream.cpp +++ b/src/ZoneCommon/Zone/Definition/ZoneDefinitionStream.cpp @@ -7,6 +7,7 @@ #include "Parsing/Impl/DefinesStreamProxy.h" #include "Parsing/Simple/SimpleLexer.h" +#include "Parsing/ZoneDefinition/ZoneDefinitionLexer.h" #include "Parsing/ZoneDefinition/ZoneDefinitionParser.h" ZoneDefinitionInputStream::ZoneDefinitionInputStream(std::istream& stream, std::string fileName, bool verbose) @@ -42,7 +43,7 @@ std::unique_ptr ZoneDefinitionInputStream::ReadDefinition() std::cout << "Reading zone definition file: " << m_file_name << std::endl; } - const auto lexer = std::make_unique(m_stream); + const auto lexer = std::make_unique(m_stream); const auto parser = std::make_unique(lexer.get()); const auto start = std::chrono::steady_clock::now();