zone/OpenAssetTools
zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-05-07 13:04:58 +00:00
Code Issues Packages Projects Wiki Activity

refactor: adjust zcg code for working in x64

Browse Source
This commit is contained in:
Jan 2025-04-25 19:21:22 +01:00 committed by Jan Laupetin
parent c61dddf0e2
commit 60f5c1a18f
No known key found for this signature in database
GPG Key ID: 44B581F78FF5C57C
19 changed files with 322 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/Parser/Parsing/ILexer.h
View File

@ -3,11 +3,10 @@
#include "IParserLineStream.h"
#include "Parsing/IParserValue.h"
template<typename TokenType> class ILexer
{
// TokenType must inherit IParserValue
static_assert(std::is_base_of<IParserValue, TokenType>::value);
#include <concepts>
template<std::derived_from<IParserValue> TokenType> class ILexer
{
public:
ILexer() = default;
virtual ~ILexer() = default;
@ -16,10 +15,10 @@ public:
ILexer& operator=(const ILexer& other) = default;
ILexer& operator=(ILexer&& other) noexcept = default;
virtual const TokenType& GetToken(unsigned index) = 0;
virtual void PopTokens(int amount) = 0;
virtual const TokenType& GetToken(size_t index) = 0;
virtual void PopTokens(size_t amount) = 0;
_NODISCARD virtual bool IsEof() = 0;
_NODISCARD virtual const TokenPos& GetPos() = 0;
_NODISCARD virtual ParserLine GetLineForPos(const TokenPos& pos) const = 0;
[[nodiscard]] virtual bool IsEof() = 0;
[[nodiscard]] virtual const TokenPos& GetPos() = 0;
[[nodiscard]] virtual ParserLine GetLineForPos(const TokenPos& pos) const = 0;
};

148
src/Parser/Parsing/Impl/AbstractLexer.h
View File

@ -7,21 +7,82 @@
#include "Utils/StringUtils.h"
#include <cassert>
#include <concepts>
#include <deque>
#include <sstream>
template<typename TokenType> class AbstractLexer : public ILexer<TokenType>
template<std::derived_from<IParserValue> TokenType> class AbstractLexer : public ILexer<TokenType>
{
// TokenType must inherit IParserValue
static_assert(std::is_base_of<IParserValue, TokenType>::value);
public:
const TokenType& GetToken(const size_t index) override
{
while (index >= m_token_cache.size())
m_token_cache.emplace_back(GetNextToken());
return m_token_cache[index];
}
void PopTokens(size_t amount) override
{
if (amount == 0 || m_token_cache.empty())
return;
if (m_token_cache.size() <= amount)
{
const auto& lastToken = m_token_cache.back();
while (
!m_line_cache.empty()
&& (m_line_cache.front().m_line_number != lastToken.GetPos().m_line || *m_line_cache.front().m_filename != lastToken.GetPos().m_filename.get()))
{
m_line_cache.pop_front();
m_line_index--;
}
m_token_cache.clear();
}
else
{
m_token_cache.erase(m_token_cache.begin(), m_token_cache.begin() + amount);
const auto& firstToken = m_token_cache.front();
while (!m_line_cache.empty()
&& (m_line_cache.front().m_line_number != firstToken.GetPos().m_line
|| *m_line_cache.front().m_filename != firstToken.GetPos().m_filename.get()))
{
m_line_cache.pop_front();
m_line_index--;
}
}
}
[[nodiscard]] bool IsEof() override
{
return GetToken(0).IsEof();
}
[[nodiscard]] const TokenPos& GetPos() override
{
return GetToken(0).GetPos();
}
[[nodiscard]] ParserLine GetLineForPos(const TokenPos& pos) const override
{
for (const auto& line : m_line_cache)
{
if (line.m_filename && *line.m_filename == pos.m_filename.get() && line.m_line_number == pos.m_line)
{
return line;
}
}
return ParserLine();
}
protected:
std::deque<TokenType> m_token_cache;
std::deque<ParserLine> m_line_cache;
IParserLineStream* const m_stream;
IParserLineStream* m_stream;
unsigned m_line_index;
unsigned m_current_line_offset;
size_t m_line_index;
size_t m_current_line_offset;
explicit AbstractLexer(IParserLineStream* stream)
: m_stream(stream),
@ -83,28 +144,28 @@ protected:
return m_line_cache[peekLine].m_line[peekLineOffset];
}
_NODISCARD const ParserLine& CurrentLine() const
[[nodiscard]] const ParserLine& CurrentLine() const
{
return m_line_cache[m_line_index];
}
_NODISCARD bool IsLineEnd() const
[[nodiscard]] bool IsLineEnd() const
{
return m_current_line_offset >= CurrentLine().m_line.size();
}
_NODISCARD bool NextCharInLineIs(const char c)
[[nodiscard]] bool NextCharInLineIs(const char c)
{
return !IsLineEnd() && PeekChar() == c;
}
_NODISCARD TokenPos GetPreviousCharacterPos() const
[[nodiscard]] TokenPos GetPreviousCharacterPos() const
{
const auto& currentLine = CurrentLine();
return TokenPos(*currentLine.m_filename, currentLine.m_line_number, m_current_line_offset);
}
_NODISCARD TokenPos GetNextCharacterPos()
[[nodiscard]] TokenPos GetNextCharacterPos()
{
const auto& currentLine = CurrentLine();
if (m_current_line_offset + 1 >= currentLine.m_line.size())
@ -227,7 +288,7 @@ protected:
m_current_line_offset += numberLength - 1;
}
_NODISCARD bool IsIntegerNumber() const
[[nodiscard]] bool IsIntegerNumber() const
{
const auto& currentLine = CurrentLine();
const auto* currentCharacter = &currentLine.m_line.c_str()[m_current_line_offset - 1];
@ -350,67 +411,4 @@ protected:
integerValue = ReadInteger();
}
}
public:
const TokenType& GetToken(unsigned index) override
{
while (index >= m_token_cache.size())
m_token_cache.emplace_back(GetNextToken());
return m_token_cache[index];
}
void PopTokens(int amount) override
{
if (amount <= 0 || m_token_cache.empty())
return;
if (static_cast<int>(m_token_cache.size()) <= amount)
{
const auto& lastToken = m_token_cache.back();
while (
!m_line_cache.empty()
&& (m_line_cache.front().m_line_number != lastToken.GetPos().m_line || *m_line_cache.front().m_filename != lastToken.GetPos().m_filename.get()))
{
m_line_cache.pop_front();
m_line_index--;
}
m_token_cache.clear();
}
else
{
m_token_cache.erase(m_token_cache.begin(), m_token_cache.begin() + amount);
const auto& firstToken = m_token_cache.front();
while (!m_line_cache.empty()
&& (m_line_cache.front().m_line_number != firstToken.GetPos().m_line
|| *m_line_cache.front().m_filename != firstToken.GetPos().m_filename.get()))
{
m_line_cache.pop_front();
m_line_index--;
}
}
}
_NODISCARD bool IsEof() override
{
return GetToken(0).IsEof();
}
_NODISCARD const TokenPos& GetPos() override
{
return GetToken(0).GetPos();
}
_NODISCARD ParserLine GetLineForPos(const TokenPos& pos) const override
{
for (const auto& line : m_line_cache)
{
if (line.m_filename && *line.m_filename == pos.m_filename.get() && line.m_line_number == pos.m_line)
{
return line;
}
}
return ParserLine();
}
};

49
src/Parser/Parsing/Impl/DefinesStreamProxy.cpp
View File

@ -5,7 +5,6 @@
#include "Parsing/Simple/Expression/ISimpleExpression.h"
#include "Parsing/Simple/Expression/SimpleExpressionMatchers.h"
#include "Parsing/Simple/SimpleExpressionInterpreter.h"
#include "Utils/ClassUtils.h"
#include "Utils/StringUtils.h"
#include <regex>
@ -114,7 +113,7 @@ void DefinesStreamProxy::Define::IdentifyParameters(const std::vector<std::strin
{
m_contains_token_pasting_operators = true;
// Skip next char since it's # anyway and we do not want to count it as stringize
// Skip next char since it's # anyway, and we do not want to count it as stringize
i++;
}
}
@ -484,7 +483,7 @@ bool DefinesStreamProxy::MatchEndifDirective(const ParserLine& line, const size_
return true;
}
bool DefinesStreamProxy::MatchDirectives(ParserLine& line)
bool DefinesStreamProxy::MatchDirectives(const ParserLine& line)
{
size_t directiveStartPos;
size_t directiveEndPos;
@ -522,7 +521,7 @@ bool DefinesStreamProxy::FindMacroForIdentifier(const std::string& input,
}
void DefinesStreamProxy::ExtractParametersFromMacroUsage(
const ParserLine& line, unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos)
const ParserLine& line, const unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos)
{
if (input[inputPos] != '(')
return;
@ -588,7 +587,7 @@ void DefinesStreamProxy::ExpandDefinedExpressions(ParserLine& line) const
}
}
bool DefinesStreamProxy::FindNextMacro(const std::string& input, unsigned& inputPos, unsigned& defineStart, const DefinesStreamProxy::Define*& define)
bool DefinesStreamProxy::FindNextMacro(const std::string& input, unsigned& inputPos, unsigned& defineStart, const Define*& define) const
{
const auto inputSize = input.size();
auto wordStart = 0u;
@ -655,7 +654,7 @@ bool DefinesStreamProxy::FindNextMacro(const std::string& input, unsigned& input
namespace
{
enum class TokenPasteTokenType
enum class TokenPasteTokenType : std::uint8_t
{
NONE,
STRING,
@ -673,13 +672,7 @@ namespace
{
}
~TokenPasteToken() = default;
TokenPasteToken(const TokenPasteToken& other) = default;
TokenPasteToken(TokenPasteToken&& other) = default;
TokenPasteToken& operator=(const TokenPasteToken& other) = default;
TokenPasteToken& operator=(TokenPasteToken&& other) noexcept = default;
void SetFromInput(ParserLine& line, unsigned& linePos, const std::string& input, unsigned& offset)
void SetFromInput(const ParserLine& line, const unsigned& linePos, const std::string& input, unsigned& offset)
{
m_start = offset;
@ -698,8 +691,8 @@ namespace
}
if (offset >= inputSize)
throw new ParsingException(TokenPos(*line.m_filename, line.m_line_number, static_cast<int>(linePos + 1)),
"Token-pasting operator cannot be used on unclosed string");
throw ParsingException(TokenPos(*line.m_filename, line.m_line_number, static_cast<int>(linePos + 1)),
"Token-pasting operator cannot be used on unclosed string");
offset++;
}
@ -743,12 +736,16 @@ namespace
unsigned m_end;
};
void EmitPastedTokens(
ParserLine& line, unsigned& linePos, std::ostream& out, const std::string& input, const TokenPasteToken& token0, const TokenPasteToken& token1)
void EmitPastedTokens(const ParserLine& line,
const unsigned& linePos,
std::ostream& out,
const std::string& input,
const TokenPasteToken& token0,
const TokenPasteToken& token1)
{
if ((token0.m_type == TokenPasteTokenType::STRING) != (token1.m_type == TokenPasteTokenType::STRING))
throw new ParsingException(TokenPos(*line.m_filename, line.m_line_number, static_cast<int>(linePos + 1)),
"String token can only use token-pasting operator on other string token");
throw ParsingException(TokenPos(*line.m_filename, line.m_line_number, static_cast<int>(linePos + 1)),
"String token can only use token-pasting operator on other string token");
if (token0.m_type == TokenPasteTokenType::STRING)
{
out << '"';
@ -767,7 +764,7 @@ namespace
} // namespace
void DefinesStreamProxy::ProcessTokenPastingOperators(
ParserLine& line, unsigned& linePos, std::vector<const Define*>& callstack, std::string& input, unsigned& inputPos)
const ParserLine& line, const unsigned& linePos, std::vector<const Define*>& callstack, std::string& input, unsigned& inputPos)
{
std::ostringstream ss;
@ -789,7 +786,7 @@ void DefinesStreamProxy::ProcessTokenPastingOperators(
if (c == '#' && IsTokenPastingOperatorForwardLookup(input, i))
{
if (currentToken.m_type == TokenPasteTokenType::NONE)
throw new ParsingException(CreatePos(line, linePos), "Cannot use token-pasting operator without previous token");
throw ParsingException(CreatePos(line, linePos), "Cannot use token-pasting operator without previous token");
if (previousToken.m_end < currentToken.m_start)
ss << std::string(input, previousToken.m_end, currentToken.m_start - previousToken.m_end);
@ -816,12 +813,12 @@ void DefinesStreamProxy::ProcessTokenPastingOperators(
ss << std::string(input, previousToken.m_end, inputSize - previousToken.m_end);
if (pasteNext)
throw new ParsingException(CreatePos(line, linePos), "Cannot use token-pasting operator without following token");
throw ParsingException(CreatePos(line, linePos), "Cannot use token-pasting operator without following token");
input = ss.str();
}
void DefinesStreamProxy::InsertMacroParameters(std::ostringstream& out, const DefinesStreamProxy::Define* macro, std::vector<std::string>& parameterValues)
void DefinesStreamProxy::InsertMacroParameters(std::ostringstream& out, const Define* macro, const std::vector<std::string>& parameterValues)
{
if (parameterValues.empty() || macro->m_parameter_positions.empty())
{
@ -829,7 +826,7 @@ void DefinesStreamProxy::InsertMacroParameters(std::ostringstream& out, const De
return;
}
auto lastPos = 0u;
auto lastPos = 0uz;
for (const auto& parameterPosition : macro->m_parameter_positions)
{
if (lastPos < parameterPosition.m_parameter_position)
@ -857,7 +854,7 @@ void DefinesStreamProxy::ExpandMacro(ParserLine& line,
std::ostringstream& out,
std::vector<const Define*>& callstack,
const DefinesStreamProxy::Define* macro,
std::vector<std::string>& parameterValues)
const std::vector<std::string>& parameterValues)
{
std::ostringstream rawOutput;
InsertMacroParameters(rawOutput, macro, parameterValues);
@ -873,7 +870,7 @@ void DefinesStreamProxy::ExpandMacro(ParserLine& line,
}
void DefinesStreamProxy::ContinueMacroParameters(
const ParserLine& line, unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos)
const ParserLine& line, const unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos)
{
const auto inputLength = input.size();
while (state.m_parameter_state != ParameterState::NOT_IN_PARAMETERS && inputPos < inputLength)

19
src/Parser/Parsing/Impl/DefinesStreamProxy.h
View File

@ -107,26 +107,29 @@ private:
_NODISCARD bool MatchIfdefDirective(const ParserLine& line, size_t directiveStartPosition, size_t directiveEndPosition);
_NODISCARD bool MatchElseDirective(const ParserLine& line, size_t directiveStartPosition, size_t directiveEndPosition);
_NODISCARD bool MatchEndifDirective(const ParserLine& line, size_t directiveStartPosition, size_t directiveEndPosition);
_NODISCARD bool MatchDirectives(ParserLine& line);
_NODISCARD bool MatchDirectives(const ParserLine& line);
void ExtractParametersFromMacroUsage(const ParserLine& line, unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos);
bool FindMacroForIdentifier(const std::string& input, unsigned wordStart, unsigned wordEnd, const Define*& value) const;
static void ExtractParametersFromMacroUsage(
const ParserLine& line, const unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos);
bool FindMacroForIdentifier(const std::string& input, unsigned identifierStart, unsigned identifierEnd, const Define*& value) const;
static bool MatchDefinedExpression(const ParserLine& line, size_t& pos, std::string& definitionName);
void ExpandDefinedExpressions(ParserLine& line) const;
bool FindNextMacro(const std::string& input, unsigned& inputPos, unsigned& defineStart, const DefinesStreamProxy::Define*& define);
bool FindNextMacro(const std::string& input, unsigned& inputPos, unsigned& defineStart, const DefinesStreamProxy::Define*& define) const;
void ProcessTokenPastingOperators(ParserLine& line, unsigned& linePos, std::vector<const Define*>& callstack, std::string& input, unsigned& inputPos);
void InsertMacroParameters(std::ostringstream& out, const DefinesStreamProxy::Define* macro, std::vector<std::string>& parameterValues);
static void ProcessTokenPastingOperators(
const ParserLine& line, const unsigned& linePos, std::vector<const Define*>& callstack, std::string& input, unsigned& inputPos);
static void InsertMacroParameters(std::ostringstream& out, const DefinesStreamProxy::Define* macro, const std::vector<std::string>& parameterValues);
void ExpandMacro(ParserLine& line,
unsigned& linePos,
std::ostringstream& out,
std::vector<const Define*>& callstack,
const DefinesStreamProxy::Define* macro,
std::vector<std::string>& parameterValues);
const std::vector<std::string>& parameterValues);
void ContinueMacroParameters(const ParserLine& line, unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos);
static void
ContinueMacroParameters(const ParserLine& line, const unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos);
void ContinueMultiLineMacro(ParserLine& line);
void ProcessNestedMacros(ParserLine& line, unsigned& linePos, std::vector<const Define*>& callstack, std::string& input, unsigned& inputPos);

47
src/Parser/Parsing/Matcher/MatcherResult.cpp Normal file
View File

@ -0,0 +1,47 @@
#include "MatcherResult.h"
namespace
{
// The highest bit is the fabricated flag
constexpr size_t FABRICATED_FLAG_MASK = 1uz << (sizeof(size_t) * 8uz - 1uz);
constexpr size_t TOKEN_INDEX_MASK = ~FABRICATED_FLAG_MASK;
} // namespace
MatcherResultTokenIndex::MatcherResultTokenIndex(const size_t index, const bool isFabricated)
{
m_token_index = index & TOKEN_INDEX_MASK;
if (isFabricated)
m_token_index |= FABRICATED_FLAG_MASK;
}
bool MatcherResultTokenIndex::IsFabricated() const
{
return m_token_index & FABRICATED_FLAG_MASK;
}
size_t MatcherResultTokenIndex::GetTokenIndex() const
{
return m_token_index & TOKEN_INDEX_MASK;
}
MatcherResultCapture::MatcherResultCapture(const int captureId, const unsigned tokenIndex)
: MatcherResultCapture(captureId, tokenIndex, false)
{
}
MatcherResultCapture::MatcherResultCapture(const int captureId, const unsigned tokenIndex, const bool isFabricated)
: m_capture_id(captureId),
m_token_index(tokenIndex, isFabricated)
{
}
MatcherResultCapture::MatcherResultCapture(const int captureId, const MatcherResultTokenIndex index)
: m_capture_id(captureId),
m_token_index(index)
{
}
int MatcherResultCapture::GetCaptureId() const
{
return m_capture_id;
}

116
src/Parser/Parsing/Matcher/MatcherResult.h
View File

@ -1,89 +1,40 @@
#pragma once
#include "Parsing/IParserValue.h"
#include "Utils/ClassUtils.h"
#include <algorithm>
#include <concepts>
#include <iterator>
#include <limits>
#include <vector>
template<typename TokenType> class MatcherResult
class MatcherResultTokenIndex
{
// TokenType must inherit IParserValue
static_assert(std::is_base_of<IParserValue, TokenType>::value);
public:
class TokenIndex
{
static constexpr unsigned FABRICATED_FLAG_MASK = std::numeric_limits<unsigned>::max() ^ std::numeric_limits<int>::max();
static constexpr unsigned TOKEN_INDEX_MASK = ~FABRICATED_FLAG_MASK;
unsigned m_token_index;
public:
TokenIndex(const unsigned index, const bool isFabricated)
{
m_token_index = index & TOKEN_INDEX_MASK;
if (isFabricated)
m_token_index |= FABRICATED_FLAG_MASK;
}
_NODISCARD bool IsFabricated() const
{
return m_token_index & FABRICATED_FLAG_MASK;
}
_NODISCARD unsigned GetTokenIndex() const
{
return m_token_index & TOKEN_INDEX_MASK;
}
};
class Capture
{
public:
int m_capture_id;
TokenIndex m_token_index;
Capture(const int captureId, const unsigned tokenIndex)
: Capture(captureId, tokenIndex, false)
{
}
Capture(const int captureId, const unsigned tokenIndex, const bool isFabricated)
: m_capture_id(captureId),
m_token_index(tokenIndex, isFabricated)
{
}
Capture(const int captureId, const TokenIndex index)
: m_capture_id(captureId),
m_token_index(index)
{
}
_NODISCARD int GetCaptureId() const
{
return m_capture_id;
}
};
bool m_matches;
unsigned m_consumed_token_count;
std::vector<int> m_tags;
std::vector<Capture> m_captures;
std::vector<TokenIndex> m_matched_tokens;
std::vector<TokenType> m_fabricated_tokens;
MatcherResultTokenIndex(size_t index, bool isFabricated);
[[nodiscard]] bool IsFabricated() const;
[[nodiscard]] size_t GetTokenIndex() const;
private:
MatcherResult(const bool matches, const unsigned consumedTokenCount)
: m_matches(matches),
m_consumed_token_count(consumedTokenCount)
{
}
size_t m_token_index;
};
class MatcherResultCapture
{
public:
static MatcherResult Match(unsigned consumedTokenCount)
MatcherResultCapture(int captureId, unsigned tokenIndex);
MatcherResultCapture(int captureId, unsigned tokenIndex, bool isFabricated);
MatcherResultCapture(int captureId, MatcherResultTokenIndex index);
[[nodiscard]] int GetCaptureId() const;
int m_capture_id;
MatcherResultTokenIndex m_token_index;
};
template<std::derived_from<IParserValue> TokenType> class MatcherResult
{
public:
static MatcherResult Match(const unsigned consumedTokenCount)
{
return MatcherResult(true, consumedTokenCount);
}
@ -98,12 +49,13 @@ public:
m_consumed_token_count += other.m_consumed_token_count;
if (!other.m_tags.empty())
std::copy(other.m_tags.begin(), other.m_tags.end(), std::back_inserter(m_tags));
std::ranges::copy(other.m_tags, std::back_inserter(m_tags));
for (const auto& capture : other.m_captures)
{
if (capture.m_token_index.IsFabricated())
m_captures.emplace_back(capture.GetCaptureId(), TokenIndex(m_fabricated_tokens.size() + capture.m_token_index.GetTokenIndex(), true));
m_captures.emplace_back(capture.GetCaptureId(),
MatcherResultTokenIndex(m_fabricated_tokens.size() + capture.m_token_index.GetTokenIndex(), true));
else
m_captures.emplace_back(capture.GetCaptureId(), capture.m_token_index);
}
@ -121,4 +73,18 @@ public:
m_fabricated_tokens.emplace_back(std::move(fabricated));
}
}
bool m_matches;
unsigned m_consumed_token_count;
std::vector<int> m_tags;
std::vector<MatcherResultCapture> m_captures;
std::vector<MatcherResultTokenIndex> m_matched_tokens;
std::vector<TokenType> m_fabricated_tokens;
private:
MatcherResult(const bool matches, const unsigned consumedTokenCount)
: m_matches(matches),
m_consumed_token_count(consumedTokenCount)
{
}
};

8
src/Parser/Parsing/Sequence/SequenceResult.h
View File

@ -5,9 +5,10 @@
#include "Parsing/ParsingException.h"
#include "Utils/ClassUtils.h"
#include <concepts>
#include <unordered_map>
template<typename TokenType> class SequenceResult
template<std::derived_from<IParserValue> TokenType> class SequenceResult
{
class Capture
{
@ -21,9 +22,6 @@ template<typename TokenType> class SequenceResult
}
};
// TokenType must inherit IParserValue
static_assert(std::is_base_of<IParserValue, TokenType>::value);
std::vector<int> m_tags;
std::unordered_map<int, Capture> m_captures;
@ -34,7 +32,7 @@ public:
: m_tags(result.m_tags),
m_tag_offset(0)
{
for (const typename MatcherResult<TokenType>::Capture& capture : result.m_captures)
for (const MatcherResultCapture& capture : result.m_captures)
{
if (capture.m_token_index.IsFabricated())
m_captures[capture.GetCaptureId()].m_tokens.push_back(result.m_fabricated_tokens[capture.m_token_index.GetTokenIndex()]);

9
src/Parser/Parsing/Simple/Expression/SimpleExpressionMatchers.cpp
View File

@ -7,6 +7,7 @@
#include <algorithm>
#include <list>
#include <ranges>
static constexpr int TAG_EXPRESSION = SimpleExpressionMatchers::TAG_OFFSET_EXPRESSION + 1;
static constexpr int TAG_OPERAND = SimpleExpressionMatchers::TAG_OFFSET_EXPRESSION + 2;
@ -103,7 +104,7 @@ std::unique_ptr<ISimpleExpression> SimpleExpressionMatchers::ProcessConditionalO
return std::make_unique<SimpleExpressionConditionalOperator>(std::move(condition), std::move(trueExpression), std::move(falseExpression));
}
std::unique_ptr<ISimpleExpression> SimpleExpressionMatchers::ProcessOperand(SequenceResult<SimpleParserValue>& result) const
std::unique_ptr<ISimpleExpression> SimpleExpressionMatchers::ProcessOperand(SequenceResult<SimpleParserValue>& result)
{
const auto& operandToken = result.NextCapture(CAPTURE_OPERAND);
@ -133,7 +134,7 @@ std::unique_ptr<ISimpleExpression> SimpleExpressionMatchers::ProcessExpression(S
return nullptr;
std::vector<std::unique_ptr<ISimpleExpression>> operands;
std::list<std::pair<unsigned, const SimpleExpressionBinaryOperationType*>> operators;
std::list<std::pair<size_t, const SimpleExpressionBinaryOperationType*>> operators;
while (true)
{
@ -200,7 +201,7 @@ std::unique_ptr<ISimpleExpression> SimpleExpressionMatchers::ProcessExpression(S
}
operators.sort(
[](const std::pair<unsigned, const SimpleExpressionBinaryOperationType*>& p1, const std::pair<unsigned, const SimpleExpressionBinaryOperationType*>& p2)
[](const std::pair<size_t, const SimpleExpressionBinaryOperationType*>& p1, const std::pair<size_t, const SimpleExpressionBinaryOperationType*>& p2)
{
if (p1.second->m_precedence != p2.second->m_precedence)
return p1.second->m_precedence > p2.second->m_precedence;
@ -219,7 +220,7 @@ std::unique_ptr<ISimpleExpression> SimpleExpressionMatchers::ProcessExpression(S
operators.pop_back();
for (auto& [opIndex, _] : operators)
for (auto& opIndex : operators | std::views::keys)
{
if (opIndex > operatorIndex)
opIndex--;

2
src/Parser/Parsing/Simple/Expression/SimpleExpressionMatchers.h
View File

@ -59,7 +59,7 @@ private:
std::unique_ptr<ISimpleExpression> ProcessExpressionInParenthesis(SequenceResult<SimpleParserValue>& result) const;
std::unique_ptr<ISimpleExpression> ProcessConditionalOperation(std::unique_ptr<ISimpleExpression> condition,
SequenceResult<SimpleParserValue>& result) const;
std::unique_ptr<ISimpleExpression> ProcessOperand(SequenceResult<SimpleParserValue>& result) const;
static std::unique_ptr<ISimpleExpression> ProcessOperand(SequenceResult<SimpleParserValue>& result);
public:
std::unique_ptr<matcher_t> Expression(const supplier_t* labelSupplier) const;

7
src/Parser/Parsing/Simple/SimpleLexer.cpp
View File

@ -14,7 +14,12 @@ SimpleLexer::MultiCharacterTokenLookupEntry::MultiCharacterTokenLookupEntry(cons
SimpleLexer::SimpleLexer(IParserLineStream* stream)
: AbstractLexer(stream),
m_config{false, true, false, true, true, {}},
m_config{.m_emit_new_line_tokens = false,
.m_read_strings = true,
.m_string_escape_sequences = false,
.m_read_integer_numbers = true,
.m_read_floating_point_numbers = true,
.m_multi_character_tokens = {}},
m_check_for_multi_character_tokens(false),
m_last_line(1)
{

15
src/Parser/Parsing/Simple/SimpleLexer.h
View File

@ -7,7 +7,7 @@
#include <limits>
#include <memory>
class SimpleLexer : public AbstractLexer<SimpleParserValue>
class SimpleLexer final : public AbstractLexer<SimpleParserValue>
{
public:
class Config
@ -30,6 +30,9 @@ public:
std::vector<MultiCharacterToken> m_multi_character_tokens;
};
explicit SimpleLexer(IParserLineStream* stream);
SimpleLexer(IParserLineStream* stream, Config config);
protected:
class MultiCharacterTokenLookupEntry
{
@ -41,10 +44,6 @@ protected:
MultiCharacterTokenLookupEntry(int id, std::string value);
};
Config m_config;
bool m_check_for_multi_character_tokens;
int m_last_line;
std::unique_ptr<MultiCharacterTokenLookupEntry> m_multi_character_token_lookup[std::numeric_limits<uint8_t>::max() + 1];
void AddMultiCharacterTokenConfigToLookup(Config::MultiCharacterToken tokenConfig);
@ -52,7 +51,7 @@ protected:
SimpleParserValue GetNextToken() override;
public:
explicit SimpleLexer(IParserLineStream* stream);
SimpleLexer(IParserLineStream* stream, Config config);
Config m_config;
bool m_check_for_multi_character_tokens;
size_t m_last_line;
};

11
src/Parser/Parsing/TokenPos.cpp
View File

@ -1,15 +1,18 @@
#include "TokenPos.h"
const std::string TokenPos::EMPTY_FILENAME;
namespace
{
const std::string EMPTY_FILENAME;
}
TokenPos::TokenPos()
: m_filename(EMPTY_FILENAME),
m_line(1),
m_column(1)
m_line(1uz),
m_column(1uz)
{
}
TokenPos::TokenPos(const std::string& filename, const int line, const int column)
TokenPos::TokenPos(const std::string& filename, const size_t line, const size_t column)
: m_filename(filename),
m_line(line),
m_column(column)

12
src/Parser/Parsing/TokenPos.h
View File

@ -5,13 +5,11 @@
class TokenPos
{
static const std::string EMPTY_FILENAME;
public:
std::reference_wrapper<const std::string> m_filename;
int m_line;
int m_column;
TokenPos();
TokenPos(const std::string& filename, int line, int column);
TokenPos(const std::string& filename, size_t line, size_t column);
std::reference_wrapper<const std::string> m_filename;
size_t m_line;
size_t m_column;
};

10
src/Utils/Utils/Arguments/ArgumentParser.cpp
View File

@ -42,7 +42,7 @@ bool ArgumentParser::ParseArguments(std::vector<std::string>& args)
m_path = args[0];
const auto argCount = args.size();
for (unsigned argIndex = 1u; argIndex < argCount; argIndex++)
for (auto argIndex = 1uz; argIndex < argCount; argIndex++)
{
auto& arg = args[argIndex];
@ -85,7 +85,7 @@ bool ArgumentParser::ParseArguments(std::vector<std::string>& args)
return false;
}
if (m_matched_options.find(matchedOption) != m_matched_options.end())
if (m_matched_options.contains(matchedOption))
{
if (!matchedOption->m_multi_use)
{
@ -106,7 +106,7 @@ bool ArgumentParser::ParseArguments(std::vector<std::string>& args)
}
auto& parameters = m_matched_options[matchedOption];
for (unsigned parameterIndex = 0; parameterIndex < parameterCount; parameterIndex++)
for (auto parameterIndex = 0uz; parameterIndex < parameterCount; parameterIndex++)
{
std::string& param = args[argIndex + parameterIndex + 1];
@ -116,14 +116,14 @@ bool ArgumentParser::ParseArguments(std::vector<std::string>& args)
return false;
}
parameters.push_back(param);
parameters.emplace_back(param);
}
argIndex += parameterCount;
}
else
{
m_matched_arguments.push_back(arg);
m_matched_arguments.emplace_back(arg);
}
}

8
src/ZoneCodeGeneratorLib/Domain/Computations/MemberDeclarationModifierComputations.cpp
View File

@ -12,18 +12,18 @@ DeclarationModifierComputations::DeclarationModifierComputations(const MemberInf
m_modifier_indices(std::move(modifierIndices))
{
auto combinedIndex = 0;
auto arraySizes = MemberComputations(m_information).GetArraySizes();
const auto arraySizes = MemberComputations(m_information).GetArraySizes();
std::vector<int> sizePerDepth(arraySizes.size());
auto currentDepthSize = 1;
for (int i = arraySizes.size(); i > 0; i--)
for (int i = static_cast<int>(arraySizes.size()); i > 0; i--)
{
sizePerDepth[i - 1] = currentDepthSize;
currentDepthSize *= arraySizes[i - 1];
}
auto currentDepth = 0;
for (auto modifierIndex : m_modifier_indices)
for (const auto modifierIndex : m_modifier_indices)
{
combinedIndex += sizePerDepth[currentDepth++] * modifierIndex;
}
@ -62,7 +62,7 @@ std::vector<DeclarationModifier*> DeclarationModifierComputations::GetFollowingD
if (m_modifier_indices.size() + 1 < declarationModifiers.size())
{
for (auto i = declarationModifiers.begin() + m_modifier_indices.size() + 1; i != declarationModifiers.end(); ++i)
for (auto i = declarationModifiers.begin() + m_modifier_indices.size() + 1uz; i != declarationModifiers.end(); ++i)
{
following.push_back(i->get());
}

119
src/ZoneCodeGeneratorLib/Parsing/Commands/Impl/CommandsParserState.cpp
View File

@ -1,5 +1,63 @@
#include "CommandsParserState.h"
namespace
{
MemberInformation* GetMemberWithName(const std::string& memberName, const StructureInformation* type)
{
for (const auto& member : type->m_ordered_members)
{
if (member->m_member->m_name == memberName)
{
return member.get();
}
}
return nullptr;
}
bool GetNextTypenameSeparatorPos(const std::string& typeNameValue, const size_t startPos, size_t& separatorPos)
{
const auto typeNameValueSize = typeNameValue.size();
for (auto currentHead = startPos + 1; currentHead < typeNameValueSize; currentHead++)
{
if (typeNameValue[currentHead] == ':' && typeNameValue[currentHead - 1] == ':')
{
separatorPos = currentHead - 1;
return true;
}
}
return false;
}
bool ExtractMembersFromTypenameInternal(const std::string& typeNameValue,
size_t typeNameOffset,
const StructureInformation* type,
std::vector<MemberInformation*>& members)
{
auto startOffset = typeNameOffset;
while (GetNextTypenameSeparatorPos(typeNameValue, typeNameOffset, typeNameOffset))
{
auto* foundMember = GetMemberWithName(std::string(typeNameValue, startOffset, typeNameOffset - startOffset), type);
if (foundMember == nullptr)
return false;
members.push_back(foundMember);
type = foundMember->m_type;
typeNameOffset += 2;
startOffset = typeNameOffset;
}
auto* foundMember = GetMemberWithName(std::string(typeNameValue, startOffset, typeNameValue.size() - startOffset), type);
if (foundMember == nullptr)
return false;
members.push_back(foundMember);
return true;
}
} // namespace
CommandsParserState::CommandsParserState(IDataRepository* repository)
: m_repository(repository),
m_in_use(nullptr)
@ -36,63 +94,8 @@ void CommandsParserState::SetInUse(StructureInformation* structure)
m_in_use = structure;
}
MemberInformation* CommandsParserState::GetMemberWithName(const std::string& memberName, StructureInformation* type)
{
for (const auto& member : type->m_ordered_members)
{
if (member->m_member->m_name == memberName)
{
return member.get();
}
}
return nullptr;
}
bool CommandsParserState::GetNextTypenameSeparatorPos(const std::string& typeNameValue, const unsigned startPos, unsigned& separatorPos)
{
const auto typeNameValueSize = typeNameValue.size();
for (auto currentHead = startPos + 1; currentHead < typeNameValueSize; currentHead++)
{
if (typeNameValue[currentHead] == ':' && typeNameValue[currentHead - 1] == ':')
{
separatorPos = currentHead - 1;
return true;
}
}
return false;
}
bool CommandsParserState::ExtractMembersFromTypenameInternal(const std::string& typeNameValue,
unsigned typeNameOffset,
StructureInformation* type,
std::vector<MemberInformation*>& members)
{
auto startOffset = typeNameOffset;
while (GetNextTypenameSeparatorPos(typeNameValue, typeNameOffset, typeNameOffset))
{
auto* foundMember = GetMemberWithName(std::string(typeNameValue, startOffset, typeNameOffset - startOffset), type);
if (foundMember == nullptr)
return false;
members.push_back(foundMember);
type = foundMember->m_type;
typeNameOffset += 2;
startOffset = typeNameOffset;
}
auto* foundMember = GetMemberWithName(std::string(typeNameValue, startOffset, typeNameValue.size() - startOffset), type);
if (foundMember == nullptr)
return false;
members.push_back(foundMember);
return true;
}
bool CommandsParserState::GetMembersFromTypename(const std::string& typeNameValue,
StructureInformation* baseType,
const StructureInformation* baseType,
std::vector<MemberInformation*>& members) const
{
return m_in_use != nullptr && ExtractMembersFromTypenameInternal(typeNameValue, 0, m_in_use, members)
@ -114,7 +117,7 @@ bool CommandsParserState::GetTypenameAndMembersFromTypename(const std::string& t
}
DataDefinition* foundDefinition = nullptr;
unsigned currentSeparatorPos = 0;
auto currentSeparatorPos = 0uz;
while (GetNextTypenameSeparatorPos(typeNameValue, currentSeparatorPos, currentSeparatorPos))
{
std::string currentTypename(typeNameValue, 0, currentSeparatorPos);
@ -134,7 +137,7 @@ bool CommandsParserState::GetTypenameAndMembersFromTypename(const std::string& t
if (foundDefinition == nullptr)
return false;
auto* definitionWithMembers = dynamic_cast<DefinitionWithMembers*>(foundDefinition);
const auto* definitionWithMembers = dynamic_cast<DefinitionWithMembers*>(foundDefinition);
if (definitionWithMembers == nullptr)
return false;

9
src/ZoneCodeGeneratorLib/Parsing/Commands/Impl/CommandsParserState.h
View File

@ -19,17 +19,10 @@ public:
[[nodiscard]] StructureInformation* GetInUse() const;
void SetInUse(StructureInformation* structure);
bool GetMembersFromTypename(const std::string& typeNameValue, StructureInformation* baseType, std::vector<MemberInformation*>& members) const;
bool GetMembersFromTypename(const std::string& typeNameValue, const StructureInformation* baseType, std::vector<MemberInformation*>& members) const;
bool GetTypenameAndMembersFromTypename(const std::string& typeNameValue, StructureInformation*& structure, std::vector<MemberInformation*>& members) const;
private:
static MemberInformation* GetMemberWithName(const std::string& memberName, StructureInformation* type);
static bool GetNextTypenameSeparatorPos(const std::string& typeNameValue, unsigned startPos, unsigned& separatorPos);
static bool ExtractMembersFromTypenameInternal(const std::string& typeNameValue,
unsigned typeNameOffset,
StructureInformation* type,
std::vector<MemberInformation*>& members);
IDataRepository* m_repository;
StructureInformation* m_in_use;
};

7
src/ZoneCodeGeneratorLib/Parsing/Commands/Matcher/CommandsCommonMatchers.cpp
View File

@ -6,6 +6,7 @@
#include "Domain/Evaluation/Operation.h"
#include <list>
#include <ranges>
#include <sstream>
#include <type_traits>
#include <vector>
@ -353,7 +354,7 @@ std::unique_ptr<IEvaluation>
currentType = nullptr;
std::vector<std::unique_ptr<IEvaluation>> operands;
std::list<std::pair<unsigned, const OperationType*>> operators;
std::list<std::pair<size_t, const OperationType*>> operators;
while (true)
{
@ -384,7 +385,7 @@ std::unique_ptr<IEvaluation>
}
operators.sort(
[](const std::pair<unsigned, const OperationType*>& p1, const std::pair<unsigned, const OperationType*>& p2)
[](const std::pair<size_t, const OperationType*>& p1, const std::pair<size_t, const OperationType*>& p2)
{
if (p1.second->m_precedence != p2.second->m_precedence)
return p1.second->m_precedence > p2.second->m_precedence;
@ -402,7 +403,7 @@ std::unique_ptr<IEvaluation>
operators.pop_back();
for (auto& [opIndex, _] : operators)
for (auto& opIndex : operators | std::views::keys)
{
if (opIndex > operatorIndex)
opIndex--;

21
test/ParserTestUtils/Parsing/Mock/MockLexer.h
View File

@ -3,18 +3,12 @@
#include "Parsing/ILexer.h"
#include "Utils/ClassUtils.h"
#include <concepts>
#include <iterator>
#include <vector>
template<typename TokenType> class MockLexer final : public ILexer<TokenType>
template<std::derived_from<IParserValue> TokenType> class MockLexer final : public ILexer<TokenType>
{
// TokenType must inherit IParserValue
static_assert(std::is_base_of<IParserValue, TokenType>::value);
std::vector<TokenType> m_tokens;
TokenType m_eof;
size_t m_pop_count;
public:
MockLexer(std::initializer_list<Movable<TokenType>> tokens, TokenType eof)
: m_tokens(std::make_move_iterator(tokens.begin()), std::make_move_iterator(tokens.end())),
@ -36,7 +30,7 @@ public:
MockLexer& operator=(const MockLexer& other) = delete;
MockLexer& operator=(MockLexer&& other) noexcept = default;
const TokenType& GetToken(const unsigned index) override
const TokenType& GetToken(const size_t index) override
{
const auto absoluteIndex = m_pop_count + index;
if (absoluteIndex < m_tokens.size())
@ -45,10 +39,10 @@ public:
return m_eof;
}
void PopTokens(const int amount) override
void PopTokens(const size_t amount) override
{
assert(amount >= 0);
m_pop_count += static_cast<size_t>(amount);
m_pop_count += amount;
}
bool IsEof() override
@ -70,4 +64,9 @@ public:
{
return ParserLine();
}
private:
std::vector<TokenType> m_tokens;
TokenType m_eof;
size_t m_pop_count;
};