Add file inclusion for ZCG cpp

This commit is contained in:
Jan 2021-02-10 14:21:58 +01:00
parent 21440daf27
commit d876bc5e25
25 changed files with 779 additions and 2 deletions

View File

@ -0,0 +1,37 @@
#pragma once
#include <cassert>
#include <deque>
#include "ILexer.h"
#include "IParserLineStream.h"
template <typename TokenType>
class AbstractLexer : public ILexer
{
std::deque<TokenType> m_token_cache;
IParserLineStream* const m_stream;
protected:
explicit AbstractLexer(IParserLineStream* stream)
: m_stream(stream)
{
}
virtual TokenType GetNextToken() = 0;
public:
const TokenType& GetToken(int index)
{
assert(index >= 0);
while (index <= m_token_cache.size())
m_token_cache.emplace_back(std::move(GetNextToken()));
return m_token_cache[index];
}
void PopTokens(int amount) override
{
m_token_cache.erase(m_token_cache.begin(), m_token_cache.begin() + amount);
}
};

View File

@ -0,0 +1,11 @@
#include "CommandsLexer.h"
CommandsLexer::CommandsLexer(IParserLineStream* stream)
: AbstractLexer(stream)
{
}
CommandsParserValue CommandsLexer::GetNextToken()
{
return CommandsParserValue::Invalid(TokenPos());
}

View File

@ -0,0 +1,13 @@
#pragma once
#include "CommandsParserValue.h"
#include "Parsing/AbstractLexer.h"
class CommandsLexer final : public AbstractLexer<CommandsParserValue>
{
protected:
CommandsParserValue GetNextToken() override;
public:
explicit CommandsLexer(IParserLineStream* stream);
};

View File

@ -0,0 +1,6 @@
#pragma once
class CommandsParser
{
};

View File

@ -0,0 +1,174 @@
#include "CommandsParserValue.h"
#include <cassert>
CommandsParserValue CommandsParserValue::Invalid(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::INVALID);
return pv;
}
CommandsParserValue CommandsParserValue::EndOfFile(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::END_OF_FILE);
return pv;
}
CommandsParserValue CommandsParserValue::Character(const TokenPos pos, const char c)
{
CommandsParserValue pv(pos, c);
return pv;
}
CommandsParserValue CommandsParserValue::ShiftLeft(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::SHIFT_LEFT);
return pv;
}
CommandsParserValue CommandsParserValue::ShiftRight(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::SHIFT_RIGHT);
return pv;
}
CommandsParserValue CommandsParserValue::Equals(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::EQUALS);
return pv;
}
CommandsParserValue CommandsParserValue::NotEqual(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::NOT_EQUAL);
return pv;
}
CommandsParserValue CommandsParserValue::GreaterEqual(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::GREATER_EQUAL);
return pv;
}
CommandsParserValue CommandsParserValue::LessEqual(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::LESS_EQUAL);
return pv;
}
CommandsParserValue CommandsParserValue::LogicalAnd(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::LOGICAL_AND);
return pv;
}
CommandsParserValue CommandsParserValue::LogicalOr(const TokenPos pos)
{
CommandsParserValue pv(pos, CommandsParserValueType::LOGICAL_OR);
return pv;
}
CommandsParserValue CommandsParserValue::Integer(const TokenPos pos, const int value)
{
CommandsParserValue pv(pos, CommandsParserValueType::INTEGER);
pv.m_value.int_value = value;
return pv;
}
CommandsParserValue CommandsParserValue::FloatingPoint(const TokenPos pos, const double value)
{
CommandsParserValue pv(pos, CommandsParserValueType::FLOATING_POINT);
pv.m_value.double_value = value;
return pv;
}
CommandsParserValue CommandsParserValue::String(const TokenPos pos, std::string* stringValue)
{
CommandsParserValue pv(pos, CommandsParserValueType::STRING);
pv.m_value.string_value = stringValue;
return pv;
}
CommandsParserValue CommandsParserValue::Identifier(const TokenPos pos, std::string* identifier)
{
CommandsParserValue pv(pos, CommandsParserValueType::IDENTIFIER);
pv.m_value.string_value = identifier;
return pv;
}
CommandsParserValue CommandsParserValue::TypeName(const TokenPos pos, std::string* typeName)
{
CommandsParserValue pv(pos, CommandsParserValueType::TYPE_NAME);
pv.m_value.string_value = typeName;
return pv;
}
CommandsParserValue::CommandsParserValue(const TokenPos pos, const int type)
: m_pos(pos),
m_type(type),
m_value()
{
}
CommandsParserValue::~CommandsParserValue()
{
switch (m_type)
{
case CommandsParserValueType::STRING:
case CommandsParserValueType::IDENTIFIER:
case CommandsParserValueType::TYPE_NAME:
delete m_value.string_value;
break;
default:
break;
}
m_value = ValueType();
}
CommandsParserValue::CommandsParserValue(CommandsParserValue&& other) noexcept
: m_type(other.m_type),
m_value(other.m_value)
{
other.m_value = ValueType();
}
CommandsParserValue& CommandsParserValue::operator=(CommandsParserValue&& other) noexcept
{
m_type = other.m_type;
m_value = other.m_value;
other.m_value = ValueType();
return *this;
}
int CommandsParserValue::IntegerValue() const
{
assert(m_type == CommandsParserValueType::INTEGER);
return m_value.int_value;
}
double CommandsParserValue::FloatingPointValue() const
{
assert(m_type == CommandsParserValueType::FLOATING_POINT);
return m_value.double_value;
}
std::string& CommandsParserValue::StringValue() const
{
assert(m_type == CommandsParserValueType::STRING);
return *m_value.string_value;
}
std::string& CommandsParserValue::IdentifierValue() const
{
assert(m_type == CommandsParserValueType::IDENTIFIER);
return *m_value.string_value;
}
std::string& CommandsParserValue::TypeNameValue() const
{
assert(m_type == CommandsParserValueType::TYPE_NAME);
return *m_value.string_value;
}

View File

@ -0,0 +1,88 @@
#pragma once
#include <string>
#include "Parsing/TokenPos.h"
#include "Utils/ClassUtils.h"
class CommandsParserValueType
{
CommandsParserValueType() = default;
public:
enum
{
FIRST = 0x100,
// Meta tokens
INVALID = FIRST,
END_OF_FILE,
// Symbol tokens
SHIFT_LEFT,
SHIFT_RIGHT,
EQUALS,
NOT_EQUAL,
GREATER_EQUAL,
LESS_EQUAL,
LOGICAL_AND,
LOGICAL_OR,
// Generic token types
INTEGER,
FLOATING_POINT,
STRING,
IDENTIFIER,
// Parser created
TYPE_NAME,
// End
MAX
};
};
class CommandsParserValue
{
public:
TokenPos m_pos;
int m_type;
union ValueType
{
int int_value;
double double_value;
std::string* string_value;
} m_value;
static CommandsParserValue Invalid(TokenPos pos);
static CommandsParserValue EndOfFile(TokenPos pos);
static CommandsParserValue Character(TokenPos pos, char c);
static CommandsParserValue ShiftLeft(TokenPos pos);
static CommandsParserValue ShiftRight(TokenPos pos);
static CommandsParserValue Equals(TokenPos pos);
static CommandsParserValue NotEqual(TokenPos pos);
static CommandsParserValue GreaterEqual(TokenPos pos);
static CommandsParserValue LessEqual(TokenPos pos);
static CommandsParserValue LogicalAnd(TokenPos pos);
static CommandsParserValue LogicalOr(TokenPos pos);
static CommandsParserValue Integer(TokenPos pos, int value);
static CommandsParserValue FloatingPoint(TokenPos pos, double value);
static CommandsParserValue String(TokenPos pos, std::string* stringValue);
static CommandsParserValue Identifier(TokenPos pos, std::string* identifier);
static CommandsParserValue TypeName(TokenPos pos, std::string* typeName);
private:
CommandsParserValue(TokenPos pos, int type);
public:
~CommandsParserValue();
CommandsParserValue(const CommandsParserValue& other) = delete;
CommandsParserValue(CommandsParserValue&& other) noexcept;
CommandsParserValue& operator=(const CommandsParserValue& other) = delete;
CommandsParserValue& operator=(CommandsParserValue&& other) noexcept;
_NODISCARD int IntegerValue() const;
_NODISCARD double FloatingPointValue() const;
_NODISCARD std::string& StringValue() const;
_NODISCARD std::string& IdentifierValue() const;
_NODISCARD std::string& TypeNameValue() const;
};

View File

@ -2,14 +2,39 @@
#include <iostream>
#include "Parsing/Impl/IncludingStreamProxy.h"
#include "Parsing/Impl/ParserFilesystemStream.h"
HeaderFileReader::HeaderFileReader(const ZoneCodeGeneratorArguments* args, std::string filename)
: m_args(args),
m_filename(std::move(filename))
{
}
bool HeaderFileReader::ReadHeaderFile(IDataRepository* repository)
bool HeaderFileReader::ReadHeaderFile(IDataRepository* repository) const
{
std::cout << "Reading header file: " << m_filename << std::endl;
ParserFilesystemStream stream(m_filename);
if(!stream.IsOpen())
{
std::cout << "Could not open header file" << std::endl;
return false;
}
IncludingStreamProxy includeProxy(&stream);
IParserLineStream* lineStream = &includeProxy;
while(true)
{
auto line = lineStream->NextLine();
if (line.IsEof())
break;
std::cout << "Line " << line.m_filename << ":" << line.m_line_number << ": " << line.m_line << std::endl;
}
return true;
}

View File

@ -13,5 +13,5 @@ class HeaderFileReader
public:
HeaderFileReader(const ZoneCodeGeneratorArguments* args, std::string filename);
bool ReadHeaderFile(IDataRepository* repository);
bool ReadHeaderFile(IDataRepository* repository) const;
};

View File

@ -0,0 +1,15 @@
#pragma once
class ILexer
{
public:
ILexer() = default;
virtual ~ILexer() = default;
ILexer(const ILexer& other) = default;
ILexer(ILexer&& other) noexcept = default;
ILexer& operator=(const ILexer& other) = default;
ILexer& operator=(ILexer&& other) noexcept = default;
virtual void PopTokens(int amount) = 0;
};

View File

@ -0,0 +1,13 @@
#include "IParserLineStream.h"
ParserLine::ParserLine(const std::string& filename, const int lineNumber, std::string line)
: m_filename(filename),
m_line_number(lineNumber),
m_line(std::move(line))
{
}
bool ParserLine::IsEof() const
{
return m_line_number <= 0;
}

View File

@ -0,0 +1,34 @@
#pragma once
#include <string>
#include "Utils/ClassUtils.h"
class ParserLine
{
public:
const std::string& m_filename;
const int m_line_number;
std::string m_line;
ParserLine(const std::string& filename, int lineNumber, std::string line);
_NODISCARD bool IsEof() const;
};
class IParserLineStream
{
public:
IParserLineStream() = default;
virtual ~IParserLineStream() = default;
IParserLineStream(const IParserLineStream& other) = default;
IParserLineStream(IParserLineStream&& other) noexcept = default;
IParserLineStream& operator=(const IParserLineStream& other) = default;
IParserLineStream& operator=(IParserLineStream&& other) noexcept = default;
virtual ParserLine NextLine() = 0;
virtual bool IncludeFile(const std::string& filename) = 0;
_NODISCARD virtual bool IsOpen() const = 0;
_NODISCARD virtual bool Eof() const = 0;
};

View File

@ -0,0 +1,12 @@
#pragma once
#include "Parsing/IParserLineStream.h"
class CommentRemovingStreamProxy final : IParserLineStream
{
public:
std::string NextLine() override;
bool IncludeFile(const std::string& filename) override;
bool IsOpen() const override;
_NODISCARD bool Eof() const override;
};

View File

@ -0,0 +1,116 @@
#include "IncludingStreamProxy.h"
#include <sstream>
#include "Parsing/ParsingException.h"
IncludingStreamProxy::IncludingStreamProxy(IParserLineStream* stream)
: m_stream(stream)
{
}
bool IncludingStreamProxy::MatchIncludeDirective(const ParserLine& line) const
{
unsigned includeDirectivePos = 0;
auto hasIncludeDirective = false;
for (; includeDirectivePos < line.m_line.size() - INCLUDE_DIRECTIVE_MINIMUM_TOTAL_LENGTH; includeDirectivePos++)
{
const auto c = line.m_line[includeDirectivePos];
if (isspace(c))
continue;
if (c != '#')
return false;
if (line.m_line.compare(includeDirectivePos + 1, INCLUDE_DIRECTIVE_LENGTH, INCLUDE_DIRECTIVE) != 0)
{
return false;
}
hasIncludeDirective = true;
break;
}
if (!hasIncludeDirective)
return false;
auto currentPos = includeDirectivePos + INCLUDE_DIRECTIVE_LENGTH + 1;
bool isDoubleQuotes;
if (line.m_line[currentPos] == '"')
isDoubleQuotes = true;
else if (line.m_line[currentPos] == '<')
isDoubleQuotes = false;
else
throw ParsingException(TokenPos(line.m_filename, line.m_line_number, currentPos), INCLUDE_QUOTES_ERROR);
const auto filenameStart = ++currentPos;
unsigned filenameEnd = 0;
auto filenameHasEnd = false;
for (; currentPos < line.m_line.size(); currentPos++)
{
const auto c = line.m_line[currentPos];
if (c == '"')
{
if(!isDoubleQuotes)
throw ParsingException(TokenPos(line.m_filename, line.m_line_number, currentPos), "");
filenameEnd = currentPos;
filenameHasEnd = true;
break;
}
if (c == '>')
{
if (isDoubleQuotes)
throw ParsingException(TokenPos(line.m_filename, line.m_line_number, currentPos), INCLUDE_QUOTES_ERROR);
filenameEnd = currentPos;
filenameHasEnd = true;
break;
}
}
if(!filenameHasEnd)
throw ParsingException(TokenPos(line.m_filename, line.m_line_number, currentPos), INCLUDE_QUOTES_ERROR);
if(filenameEnd <= filenameStart)
throw ParsingException(TokenPos(line.m_filename, line.m_line_number, currentPos), "No filename specified");
const auto filename = line.m_line.substr(filenameStart, filenameEnd - filenameStart);
if(!m_stream->IncludeFile(filename))
{
std::ostringstream errorStr;
errorStr << "Could not include file \"" << filename << "\"";
throw ParsingException(TokenPos(line.m_filename, line.m_line_number, currentPos), errorStr.str());
}
return true;
}
ParserLine IncludingStreamProxy::NextLine()
{
auto line = m_stream->NextLine();
if (MatchIncludeDirective(line))
return m_stream->NextLine();
return line;
}
bool IncludingStreamProxy::IncludeFile(const std::string& filename)
{
return m_stream->IncludeFile(filename);
}
bool IncludingStreamProxy::IsOpen() const
{
return m_stream->IsOpen();
}
bool IncludingStreamProxy::Eof() const
{
return m_stream->Eof();
}

View File

@ -0,0 +1,23 @@
#pragma once
#include "Parsing/IParserLineStream.h"
class IncludingStreamProxy final : public IParserLineStream
{
static constexpr const char* INCLUDE_QUOTES_ERROR = "Invalid include directive. Expected \"\" or <>";
static constexpr const char* INCLUDE_DIRECTIVE = "include ";
static constexpr int INCLUDE_DIRECTIVE_LENGTH = std::char_traits<char>::length(INCLUDE_DIRECTIVE);
static constexpr int INCLUDE_DIRECTIVE_MINIMUM_TOTAL_LENGTH = INCLUDE_DIRECTIVE_LENGTH + 1 + 2; // #=+1 ""=+2
IParserLineStream* m_stream;
bool MatchIncludeDirective(const ParserLine& line) const;
public:
explicit IncludingStreamProxy(IParserLineStream* stream);
ParserLine NextLine() override;
bool IncludeFile(const std::string& filename) override;
_NODISCARD bool IsOpen() const override;
_NODISCARD bool Eof() const override;
};

View File

@ -0,0 +1,93 @@
#include "ParserFilesystemStream.h"
#include <sstream>
#include <filesystem>
namespace fs = std::filesystem;
const std::string ParserFilesystemStream::EMPTY_FILE_NAME;
ParserFilesystemStream::FileInfo::FileInfo(std::string filePath)
: m_file_path(std::move(filePath)),
m_stream(m_file_path),
m_line_number(1)
{
}
ParserFilesystemStream::ParserFilesystemStream(std::string path)
{
m_files.emplace(FileInfo(std::move(path)));
}
bool ParserFilesystemStream::IsOpen() const
{
return !m_files.empty()
&& m_files.top().m_stream.is_open();
}
ParserLine ParserFilesystemStream::NextLine()
{
std::ostringstream str;
auto hasLength = false;
if (m_files.empty())
return ParserLine(EMPTY_FILE_NAME, 0, std::string());
while(!m_files.empty())
{
auto& fileInfo = m_files.top();
auto c = fileInfo.m_stream.get();
while (c != EOF)
{
switch (c)
{
case '\r':
c = fileInfo.m_stream.get();
if (c == '\n')
return ParserLine(fileInfo.m_file_path, fileInfo.m_line_number++, str.str());
str << '\r';
hasLength = true;
continue;
case '\n':
return ParserLine(fileInfo.m_file_path, fileInfo.m_line_number++, str.str());
default:
str << static_cast<char>(c);
hasLength = true;
break;
}
c = fileInfo.m_stream.get();
}
if(hasLength)
return ParserLine(fileInfo.m_file_path, fileInfo.m_line_number, str.str());
m_files.pop();
}
return ParserLine(EMPTY_FILE_NAME, 0, std::string());
}
bool ParserFilesystemStream::IncludeFile(const std::string& filename)
{
if (m_files.empty())
return false;
const auto newFilePath = fs::path(m_files.top().m_file_path).remove_filename().concat(filename);
FileInfo fileInfo(newFilePath.string());
if (!fileInfo.m_stream.is_open())
return false;
m_files.emplace(std::move(fileInfo));
return true;
}
bool ParserFilesystemStream::Eof() const
{
return m_files.empty()
|| m_files.top().m_stream.eof();
}

View File

@ -0,0 +1,30 @@
#pragma once
#include <stack>
#include <fstream>
#include "Parsing/IParserLineStream.h"
class ParserFilesystemStream final : public IParserLineStream
{
static const std::string EMPTY_FILE_NAME;
class FileInfo
{
public:
std::string m_file_path;
std::ifstream m_stream;
int m_line_number;
explicit FileInfo(std::string filePath);
};
std::stack<FileInfo> m_files;
public:
explicit ParserFilesystemStream(std::string path);
ParserLine NextLine() override;
bool IncludeFile(const std::string& filename) override;
_NODISCARD bool IsOpen() const override;
_NODISCARD bool Eof() const override;
};

View File

@ -0,0 +1,32 @@
#include "ParsingException.h"
#include <sstream>
ParsingException::ParsingException(const TokenPos position, std::string message)
: m_pos(position),
m_message(std::move(message))
{
std::ostringstream str;
str << m_pos.m_line << ':' << m_pos.m_column << ' ' << m_message;
m_full_message = str.str();
}
TokenPos ParsingException::Position() const
{
return m_pos;
}
const std::string& ParsingException::Message() const
{
return m_message;
}
std::string ParsingException::FullMessage() const
{
return m_full_message;
}
char const* ParsingException::what() const
{
return m_full_message.c_str();
}

View File

@ -0,0 +1,22 @@
#pragma once
#include <exception>
#include <string>
#include "TokenPos.h"
#include "Utils/ClassUtils.h"
class ParsingException final : std::exception
{
TokenPos m_pos;
std::string m_message;
std::string m_full_message;
public:
ParsingException(TokenPos position, std::string message);
_NODISCARD TokenPos Position() const;
_NODISCARD const std::string& Message() const;
_NODISCARD std::string FullMessage() const;
_NODISCARD char const* what() const override;
};

View File

@ -0,0 +1,17 @@
#include "TokenPos.h"
const std::string TokenPos::EMPTY_FILENAME;
TokenPos::TokenPos()
: m_filename(EMPTY_FILENAME),
m_line(1),
m_column(1)
{
}
TokenPos::TokenPos(const std::string& filename, const int line, const int column)
: m_filename(filename),
m_line(line),
m_column(column)
{
}

View File

@ -0,0 +1,16 @@
#pragma once
#include <string>
class TokenPos
{
static const std::string EMPTY_FILENAME;
public:
const std::string& m_filename;
int m_line;
int m_column;
TokenPos();
TokenPos(const std::string& filename, int line, int column);
};