zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-07-03 01:31:54 +00:00
Code Activity

Import code from previous AssetBuilder version

Browse Source
This commit is contained in:
Jan
2019-09-24 10:45:09 +02:00
parent 5609557516
commit 0d8432d4f7
919 changed files with 154412 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/ZoneCodeGenerator/Parsing/C_Header/BlockType.cs Normal file
View File

@ -0,0 +1,11 @@
namespace ZoneCodeGenerator.Parsing.C_Header
{
enum BlockType
{
None,
Enum,
Namespace,
Struct,
Union
}
}

25
src/ZoneCodeGenerator/Parsing/C_Header/Blocks/Block.cs Normal file
View File

@ -0,0 +1,25 @@
using System.Collections.Generic;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Blocks
{
abstract class Block
{
protected readonly IHeaderParserState State;
public BlockType Type { get; }
protected Block(IHeaderParserState headerParserState, BlockType type)
{
State = headerParserState;
Type = type;
}
public abstract IEnumerable<ITokenTest<IHeaderParserState>> GetAvailableTests();
public abstract void OnOpen();
public abstract void OnClose();
public abstract void OnChildBlockClose(Block childBlock);
}
}

114
src/ZoneCodeGenerator/Parsing/C_Header/Blocks/BlockEnum.cs Normal file
View File

@ -0,0 +1,114 @@
using System;
using System.Collections.Generic;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Tests;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Blocks
{
class BlockEnum : Block, INameAssignable, IVariableBlock
{
private readonly ITokenTest<IHeaderParserState>[] tests =
{
new TestEnumMember(),
new TestCloseBlock(true)
};
public string Name { get; }
public string AssignedName { get; private set; }
public string Namespace { get; private set; }
public DataTypeBaseType ParentType { get; }
public bool IsTypedef { get; }
public List<EnumMember> Members { get; }
private DataTypeEnum _enum;
public BlockEnum(IHeaderParserState headerParserState, string name, DataTypeBaseType parentType, bool isTypedef) : base(headerParserState, BlockType.Enum)
{
Name = name;
ParentType = parentType;
IsTypedef = isTypedef;
Members = new List<EnumMember>();
}
public override IEnumerable<ITokenTest<IHeaderParserState>> GetAvailableTests()
{
return tests;
}
public override void OnOpen()
{
Namespace = State.CurrentNamespace.ToString();
State.CurrentNamespace.Push(Name);
}
public override void OnClose()
{
var poppedNamespaceName = State.CurrentNamespace.Pop();
if (!Name.Equals(poppedNamespaceName))
throw new Exception($"Popped namespace '{poppedNamespaceName}' does not equal name of union block '{Name}'");
_enum = new DataTypeEnum(Namespace, Name, ParentType);
_enum.Members.AddRange(Members);
State.AddDataType(_enum);
if (IsTypedef)
{
var typeDeclaration = new TypeDeclaration(_enum, new List<ReferenceType>());
var typedef = new DataTypeTypedef(Namespace, AssignedName, typeDeclaration);
State.AddDataType(typedef);
}
}
public override void OnChildBlockClose(Block childBlock)
{
}
public void AssignName(string nameAssignment)
{
AssignedName = nameAssignment;
}
public virtual void AddMember(EnumMember enumMember)
{
Members.Add(enumMember);
}
public virtual EnumMember FindMember(string enumMemberName)
{
return Members.Find(member => member.Name.Equals(enumMemberName));
}
public virtual long GetNextEnumValue()
{
var nextValue = 0L;
if (Members.Count > 0)
{
nextValue = Members[Members.Count - 1].Value + 1;
}
return nextValue;
}
public bool DefinesVariable()
{
return !IsTypedef && !string.IsNullOrEmpty(AssignedName) && _enum != null;
}
public Variable GetDefinedVariable()
{
if (!DefinesVariable())
return null;
var typeDeclaration = new TypeDeclaration(_enum, new List<ReferenceType>());
return new Variable(AssignedName, typeDeclaration);
}
}
}

51
src/ZoneCodeGenerator/Parsing/C_Header/Blocks/BlockNamespace.cs Normal file
View File

@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using ZoneCodeGenerator.Parsing.C_Header.Tests;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Blocks
{
class BlockNamespace : Block
{
private readonly ITokenTest<IHeaderParserState>[] tests =
{
new TestNamespace(),
new TestEnum(),
new TestStruct(),
new TestUnion(),
new TestTypedef(),
new TestForwardDecl(),
new TestCloseBlock()
};
public string Name { get; }
public BlockNamespace(IHeaderParserState headerParserState, string namespaceName) : base(headerParserState, BlockType.Namespace)
{
Name = namespaceName;
}
public override IEnumerable<ITokenTest<IHeaderParserState>> GetAvailableTests()
{
return tests;
}
public override void OnOpen()
{
State.CurrentNamespace.Push(Name);
}
public override void OnClose()
{
var poppedNamespaceName = State.CurrentNamespace.Pop();
if (!Name.Equals(poppedNamespaceName))
throw new Exception($"Popped namespace '{poppedNamespaceName}' does not equal name of namespace block '{Name}'");
}
public override void OnChildBlockClose(Block childBlock)
{
// Do nothing
}
}
}

44
src/ZoneCodeGenerator/Parsing/C_Header/Blocks/BlockNone.cs Normal file
View File

@ -0,0 +1,44 @@
using System.Collections.Generic;
using ZoneCodeGenerator.Parsing.C_Header.Tests;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Blocks
{
class BlockNone : Block
{
private readonly ITokenTest<IHeaderParserState>[] tests =
{
new TestNamespace(),
new TestEnum(),
new TestStruct(),
new TestUnion(),
new TestTypedef(),
new TestForwardDecl()
};
public BlockNone(IHeaderParserState headerParserState) : base(headerParserState, BlockType.None)
{
}
public override IEnumerable<ITokenTest<IHeaderParserState>> GetAvailableTests()
{
return tests;
}
public override void OnOpen()
{
// Do nothing
}
public override void OnClose()
{
// Do nothing
}
public override void OnChildBlockClose(Block childBlock)
{
// Do nothing
}
}
}

117
src/ZoneCodeGenerator/Parsing/C_Header/Blocks/BlockStruct.cs Normal file
View File

@ -0,0 +1,117 @@
using System;
using System.Collections.Generic;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Tests;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Blocks
{
class BlockStruct : Block, INameAssignable, IVariableHolder, IVariableBlock
{
private readonly ITokenTest<IHeaderParserState>[] tests =
{
new TestStruct(),
new TestEnum(),
new TestUnion(),
new TestVariable(),
new TestCloseBlock(true)
};
public string Name { get; }
public string AssignedName { get; private set; }
public string Namespace { get; private set; }
public bool IsTypedef { get; }
public int? CustomAlignment { get; set; }
public int Pack { get; private set; }
public List<Variable> Variables { get; }
private DataTypeStruct _struct;
private DataTypeTypedef typedef;
public BlockStruct(IHeaderParserState headerParserState, string name, bool isTypedef) : base(headerParserState, BlockType.Struct)
{
Name = name;
IsTypedef = isTypedef;
Variables = new List<Variable>();
AssignedName = "";
_struct = null;
typedef = null;
}
public override IEnumerable<ITokenTest<IHeaderParserState>> GetAvailableTests()
{
return tests;
}
public override void OnOpen()
{
Pack = State.CurrentPack;
Namespace = State.CurrentNamespace.ToString();
State.CurrentNamespace.Push(Name);
State.AddForwardDeclaration(new ForwardDeclaration(Namespace, Name, DataTypeType.Struct));
}
public override void OnClose()
{
var poppedNamespaceName = State.CurrentNamespace.Pop();
if (!Name.Equals(poppedNamespaceName))
throw new Exception($"Popped namespace '{poppedNamespaceName}' does not equal name of struct block '{Name}'");
_struct = new DataTypeStruct(Namespace, Name, Pack);
_struct.Members.AddRange(Variables);
State.AddDataType(_struct);
if (IsTypedef)
{
var typeDeclaration = new TypeDeclaration(_struct, new List<ReferenceType>());
typedef = new DataTypeTypedef(Namespace, AssignedName, typeDeclaration);
State.AddDataType(typedef);
}
}
public override void OnChildBlockClose(Block childBlock)
{
if (childBlock is IVariableBlock variableBlock)
{
AddVariable(variableBlock.GetDefinedVariable());
}
}
public void AssignName(string nameAssignment)
{
AssignedName = nameAssignment;
}
public void AddVariable(Variable variable)
{
Variables.Add(variable);
}
public Variable GetDefinedVariable()
{
var name = "";
DataType dataType = _struct;
if (!IsTypedef)
name = AssignedName;
else
dataType = typedef;
var typeDeclaration = new TypeDeclaration(dataType, new List<ReferenceType>());
return new Variable(name, typeDeclaration);
}
public void Inherit(DataTypeStruct parent)
{
foreach(var variable in parent.Members)
AddVariable(variable);
}
}
}

120
src/ZoneCodeGenerator/Parsing/C_Header/Blocks/BlockUnion.cs Normal file
View File

@ -0,0 +1,120 @@
using System;
using System.Collections.Generic;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Tests;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Blocks
{
class BlockUnion : Block, INameAssignable, IVariableHolder, IVariableBlock
{
private readonly ITokenTest<IHeaderParserState>[] tests =
{
new TestStruct(),
new TestEnum(),
new TestUnion(),
new TestVariable(),
new TestCloseBlock(true)
};
public string Name { get; }
public string AssignedName { get; private set; }
public string Namespace { get; private set; }
public bool IsTypedef { get; }
public int? CustomAlignment { get; set; }
public int Pack { get; private set; }
public List<Variable> Variables { get; }
private DataTypeUnion union;
private DataTypeTypedef typedef;
public BlockUnion(IHeaderParserState headerParserState, string name, bool isTypedef) : base(headerParserState, BlockType.Union)
{
Name = name;
IsTypedef = isTypedef;
Variables = new List<Variable>();
CustomAlignment = null;
AssignedName = "";
union = null;
typedef = null;
}
public override IEnumerable<ITokenTest<IHeaderParserState>> GetAvailableTests()
{
return tests;
}
public override void OnOpen()
{
Namespace = State.CurrentNamespace.ToString();
Pack = State.CurrentPack;
State.CurrentNamespace.Push(Name);
State.AddForwardDeclaration(new ForwardDeclaration(Namespace, Name, DataTypeType.Union));
}
public override void OnClose()
{
var poppedNamespaceName = State.CurrentNamespace.Pop();
if (!Name.Equals(poppedNamespaceName))
throw new Exception($"Popped namespace '{poppedNamespaceName}' does not equal name of union block '{Name}'");
union = new DataTypeUnion(Namespace, Name, Pack)
{
AlignmentOverride = CustomAlignment
};
union.Members.AddRange(Variables);
State.AddDataType(union);
if (IsTypedef)
{
var typeDeclaration = new TypeDeclaration(union, new List<ReferenceType>());
typedef = new DataTypeTypedef(Namespace, AssignedName, typeDeclaration);
State.AddDataType(typedef);
}
}
public override void OnChildBlockClose(Block childBlock)
{
if (childBlock is IVariableBlock variableBlock)
{
AddVariable(variableBlock.GetDefinedVariable());
}
}
public void AssignName(string nameAssignment)
{
AssignedName = nameAssignment;
}
public void AddVariable(Variable variable)
{
Variables.Add(variable);
}
public Variable GetDefinedVariable()
{
var name = "";
DataType dataType = union;
if (!IsTypedef)
name = AssignedName;
else
dataType = typedef;
var typeDeclaration = new TypeDeclaration(dataType, new List<ReferenceType>());
return new Variable(name, typeDeclaration);
}
public void Inherit(DataTypeUnion parent)
{
foreach(var variable in parent.Members)
AddVariable(variable);
}
}
}

7
src/ZoneCodeGenerator/Parsing/C_Header/Blocks/INameAssignable.cs Normal file
View File

@ -0,0 +1,7 @@
namespace ZoneCodeGenerator.Parsing.C_Header.Blocks
{
interface INameAssignable
{
void AssignName(string nameAssignment);
}
}

9
src/ZoneCodeGenerator/Parsing/C_Header/Blocks/IVariableBlock.cs Normal file
View File

@ -0,0 +1,9 @@
using ZoneCodeGenerator.Domain;
namespace ZoneCodeGenerator.Parsing.C_Header.Blocks
{
interface IVariableBlock
{
Variable GetDefinedVariable();
}
}

9
src/ZoneCodeGenerator/Parsing/C_Header/Blocks/IVariableHolder.cs Normal file
View File

@ -0,0 +1,9 @@
using ZoneCodeGenerator.Domain;
namespace ZoneCodeGenerator.Parsing.C_Header.Blocks
{
interface IVariableHolder
{
void AddVariable(Variable variable);
}
}

79
src/ZoneCodeGenerator/Parsing/C_Header/HeaderReader.cs Normal file
View File

@ -0,0 +1,79 @@
using System;
using System.IO;
using System.Linq;
using ZoneCodeGenerator.Parsing.C_Header.Impl;
using ZoneCodeGenerator.Parsing.C_Header.PostProcessor;
using ZoneCodeGenerator.Parsing.Impl;
using ZoneCodeGenerator.Persistence;
namespace ZoneCodeGenerator.Parsing.C_Header
{
static class HeaderReader
{
private static readonly IDataPostProcessor[] postProcessors =
{
new PostProcessorUsages(),
};
public static IDataRepository ReadFile(string path)
{
try
{
using (IIncludingParsingStream streamFileSystem = new IncludingStreamFileSystem(path))
{
var state = new HeaderParserState();
IDataRepository dataRepository;
using (IParsingFileStream preprocessorStream = new Preprocessor(streamFileSystem, state))
{
var lexer = new Lexer(preprocessorStream);
var parser = new Parser<HeaderParserState>(state, lexer);
if (!parser.Parse())
return null;
dataRepository = new InMemoryDataRepository();
try
{
state.FinishAndSaveTo(dataRepository);
}
catch (LoadingException e)
{
PrintFinishingError(e);
return null;
}
preprocessorStream.Close();
streamFileSystem.Close();
}
if (!PostProcessRepository(dataRepository))
{
Console.WriteLine("Postprocessing data failed.");
return null;
}
return dataRepository;
}
}
catch (IOException)
{
return null;
}
}
private static bool PostProcessRepository(IDataRepository repository)
{
return postProcessors.All(postProcessor => postProcessor.PostProcess(repository));
}
private static void PrintFinishingError(LoadingException e)
{
Console.WriteLine("Parsing failed.");
Console.WriteLine(e.Message);
Console.WriteLine();
}
}
}

28
src/ZoneCodeGenerator/Parsing/C_Header/IHeaderParserState.cs Normal file
View File

@ -0,0 +1,28 @@
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Blocks;
using ZoneCodeGenerator.Persistence;
namespace ZoneCodeGenerator.Parsing.C_Header
{
interface IHeaderParserState : IParserState<IHeaderParserState>
{
Namespace CurrentNamespace { get; }
int CurrentPack { get; }
Block CurrentBlock { get; }
Block PreviousBlock { get; }
void PushPack(int pack);
int PopPack();
void PushBlock(Block block);
Block PopBlock();
void AddDataType(DataType dataType);
void AddForwardDeclaration(ForwardDeclaration forwardDeclaration);
DataType FindType(string typename);
EnumMember FindEnumMember(string enumMemberName);
void FinishAndSaveTo(IDataRepository dataRepository);
}
}

208
src/ZoneCodeGenerator/Parsing/C_Header/Impl/HeaderParserState.cs Normal file
View File

@ -0,0 +1,208 @@
using System;
using System.Collections.Generic;
using System.Linq;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Blocks;
using ZoneCodeGenerator.Parsing.Testing;
using ZoneCodeGenerator.Persistence;
namespace ZoneCodeGenerator.Parsing.C_Header.Impl
{
class HeaderParserState : IHeaderParserState
{
private const int DefaultPack = 8;
private readonly Block defaultBlock;
private readonly Stack<int> packStack;
private readonly Stack<Block> blockStack;
private readonly Dictionary<string, DataType> dataTypes;
private readonly Dictionary<string, ForwardDeclaration> forwardDeclarations;
public Namespace CurrentNamespace { get; }
public HeaderParserState()
{
defaultBlock = new BlockNone(this);
packStack = new Stack<int>();
blockStack = new Stack<Block>();
dataTypes = new Dictionary<string, DataType>();
forwardDeclarations = new Dictionary<string, ForwardDeclaration>();
CurrentNamespace = new Namespace();
}
public int CurrentPack => packStack.Count == 0 ? DefaultPack : packStack.Peek();
public Block CurrentBlock => blockStack.Count == 0 ? defaultBlock : blockStack.Peek();
public Block PreviousBlock => blockStack.Count < 2 ? defaultBlock : blockStack.ElementAt(1);
public void PushPack(int pack)
{
packStack.Push(pack);
}
public int PopPack()
{
return packStack.Pop();
}
public void PushBlock(Block block)
{
block.OnOpen();
blockStack.Push(block);
}
public Block PopBlock()
{
var block = blockStack.Pop();
block.OnClose();
CurrentBlock.OnChildBlockClose(block);
return block;
}
public void AddDataType(DataType dataType)
{
if (dataTypes.ContainsKey(dataType.FullName))
throw new ParserException($"Type '{dataType.FullName}' is already defined.");
dataTypes.Add(dataType.FullName, dataType);
}
public void AddForwardDeclaration(ForwardDeclaration forwardDeclaration)
{
if (forwardDeclarations.ContainsKey(forwardDeclaration.FullName))
{
var existingForwardDeclaration = forwardDeclarations[forwardDeclaration.FullName];
if(existingForwardDeclaration.Type != forwardDeclaration.Type)
throw new ParserException($"Forward declaration of type '{forwardDeclaration.FullName}' already done with different type.");
}
else
{
forwardDeclarations.Add(forwardDeclaration.FullName, forwardDeclaration);
}
}
public DataType FindType(string typename)
{
var currentNamespaceTypename = Namespace.Combine(CurrentNamespace, typename);
var baseType = DataTypeBaseType.BASE_TYPES.FirstOrDefault(databaseBaseType => databaseBaseType.Name.Equals(typename));
if (baseType != null)
return baseType;
if (dataTypes.ContainsKey(typename))
{
return dataTypes[typename];
}
if (dataTypes.ContainsKey(currentNamespaceTypename))
{
return dataTypes[currentNamespaceTypename];
}
if (forwardDeclarations.ContainsKey(typename))
{
return forwardDeclarations[typename];
}
if (forwardDeclarations.ContainsKey(currentNamespaceTypename))
{
return forwardDeclarations[currentNamespaceTypename];
}
return null;
}
public EnumMember FindEnumMember(string enumMemberName)
{
foreach(var block in blockStack)
{
if (!(block is BlockEnum blockEnum)) continue;
var foundMember = blockEnum.FindMember(enumMemberName);
if (foundMember != null)
return foundMember;
}
foreach (var dataType in dataTypes)
{
if (!(dataType.Value is DataTypeEnum _enum))
continue;
var foundMember = _enum.Members.Find(member => member.Name.Equals(enumMemberName));
if (foundMember != null)
return foundMember;
}
return null;
}
private void ResolveForwardDeclarations()
{
foreach(var forwardDeclaration in forwardDeclarations.Values)
{
var foundType = FindType(forwardDeclaration.FullName);
if (foundType == null || foundType is ForwardDeclaration)
throw new ParserException($"Type {forwardDeclaration.FullName} was declared but not defined.");
forwardDeclaration.ForwardedType = foundType;
}
foreach (var dataType in dataTypes.Values)
{
if (!(dataType is DataTypeWithMembers dataTypeWithMembers)) continue;
foreach (var variable in dataTypeWithMembers.Members)
{
if (variable.VariableType.Type is ForwardDeclaration forwardDeclaration)
variable.VariableType.Type = forwardDeclaration.ForwardedType;
}
}
}
public void FinishAndSaveTo(IDataRepository dataRepository)
{
if(blockStack.Count > 0)
throw new ParserException($"Parsing finished but {blockStack.Count} blocks were not closed.");
ResolveForwardDeclarations();
foreach (var dataType in dataTypes.Values)
{
switch (dataType)
{
case DataTypeEnum _enum:
dataRepository.Add(_enum);
break;
case DataTypeStruct _struct:
_struct.FinalizeDataType();
dataRepository.Add(_struct);
break;
case DataTypeTypedef typedef:
dataRepository.Add(typedef);
break;
case DataTypeUnion union:
union.FinalizeDataType();
dataRepository.Add(union);
break;
default:
throw new ArgumentOutOfRangeException(nameof(dataType));
}
}
}
public IEnumerable<ITokenTest<IHeaderParserState>> GetTests()
{
return CurrentBlock.GetAvailableTests();
}
}
}

127
src/ZoneCodeGenerator/Parsing/C_Header/Impl/Preprocessor.cs Normal file
View File

@ -0,0 +1,127 @@
using System.Collections.Generic;
using System.Text.RegularExpressions;
using ZoneCodeGenerator.Parsing.Impl;
namespace ZoneCodeGenerator.Parsing.C_Header.Impl
{
class Preprocessor : IParsingFileStream
{
private static readonly Regex packPushRegex = new Regex(@"^\s*#pragma\s+pack\s*\(\s*push\s*,\s*(\d+)\s*\)\s*$");
private static readonly Regex packPopRegex = new Regex(@"^\s*#pragma\s+pack\s*\(\s*pop\s*\)\s*$");
private static readonly Regex defineRegex = new Regex(@"^\s*#define\s*(\w+)(?:\s*(.*))?$");
private static readonly Regex undefRegex = new Regex(@"^\s*#undef\s*(\w+)\s*$");
private static readonly Regex includeRegex = new Regex(@"^\s*#include\s*(?:\""(.*)\""|\<(.*)\>)\s*$");
private readonly IIncludingParsingStream streamFileSystem;
private readonly IHeaderParserState state;
private readonly ICommentProcessor commentProcessor;
private readonly Dictionary<string, string> defines;
public bool EndOfStream => streamFileSystem.EndOfStream;
public string Filename => streamFileSystem.Filename;
public int Line => streamFileSystem.Line;
public Preprocessor(IIncludingParsingStream includingParsingStream, IHeaderParserState state)
{
streamFileSystem = includingParsingStream;
this.state = state;
defines = new Dictionary<string, string>();
commentProcessor = new CommentProcessor();
}
private void ParseCompilerExpression(string line)
{
Match packPush = packPushRegex.Match(line.ToLower());
if(packPush.Success)
{
state.PushPack(int.Parse(packPush.Groups[1].Value));
return;
}
Match packPop = packPopRegex.Match(line.ToLower());
if(packPop.Success)
{
state.PopPack();
return;
}
Match define = defineRegex.Match(line);
if (define.Success)
{
string key = define.Groups[1].Value;
string value = define.Groups[2].Value.Trim();
if (defines.ContainsKey(key))
defines[key] = value;
else
defines.Add(key, value);
return;
}
Match undef = undefRegex.Match(line);
if (undef.Success)
{
var key = undef.Groups[1].Value;
if(defines.ContainsKey(key))
defines.Remove(key);
return;
}
Match include = includeRegex.Match(line);
if (include.Success)
{
string filename = include.Groups[1].Success ? include.Groups[1].Value : include.Groups[2].Value;
streamFileSystem.IncludeFile(filename);
return;
}
}
private string Preprocess(string line)
{
bool defineMatched;
line = commentProcessor.RemoveComments(line);
if (line.StartsWith("#"))
{
ParseCompilerExpression(line);
return "";
}
do
{
defineMatched = false;
foreach (KeyValuePair<string, string> define in defines)
{
Match match = Regex.Match(line, $@"^(.*\W)?{define.Key}(\W.*)?$");
if (!match.Success) continue;
line = match.Groups[1].Value + define.Value + match.Groups[2].Value;
defineMatched = true;
break;
}
} while (defineMatched);
return line;
}
public string ReadLine()
{
return Preprocess(streamFileSystem.ReadLine());
}
public void Dispose()
{
Close();
}
public void Close()
{
streamFileSystem.Close();
}
}
}

57
src/ZoneCodeGenerator/Parsing/C_Header/PostProcessor/PostProcessorUsages.cs Normal file
View File

@ -0,0 +1,57 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Persistence;
namespace ZoneCodeGenerator.Parsing.C_Header.PostProcessor
{
class PostProcessorUsages : IDataPostProcessor
{
public bool PostProcess(IDataRepository repository)
{
foreach (var dataTypeWithMembers in repository.GetAllStructs()
.AsEnumerable<DataTypeWithMembers>()
.Concat(repository.GetAllUnions()))
{
var information = repository.GetInformationFor(dataTypeWithMembers);
foreach (var memberInformation in information.OrderedMembers)
{
if (memberInformation.StructureType == null) continue;
memberInformation.StructureType.Usages.Add(information);
if (IsNonEmbeddedReference(memberInformation.Member))
memberInformation.StructureType.NonEmbeddedReferenceExists = true;
if (IsPointerReference(memberInformation.Member))
memberInformation.StructureType.PointerReferenceExists = true;
if (IsArrayReference(memberInformation.Member))
memberInformation.StructureType.PointerReferenceExists = true;
}
}
return true;
}
private static bool IsNonEmbeddedReference(Variable var)
{
return var.VariableType.References.Any();
}
private static bool IsPointerReference(Variable var)
{
return var.VariableType.References.Any()
&& var.VariableType.References.Last() is ReferenceTypePointer;
}
private static bool IsArrayReference(Variable var)
{
return var.VariableType.References.Any()
&& var.VariableType.References.Last() is ReferenceTypeArray;
}
}
}

48
src/ZoneCodeGenerator/Parsing/C_Header/Tests/TestCloseBlock.cs Normal file
View File

@ -0,0 +1,48 @@
using ZoneCodeGenerator.Parsing.C_Header.Blocks;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Tests
{
class TestCloseBlock : AbstractTokenTest<IHeaderParserState>
{
private const string NameToken = "close_with_name";
private const string SemicolonToken = "close_with_semi_colon";
private readonly bool semicolonRequired;
private static readonly TokenMatcher[] matchers = {
new MatcherLiteral("}"),
new MatcherGroupOptional( new MatcherGroupAnd(
new MatcherGroupOptional(new MatcherName().WithName(NameToken)),
new MatcherLiteral(";").WithName(SemicolonToken)
))
};
public TestCloseBlock(bool semicolonRequired = false) : base(matchers)
{
this.semicolonRequired = semicolonRequired;
}
protected override void ProcessMatch(IHeaderParserState state)
{
if (HasMatcherTokens(NameToken))
{
if (state.CurrentBlock is INameAssignable nameAssignableBlock)
{
nameAssignableBlock.AssignName(GetMatcherTokens(NameToken)[0]);
}
else
{
throw new TestFailedException($"Cannot give block '{state.CurrentBlock.Type}' a name when closing");
}
}
if (!HasMatcherTokens(SemicolonToken) && semicolonRequired)
throw new TestFailedException($"Block '{state.CurrentBlock.Type}' needs to be closed with a '{'}'};'");
state.PopBlock();
}
}
}

57
src/ZoneCodeGenerator/Parsing/C_Header/Tests/TestEnum.cs Normal file
View File

@ -0,0 +1,57 @@
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Blocks;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
using ZoneCodeGenerator.Utils;
namespace ZoneCodeGenerator.Parsing.C_Header.Tests
{
class TestEnum : AbstractTokenTest<IHeaderParserState>
{
private const string TypedefToken = "typedef";
private const string NameToken = "name";
private const string TypenameToken = "typename";
private static readonly TokenMatcher[] matchers = {
new MatcherGroupOptional(new MatcherLiteral("typedef").WithName(TypedefToken)),
new MatcherLiteral("enum"),
new MatcherGroupOptional(new MatcherName().WithName(NameToken)),
new MatcherGroupOptional(new MatcherGroupAnd(new MatcherLiteral(":"), new MatcherTypename().WithName(TypenameToken))),
new MatcherLiteral("{")
};
public TestEnum() : base(matchers)
{
}
protected override void ProcessMatch(IHeaderParserState state)
{
var isTypedef = HasMatcherTokens(TypedefToken);
var name = HasMatcherTokens(NameToken) ? GetMatcherTokens(NameToken)[0] : RandomName.GenerateName();
var parentType = DataTypeBaseType.INT;
if (HasMatcherTokens(TypenameToken))
{
var typeName = GetMatcherTokens(TypenameToken)[0];
var type = state.FindType(typeName);
if (type == null)
throw new TestFailedException($"Parent type '{typeName}' of enum '{name}' not found.");
while (type is DataTypeTypedef typedef)
{
if (typedef.TypeDefinition.References.Count > 0)
throw new TestFailedException($"Parent type of enum '{name}' cannot be a typedef that is an array or a pointer.");
type = typedef.TypeDefinition.Type;
}
parentType = type as DataTypeBaseType ?? throw new TestFailedException($"Parent type of enum '{name}' must be a base type.");
}
state.PushBlock(new BlockEnum(state, name, parentType, isTypedef));
}
}
}

66
src/ZoneCodeGenerator/Parsing/C_Header/Tests/TestEnumMember.cs Normal file
View File

@ -0,0 +1,66 @@
using System;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Blocks;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Tests
{
class TestEnumMember : AbstractTokenTest<IHeaderParserState>
{
private const string NameToken = "name";
private const string NumberValueToken = "value_number";
private const string EnumMemberValueToken = "value_string";
private static readonly TokenMatcher[] matchers = {
new MatcherName().WithName(NameToken),
new MatcherGroupOptional(new MatcherGroupAnd(
new MatcherLiteral("="),
new MatcherGroupOr(
new MatcherNumber().WithName(NumberValueToken),
new MatcherName().WithName(EnumMemberValueToken)
))),
new MatcherGroupOr(
new MatcherLiteral(","),
new MatcherLiteral("}").NonConsuming()
)
};
public TestEnumMember() : base(matchers)
{
}
protected override void ProcessMatch(IHeaderParserState state)
{
var name = GetMatcherTokens(NameToken)[0];
long value;
if (!(state.CurrentBlock is BlockEnum _enum))
throw new Exception("Excepted enum to be top block when parsing an enum member.");
if (HasMatcherTokens(NumberValueToken))
{
value = long.Parse(GetMatcherTokens(NumberValueToken)[0]);
}
else if(HasMatcherTokens(EnumMemberValueToken))
{
var stringValue = GetMatcherTokens(EnumMemberValueToken)[0];
var memberWithFittingName = state.FindEnumMember(stringValue);
if(memberWithFittingName == null)
throw new TestFailedException($"Could not find value for '{stringValue}'.");
value = memberWithFittingName.Value;
}
else
{
value = _enum.GetNextEnumValue();
}
var enumMember = new EnumMember(name, value);
_enum.AddMember(enumMember);
}
}
}

52
src/ZoneCodeGenerator/Parsing/C_Header/Tests/TestForwardDecl.cs Normal file
View File

@ -0,0 +1,52 @@
using System;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Tests
{
class TestForwardDecl : AbstractTokenTest<IHeaderParserState>
{
private const string EnumToken = "enum";
private const string StructToken = "struct";
private const string UnionToken = "union";
private const string NameToken = "name";
private static readonly TokenMatcher[] matchers = {
new MatcherGroupOr(
new MatcherLiteral("enum").WithName(EnumToken),
new MatcherLiteral("struct").WithName(StructToken),
new MatcherLiteral("union").WithName(UnionToken)
),
new MatcherName().WithName(NameToken),
new MatcherLiteral(";")
};
public TestForwardDecl() : base(matchers)
{
}
protected override void ProcessMatch(IHeaderParserState state)
{
var name = GetMatcherTokens(NameToken)[0];
var _namespace = state.CurrentNamespace.ToString();
if (HasMatcherTokens(EnumToken))
{
state.AddForwardDeclaration(new ForwardDeclaration(_namespace, name, DataTypeType.Enum));
}
else if (HasMatcherTokens(StructToken))
{
state.AddForwardDeclaration(new ForwardDeclaration(_namespace, name, DataTypeType.Struct));
}
else if (HasMatcherTokens(UnionToken))
{
state.AddForwardDeclaration(new ForwardDeclaration(_namespace, name, DataTypeType.Union));
}
else
throw new Exception("Unknown type for forward declaration.");
}
}
}

29
src/ZoneCodeGenerator/Parsing/C_Header/Tests/TestNamespace.cs Normal file
View File

@ -0,0 +1,29 @@
using ZoneCodeGenerator.Parsing.C_Header.Blocks;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Tests
{
class TestNamespace : AbstractTokenTest<IHeaderParserState>
{
private const string NamespaceNameToken = "namespace_name";
private static readonly TokenMatcher[] matchers =
{
new MatcherLiteral("namespace"),
new MatcherName().WithName(NamespaceNameToken),
new MatcherLiteral("{")
};
public TestNamespace() : base(matchers)
{
}
protected override void ProcessMatch(IHeaderParserState state)
{
state.PushBlock(new BlockNamespace(state, GetMatcherTokens(NamespaceNameToken)[0]));
}
}
}

69
src/ZoneCodeGenerator/Parsing/C_Header/Tests/TestStruct.cs Normal file
View File

@ -0,0 +1,69 @@
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Blocks;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
using ZoneCodeGenerator.Utils;
namespace ZoneCodeGenerator.Parsing.C_Header.Tests
{
class TestStruct : AbstractTokenTest<IHeaderParserState>
{
private const string TypedefToken = "typedef";
private const string NameToken = "name";
private const string AlignToken = "align";
private const string ParentToken = "parent";
private static readonly TokenMatcher[] matchers = {
new MatcherGroupOptional(new MatcherLiteral("typedef").WithName(TypedefToken)),
new MatcherGroupOptional(new MatcherLiteral("const")),
new MatcherLiteral("struct"),
new MatcherGroupOptional(new MatcherGroupAnd(
new MatcherLiteral("__declspec"),
new MatcherLiteral("("),
new MatcherLiteral("align"),
new MatcherLiteral("("),
new MatcherNumber().WithName(AlignToken),
new MatcherLiteral(")"),
new MatcherLiteral(")")
)),
new MatcherGroupOptional(new MatcherName().WithName(NameToken)),
new MatcherGroupOptional(new MatcherGroupAnd(
new MatcherLiteral(":"),
new MatcherTypename().WithName(ParentToken)
)),
new MatcherLiteral("{")
};
public TestStruct() : base(matchers)
{
}
protected override void ProcessMatch(IHeaderParserState state)
{
var isTypedef = HasMatcherTokens(TypedefToken);
var name = HasMatcherTokens(NameToken) ? GetMatcherTokens(NameToken)[0] : RandomName.GenerateName();
var block = new BlockStruct(state, name, isTypedef);
if (HasMatcherTokens(AlignToken))
block.CustomAlignment = int.Parse(GetMatcherTokens(AlignToken)[0]);
if (HasMatcherTokens(ParentToken))
{
var parentDataTypeName = GetMatcherTokens(ParentToken)[0];
var parentDataType = state.FindType(parentDataTypeName);
if(parentDataType == null)
throw new TestFailedException($"Could not find parent data type '{parentDataTypeName}' for struct '{name}'");
if(!(parentDataType is DataTypeStruct parentStruct))
throw new TestFailedException($"Parent data type '{parentDataTypeName}' for struct '{name}' must be a struct as well.");
block.Inherit(parentStruct);
}
state.PushBlock(block);
}
}
}

96
src/ZoneCodeGenerator/Parsing/C_Header/Tests/TestTypedef.cs Normal file
View File

@ -0,0 +1,96 @@
using System.Collections.Generic;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Tests
{
class TestTypedef : AbstractTokenTest<IHeaderParserState>
{
private const string PointerToArrayVariant = "pointertoarray";
private const string ArrayOfPointersVariant = "arrayofpointers";
private const string TypeNameTokens = "typename";
private const string PointerTokens = "pointer";
private const string NameToken = "name";
private const string ArrayTokens = "array";
private static readonly TokenMatcher[] arrayOfPointersMatchers = {
new MatcherLiteral("typedef"),
new MatcherTypename().WithName(TypeNameTokens),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.ZeroOneMultiple, new MatcherLiteral("*").WithName(PointerTokens)),
new MatcherName().WithName(NameToken),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.ZeroOneMultiple, new MatcherArray().WithName(ArrayTokens)),
new MatcherLiteral(";").WithName(ArrayOfPointersVariant)
};
private static readonly TokenMatcher[] pointerToArrayMatchers = {
new MatcherLiteral("typedef"),
new MatcherTypename().WithName(TypeNameTokens),
new MatcherLiteral("("),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.OneMultiple, new MatcherLiteral("*").WithName(PointerTokens)),
new MatcherName().WithName(NameToken),
new MatcherLiteral(")"),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.OneMultiple, new MatcherArray().WithName(ArrayTokens)),
new MatcherLiteral(";").WithName(PointerToArrayVariant)
};
private static readonly TokenMatcher[] matchers =
{
new MatcherGroupOr(
new MatcherGroupAnd(arrayOfPointersMatchers),
new MatcherGroupAnd(pointerToArrayMatchers)
)
};
public TestTypedef() : base(matchers)
{
}
protected override void ProcessMatch(IHeaderParserState state)
{
var name = GetMatcherTokens(NameToken)[0];
var pointerDepth = GetMatcherTokens(PointerTokens).Count;
var typeName = string.Join(" ", GetMatcherTokens(TypeNameTokens));
var type = state.FindType(typeName);
if (type == null)
throw new TestFailedException($"Could not find type '{typeName}' of typedef '{name}'.");
var arrayTokens = GetMatcherTokens(ArrayTokens);
var arraySize = new int[arrayTokens.Count];
for(var i = 0; i < arrayTokens.Count; i++)
{
if (!int.TryParse(arrayTokens[i], out arraySize[i]))
throw new TestFailedException($"Array size '{arrayTokens[i]}' is not numeric.");
}
var references = new List<ReferenceType>();
if (HasMatcherTokens(PointerToArrayVariant))
{
for (var i = 0; i < pointerDepth; i++)
references.Add(new ReferenceTypePointer());
foreach(var array in arraySize)
references.Add(new ReferenceTypeArray(array));
}
else
{
foreach(var array in arraySize)
references.Add(new ReferenceTypeArray(array));
for (var i = 0; i < pointerDepth; i++)
references.Add(new ReferenceTypePointer());
}
var typeDeclaration = new TypeDeclaration(type, references);
var typedef = new DataTypeTypedef(state.CurrentNamespace.ToString(), name, typeDeclaration);
state.AddDataType(typedef);
}
}
}

69
src/ZoneCodeGenerator/Parsing/C_Header/Tests/TestUnion.cs Normal file
View File

@ -0,0 +1,69 @@
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Blocks;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
using ZoneCodeGenerator.Utils;
namespace ZoneCodeGenerator.Parsing.C_Header.Tests
{
class TestUnion : AbstractTokenTest<IHeaderParserState>
{
private const string TypedefToken = "typedef";
private const string NameToken = "name";
private const string AlignToken = "align";
private const string ParentToken = "parent";
private static readonly TokenMatcher[] matchers = {
new MatcherGroupOptional(new MatcherLiteral("typedef").WithName(TypedefToken)),
new MatcherGroupOptional(new MatcherLiteral("const")),
new MatcherLiteral("union"),
new MatcherGroupOptional(new MatcherGroupAnd(
new MatcherLiteral("__declspec"),
new MatcherLiteral("("),
new MatcherLiteral("align"),
new MatcherLiteral("("),
new MatcherNumber().WithName(AlignToken),
new MatcherLiteral(")"),
new MatcherLiteral(")")
)),
new MatcherGroupOptional(new MatcherName().WithName(NameToken)),
new MatcherGroupOptional(new MatcherGroupAnd(
new MatcherLiteral(":"),
new MatcherTypename().WithName(ParentToken)
)),
new MatcherLiteral("{")
};
public TestUnion() : base(matchers)
{
}
protected override void ProcessMatch(IHeaderParserState state)
{
var isTypedef = HasMatcherTokens(TypedefToken);
var name = HasMatcherTokens(NameToken) ? GetMatcherTokens(NameToken)[0] : RandomName.GenerateName();
var block = new BlockUnion(state, name, isTypedef);
if (HasMatcherTokens(AlignToken))
block.CustomAlignment = int.Parse(GetMatcherTokens(AlignToken)[0]);
if (HasMatcherTokens(ParentToken))
{
var parentDataTypeName = GetMatcherTokens(ParentToken)[0];
var parentDataType = state.FindType(parentDataTypeName);
if(parentDataType == null)
throw new TestFailedException($"Could not find parent data type '{parentDataTypeName}' for union '{name}'");
if(!(parentDataType is DataTypeUnion parentUnion))
throw new TestFailedException($"Parent data type '{parentDataTypeName}' for union '{name}' must be a union as well.");
block.Inherit(parentUnion);
}
state.PushBlock(block);
}
}
}

112
src/ZoneCodeGenerator/Parsing/C_Header/Tests/TestVariable.cs Normal file
View File

@ -0,0 +1,112 @@
using System;
using System.Collections.Generic;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.C_Header.Blocks;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.C_Header.Tests
{
class TestVariable : AbstractTokenTest<IHeaderParserState>
{
private const string PointerToArrayVariant = "pointertoarray";
private const string ArrayOfPointersVariant = "arrayofpointers";
private const string TypeNameToken = "typename";
private const string PointerTokens = "pointer";
private const string NameToken = "name";
private const string ArrayTokens = "array";
private const string BitSizeToken = "bitsize";
private static readonly TokenMatcher[] arrayOfPointersMatchers = {
new MatcherTypename().WithName(TypeNameToken),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.ZeroOneMultiple, new MatcherLiteral("*").WithName(PointerTokens)),
new MatcherName().WithName(NameToken),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.ZeroOneMultiple, new MatcherArray().WithName(ArrayTokens)),
new MatcherGroupOptional(new MatcherGroupAnd(
new MatcherLiteral(":"),
new MatcherNumber().WithName(BitSizeToken)
)),
new MatcherLiteral(";").WithName(ArrayOfPointersVariant)
};
private static readonly TokenMatcher[] pointerToArrayMatchers = {
new MatcherTypename().WithName(TypeNameToken),
new MatcherLiteral("("),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.OneMultiple, new MatcherLiteral("*").WithName(PointerTokens)),
new MatcherName().WithName(NameToken),
new MatcherLiteral(")"),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.OneMultiple, new MatcherArray().WithName(ArrayTokens)),
new MatcherLiteral(";").WithName(PointerToArrayVariant)
};
private static readonly TokenMatcher[] matchers =
{
new MatcherGroupOr(
new MatcherGroupAnd(arrayOfPointersMatchers),
new MatcherGroupAnd(pointerToArrayMatchers)
)
};
public TestVariable() : base(matchers)
{
}
protected override void ProcessMatch(IHeaderParserState state)
{
var name = GetMatcherTokens(NameToken)[0];
var typeName = GetMatcherTokens(TypeNameToken)[0];
var type = state.FindType(typeName);
if (type == null)
throw new TestFailedException($"Type '{typeName}' not found.");
var pointerDepth = GetMatcherTokens(PointerTokens).Count;
var arrayTokens = GetMatcherTokens(ArrayTokens);
var arraySize = new int[arrayTokens.Count];
int? bitSize = null;
if (HasMatcherTokens(BitSizeToken))
bitSize = int.Parse(GetMatcherTokens(BitSizeToken)[0]);
for(var i = 0; i < arrayTokens.Count; i++)
{
if (!int.TryParse(arrayTokens[i], out arraySize[i]))
throw new TestFailedException($"Array size '{arrayTokens[i]}' is not numeric.");
}
if (state.CurrentBlock is IVariableHolder variableHolder)
{
var references = new List<ReferenceType>();
if (HasMatcherTokens(PointerToArrayVariant))
{
for (var i = 0; i < pointerDepth; i++)
references.Add(new ReferenceTypePointer());
foreach(var array in arraySize)
references.Add(new ReferenceTypeArray(array));
}
else
{
foreach(var array in arraySize)
references.Add(new ReferenceTypeArray(array));
for (var i = 0; i < pointerDepth; i++)
references.Add(new ReferenceTypePointer());
}
var typeDeclaration = bitSize == null ? new TypeDeclaration(type, references) : new TypeDeclaration(type, bitSize.Value, references);
var variable = new Variable(name, typeDeclaration);
variableHolder.AddVariable(variable);
}
else
{
throw new Exception("Expected current block to be a variable holder when parsing variables.");
}
}
}
}

67
src/ZoneCodeGenerator/Parsing/CommandFile/CommandFileReader.cs Normal file
View File

@ -0,0 +1,67 @@
using System;
using System.IO;
using System.Linq;
using ZoneCodeGenerator.Interface;
using ZoneCodeGenerator.Parsing.CommandFile.Impl;
using ZoneCodeGenerator.Parsing.CommandFile.PostProcessor;
using ZoneCodeGenerator.Parsing.Impl;
using ZoneCodeGenerator.Persistence;
namespace ZoneCodeGenerator.Parsing.CommandFile
{
static class CommandFileReader
{
private static readonly IDataPostProcessor[] postProcessors =
{
new PostProcessorDefaultBlock(),
};
public static bool ReadFile(string path, CUISession session)
{
try
{
using (IIncludingParsingStream streamFileSystem = new IncludingStreamFileSystem(path))
{
var state = new CommandParserState(session.Repository);
using (IParsingFileStream preprocessorStream = new CommandFilePreprocessor(streamFileSystem))
{
var lexer = new Lexer(preprocessorStream);
var parser = new Parser<ICommandParserState>(state, lexer);
if (!parser.Parse())
return false;
preprocessorStream.Close();
streamFileSystem.Close();
if (!state.Apply(session))
{
Console.WriteLine("Finalizing from command file failed");
return false;
}
if (!PostProcessRepository(session.Repository))
{
Console.WriteLine("Postprocessing commands failed.");
return false;
}
}
return true;
}
}
catch (IOException)
{
}
return false;
}
private static bool PostProcessRepository(IDataRepository repository)
{
return postProcessors.All(postProcessor => postProcessor.PostProcess(repository));
}
}
}

15
src/ZoneCodeGenerator/Parsing/CommandFile/ICommandParserState.cs Normal file
View File

@ -0,0 +1,15 @@
using System.Collections.Generic;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Domain.FastFileStructure;
using ZoneCodeGenerator.Persistence;
namespace ZoneCodeGenerator.Parsing.CommandFile
{
interface ICommandParserState : IParserState<ICommandParserState>
{
string Game { get; set; }
IReadOnlyDataRepository Repository { get; }
List<FastFileBlock> FastFileBlocks { get; }
DataTypeWithMembers DataTypeInUse { get; set; }
}
}

56
src/ZoneCodeGenerator/Parsing/CommandFile/Impl/CommandFilePreprocessor.cs Normal file
View File

@ -0,0 +1,56 @@
using System.Text.RegularExpressions;
using ZoneCodeGenerator.Parsing.Impl;
namespace ZoneCodeGenerator.Parsing.CommandFile.Impl
{
class CommandFilePreprocessor : IParsingFileStream
{
private static readonly Regex includeRegex = new Regex(@"^\s*include\s*(?:\""(.*)\""|\<(.*)\>)\s*$");
private readonly IIncludingParsingStream parsingStream;
private readonly ICommentProcessor commentProcessor;
public CommandFilePreprocessor(IIncludingParsingStream parsingStream)
{
this.parsingStream = parsingStream;
commentProcessor = new CommentProcessor();
}
public void Dispose()
{
parsingStream.Close();
}
public bool EndOfStream => parsingStream.EndOfStream;
public string Filename => parsingStream.Filename;
public int Line => parsingStream.Line;
public string ReadLine()
{
return Preprocess(parsingStream.ReadLine());
}
public void Close()
{
parsingStream.Close();
}
private string Preprocess(string line)
{
line = commentProcessor.RemoveComments(line);
var includeMatch = includeRegex.Match(line);
if (includeMatch.Success)
{
var filename = includeMatch.Groups[1].Success ? includeMatch.Groups[1].Value : includeMatch.Groups[2].Value;
parsingStream.IncludeFile(filename);
return "";
}
return line;
}
}
}

53
src/ZoneCodeGenerator/Parsing/CommandFile/Impl/CommandParserState.cs Normal file
View File

@ -0,0 +1,53 @@
using System.Collections.Generic;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Domain.FastFileStructure;
using ZoneCodeGenerator.Interface;
using ZoneCodeGenerator.Parsing.CommandFile.Tests;
using ZoneCodeGenerator.Parsing.Testing;
using ZoneCodeGenerator.Persistence;
namespace ZoneCodeGenerator.Parsing.CommandFile.Impl
{
class CommandParserState : ICommandParserState
{
private static readonly ITokenTest<ICommandParserState>[] tests = {
new TestAsset(),
new TestBlock(),
new TestCondition(),
new TestCount(),
new TestGame(),
new TestReorder(),
new TestScriptString(),
new TestUse()
};
public string Game { get; set; }
public IReadOnlyDataRepository Repository { get; }
public List<FastFileBlock> FastFileBlocks { get; }
public DataTypeWithMembers DataTypeInUse { get; set; }
public CommandParserState(IReadOnlyDataRepository repository)
{
Repository = repository;
FastFileBlocks = new List<FastFileBlock>();
DataTypeInUse = null;
}
public IEnumerable<ITokenTest<ICommandParserState>> GetTests()
{
return tests;
}
public bool Apply(CUISession session)
{
session.Game = Game;
foreach (var block in FastFileBlocks)
{
session.Repository.Add(block);
}
return true;
}
}
}

33
src/ZoneCodeGenerator/Parsing/CommandFile/PostProcessor/PostProcessorDefaultBlock.cs Normal file
View File

@ -0,0 +1,33 @@
using System.Linq;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Domain.FastFileStructure;
using ZoneCodeGenerator.Persistence;
namespace ZoneCodeGenerator.Parsing.CommandFile.PostProcessor
{
class PostProcessorDefaultBlock : IDataPostProcessor
{
public bool PostProcess(IDataRepository repository)
{
var memberDataTypes =
repository.GetAllStructs()
.AsEnumerable<DataTypeWithMembers>()
.Concat(repository.GetAllUnions());
var defaultTemp = repository.GetAllFastFileBlocks().First(block => block.BlockType == FastFileBlock.Type.Temp && block.IsDefault) ??
repository.GetAllFastFileBlocks().First(block => block.BlockType == FastFileBlock.Type.Temp);
var defaultNormal = repository.GetAllFastFileBlocks().First(block => block.BlockType == FastFileBlock.Type.Normal && block.IsDefault) ??
repository.GetAllFastFileBlocks().First(block => block.BlockType == FastFileBlock.Type.Normal);
foreach (var memberType in memberDataTypes)
{
var info = repository.GetInformationFor(memberType);
info.Block = info.IsAsset ? defaultTemp : defaultNormal;
}
return true;
}
}
}

59
src/ZoneCodeGenerator/Parsing/CommandFile/Tests/TestAsset.cs Normal file
View File

@ -0,0 +1,59 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.CommandFile.Tests
{
class TestAsset : AbstractTokenTest<ICommandParserState>
{
private const string AssetTypeNameToken = "name";
private const string AssetEnumEntryToken = "enumEntry";
private static readonly TokenMatcher[] matchers = {
new MatcherLiteral("asset"),
new MatcherTypename().WithName(AssetTypeNameToken),
new MatcherName().WithName(AssetEnumEntryToken),
new MatcherLiteral(";")
};
public TestAsset() : base(matchers)
{
}
protected override void ProcessMatch(ICommandParserState state)
{
var assetTypeName = GetMatcherTokens(AssetTypeNameToken)[0];
var assetType = state.Repository.GetDataTypeByName(assetTypeName);
if (assetType == null)
{
throw new LoadingException($"Could not find type '{assetTypeName}' to mark it as an asset.");
}
if (!(assetType is DataTypeWithMembers assetTypeWithMembers))
{
throw new LoadingException($"Type of asset '{assetTypeName}' needs to be struct or union.");
}
var assetInfo = state.Repository.GetInformationFor(assetTypeWithMembers);
if (assetType == null)
{
throw new LoadingException($"Could not find information for type '{assetTypeName}' to mark it as an asset.");
}
var enumEntryName = GetMatcherTokens(AssetEnumEntryToken)[0];
var enumEntry = state.Repository.GetAllEnums()
.SelectMany(_enum => _enum.Members)
.FirstOrDefault(member => member.Name.Equals(enumEntryName, StringComparison.CurrentCultureIgnoreCase));
assetInfo.AssetEnumEntry = enumEntry ?? throw new LoadingException(
$"Could not find enum entry '{enumEntryName}' as an asset type index for asset type '{assetTypeName}'.");
}
}
}

52
src/ZoneCodeGenerator/Parsing/CommandFile/Tests/TestBlock.cs Normal file
View File

@ -0,0 +1,52 @@
using System;
using System.Linq;
using ZoneCodeGenerator.Domain.FastFileStructure;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.CommandFile.Tests
{
class TestBlock : AbstractTokenTest<ICommandParserState>
{
private const string BlockNumberToken = "num";
private const string BlockTypeToken = "type";
private const string BlockNameToken = "name";
private const string DefaultToken = "default";
private static readonly TokenMatcher[] matchers = {
new MatcherLiteral("block"),
new MatcherNumber().WithName(BlockNumberToken),
new MatcherName().WithName(BlockTypeToken),
new MatcherName().WithName(BlockNameToken),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.ZeroOneMultiple,new MatcherGroupOr(
new MatcherLiteral("default").WithName(DefaultToken)
)),
new MatcherLiteral(";")
};
public TestBlock() : base(matchers)
{
}
protected override void ProcessMatch(ICommandParserState state)
{
var blockName = GetMatcherTokens(BlockNameToken)[0];
var blockNumber = int.Parse(GetMatcherTokens(BlockNumberToken)[0]);
var blockTypeInput = GetMatcherTokens(BlockTypeToken)[0];
if (!Enum.TryParse(blockTypeInput, true, out FastFileBlock.Type blockType))
{
var blockTypeValues = Enum.GetValues(typeof(FastFileBlock.Type)).OfType<FastFileBlock.Type>()
.Select(type => type.ToString());
throw new TestFailedException($"Unknown fastfile block type '{blockTypeInput}'. Must be one of the following: {string.Join(", ", blockTypeValues)}");
}
var block = new FastFileBlock(blockName, blockNumber, blockType, HasMatcherTokens(DefaultToken));
state.FastFileBlocks.Add(block);
}
}
}

93
src/ZoneCodeGenerator/Parsing/CommandFile/Tests/TestCondition.cs Normal file
View File

@ -0,0 +1,93 @@
using System;
using System.Collections.Generic;
using System.Text;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.CommandFile.Tests
{
class TestCondition : AbstractTokenTest<ICommandParserState>
{
private const string TypeNameToken = "typeName";
private const string ConditionStatementTag = "conditionStatement";
private const string ConditionChainTag = "conditionChain";
private const string ConditionChainLinkTag = "conditionChainLink";
private const string OperationTag = "operation";
private const string OperandTag = "operand";
private static readonly TokenMatcher operand = new MatcherGroupOr(
new MatcherGroupAnd(
new MatcherTypename(),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.ZeroOneMultiple, new MatcherArray())
),
new MatcherNumber(),
new MatcherLiteral("true"),
new MatcherLiteral("false")
).WithTag(OperandTag);
private static readonly TokenMatcher operation = new MatcherGroupOr(
new MatcherLiteral("+"),
new MatcherLiteral("-"),
new MatcherLiteral("*"),
new MatcherLiteral("/"),
new MatcherGroupAnd(new MatcherLiteral("<"), new MatcherLiteral("<")),
new MatcherGroupAnd(new MatcherLiteral(">"), new MatcherLiteral(">"))
).WithTag(OperationTag);
private static readonly TokenMatcher conditionStatement = new MatcherGroupOr(
new MatcherGroupAnd(
new MatcherLiteral("("),
new MatcherWithTag(ConditionStatementTag),
new MatcherLiteral(")")
),
new MatcherGroupAnd(
new MatcherWithTag(OperandTag),
new MatcherGroupOptional(new MatcherGroupAnd(
new MatcherWithTag(OperationTag),
new MatcherWithTag(OperandTag)
))
)
).WithTag(ConditionStatementTag);
private static readonly TokenMatcher conditionChainLink = new MatcherGroupOr(
new MatcherGroupAnd(new MatcherLiteral("="), new MatcherLiteral("=")),
new MatcherGroupAnd(new MatcherLiteral("!"), new MatcherLiteral("=")),
new MatcherGroupAnd(new MatcherLiteral("<"), new MatcherLiteral("=")),
new MatcherGroupAnd(new MatcherLiteral(">"), new MatcherLiteral("=")),
new MatcherLiteral("<"),
new MatcherLiteral(">")
).WithTag(ConditionChainLinkTag);
private static readonly TokenMatcher conditionChain = new MatcherGroupAnd(
new MatcherWithTag(ConditionStatementTag),
new MatcherWithTag(ConditionChainLinkTag),
new MatcherWithTag(ConditionStatementTag)
).WithTag(ConditionChainTag);
private static readonly TokenMatcher[] matchers = {
new MatcherLiteral("set"),
new MatcherLiteral("condition"),
new MatcherTypename().WithName(TypeNameToken),
new MatcherGroupOr(
new MatcherLiteral("never"),
new MatcherWithTag(ConditionChainTag)
),
new MatcherLiteral(";")
};
public TestCondition() : base(matchers)
{
AddTaggedMatcher(conditionChain);
AddTaggedMatcher(conditionChainLink);
AddTaggedMatcher(conditionStatement);
AddTaggedMatcher(operation);
AddTaggedMatcher(operand);
}
protected override void ProcessMatch(ICommandParserState state)
{
}
}
}

87
src/ZoneCodeGenerator/Parsing/CommandFile/Tests/TestCount.cs Normal file
View File

@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.CommandFile.Tests
{
class TestCount : AbstractTokenTest<ICommandParserState>
{
private const string TypeNameToken = "typeName";
private const string ConditionStatementTag = "conditionStatement";
private const string ConditionChainTag = "conditionChain";
private const string ConditionChainLinkTag = "conditionChainLink";
private const string OperationTag = "operation";
private const string OperandTag = "operand";
private static readonly TokenMatcher operand = new MatcherGroupOr(
new MatcherGroupAnd(
new MatcherTypename(),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.ZeroOneMultiple, new MatcherArray())
),
new MatcherNumber()
).WithTag(OperandTag);
private static readonly TokenMatcher operation = new MatcherGroupOr(
new MatcherLiteral("+"),
new MatcherLiteral("-"),
new MatcherLiteral("*"),
new MatcherLiteral("/"),
new MatcherGroupAnd(new MatcherLiteral("<"), new MatcherLiteral("<")),
new MatcherGroupAnd(new MatcherLiteral(">"), new MatcherLiteral(">"))
).WithTag(OperationTag);
private static readonly TokenMatcher conditionStatement = new MatcherGroupOr(
new MatcherGroupAnd(
new MatcherLiteral("("),
new MatcherWithTag(ConditionStatementTag),
new MatcherLiteral(")")
),
new MatcherGroupAnd(
new MatcherWithTag(OperandTag),
new MatcherGroupOptional(new MatcherGroupAnd(
new MatcherWithTag(OperationTag),
new MatcherWithTag(OperandTag)
))
)
).WithTag(ConditionStatementTag);
private static readonly TokenMatcher conditionChainLink = new MatcherGroupOr(
new MatcherGroupAnd(new MatcherLiteral("|"), new MatcherLiteral("|")),
new MatcherGroupAnd(new MatcherLiteral("&"), new MatcherLiteral("&"))
).WithTag(ConditionChainLinkTag);
private static readonly TokenMatcher conditionChain = new MatcherGroupAnd(
new MatcherWithTag(ConditionStatementTag),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.ZeroOneMultiple, new MatcherGroupAnd(
new MatcherWithTag(ConditionChainLinkTag),
new MatcherWithTag(ConditionStatementTag)
))
).WithTag(ConditionChainTag);
private static readonly TokenMatcher[] matchers = {
new MatcherLiteral("set"),
new MatcherLiteral("count"),
new MatcherTypename().WithName(TypeNameToken),
new MatcherWithTag(ConditionChainTag),
new MatcherLiteral(";")
};
public TestCount() : base(matchers)
{
AddTaggedMatcher(conditionStatement);
AddTaggedMatcher(conditionChain);
AddTaggedMatcher(conditionChainLink);
AddTaggedMatcher(operation);
AddTaggedMatcher(operand);
}
protected override void ProcessMatch(ICommandParserState state)
{
}
}
}

37
src/ZoneCodeGenerator/Parsing/CommandFile/Tests/TestGame.cs Normal file
View File

@ -0,0 +1,37 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using ZoneCodeGenerator.Domain.FastFileStructure;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.CommandFile.Tests
{
class TestGame : AbstractTokenTest<ICommandParserState>
{
private const string GameNameToken = "name";
private static readonly TokenMatcher[] matchers = {
new MatcherLiteral("game"),
new MatcherName().WithName(GameNameToken),
new MatcherLiteral(";")
};
public TestGame() : base(matchers)
{
}
protected override void ProcessMatch(ICommandParserState state)
{
if (!string.IsNullOrEmpty(state.Game))
{
throw new TestFailedException($"Game has already been set with value '{state.Game}'");
}
state.Game = GetMatcherTokens(GameNameToken)[0];
}
}
}

33
src/ZoneCodeGenerator/Parsing/CommandFile/Tests/TestReorder.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Text;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.CommandFile.Tests
{
class TestReorder : AbstractTokenTest<ICommandParserState>
{
private const string TypeNameToken = "typeName";
private const string ReorderMemberNameToken = "member";
private static readonly TokenMatcher[] matchers = {
new MatcherLiteral("reorder"),
new MatcherGroupOptional(new MatcherTypename().WithName(TypeNameToken)),
new MatcherLiteral(":"),
new MatcherGroupLoop(MatcherGroupLoop.LoopMode.OneMultiple, new MatcherName().WithName(ReorderMemberNameToken)),
new MatcherLiteral(";")
};
public TestReorder() : base(matchers)
{
}
protected override void ProcessMatch(ICommandParserState state)
{
}
}
}

33
src/ZoneCodeGenerator/Parsing/CommandFile/Tests/TestScriptString.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.CommandFile.Tests
{
class TestScriptString : AbstractTokenTest<ICommandParserState>
{
private const string MemberTypeNameToken = "name";
private static readonly TokenMatcher[] matchers = {
new MatcherLiteral("set"),
new MatcherLiteral("scriptstring"),
new MatcherTypename().WithName(MemberTypeNameToken),
new MatcherLiteral(";")
};
public TestScriptString() : base(matchers)
{
}
protected override void ProcessMatch(ICommandParserState state)
{
}
}
}

41
src/ZoneCodeGenerator/Parsing/CommandFile/Tests/TestUse.cs Normal file
View File

@ -0,0 +1,41 @@
using ZoneCodeGenerator.Domain;
using ZoneCodeGenerator.Parsing.Matching;
using ZoneCodeGenerator.Parsing.Matching.Matchers;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing.CommandFile.Tests
{
class TestUse : AbstractTokenTest<ICommandParserState>
{
private const string TypeNameToken = "typeName";
private static readonly TokenMatcher[] matchers = {
new MatcherLiteral("use"),
new MatcherTypename().WithName(TypeNameToken),
new MatcherLiteral(";")
};
public TestUse() : base(matchers)
{
}
protected override void ProcessMatch(ICommandParserState state)
{
var typeName = GetMatcherTokens(TypeNameToken)[0];
var dataTypeToUse = state.Repository.GetDataTypeByName(typeName);
if (dataTypeToUse == null)
{
throw new LoadingException($"Could not find data type '{typeName}'");
}
if (!(dataTypeToUse is DataTypeWithMembers dataTypeWithMembersToUse))
{
throw new LoadingException($"To use data type '{typeName}' it must either be a struct or a union.");
}
state.DataTypeInUse = dataTypeWithMembersToUse;
}
}
}

7
src/ZoneCodeGenerator/Parsing/ICommentProcessor.cs Normal file
View File

@ -0,0 +1,7 @@
namespace ZoneCodeGenerator.Parsing
{
interface ICommentProcessor
{
string RemoveComments(string line);
}
}

9
src/ZoneCodeGenerator/Parsing/IDataPostProcessor.cs Normal file
View File

@ -0,0 +1,9 @@
using ZoneCodeGenerator.Persistence;
namespace ZoneCodeGenerator.Parsing
{
interface IDataPostProcessor
{
bool PostProcess(IDataRepository repository);
}
}

12
src/ZoneCodeGenerator/Parsing/IIncludingParsingStream.cs Normal file
View File

@ -0,0 +1,12 @@
namespace ZoneCodeGenerator.Parsing
{
interface IIncludingParsingStream : IParsingFileStream
{
/// <summary>
/// Pushes back current file and the position in it to read from another file now instead.
/// After completion returns back to the previous file.
/// </summary>
/// <param name="filename"></param>
void IncludeFile(string filename);
}
}

14
src/ZoneCodeGenerator/Parsing/ILexer.cs Normal file
View File

@ -0,0 +1,14 @@
namespace ZoneCodeGenerator.Parsing
{
interface ILexer
{
int CurrentLine { get; }
string CurrentFile { get; }
bool IsEndOfStream { get; }
int CachedTokenCount { get; }
string NextToken();
string PeekToken(int index = 0);
void SkipTokens(int count);
}
}

10
src/ZoneCodeGenerator/Parsing/IParserState.cs Normal file
View File

@ -0,0 +1,10 @@
using System.Collections.Generic;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing
{
interface IParserState<in TState> where TState : IParserState<TState>
{
IEnumerable<ITokenTest<TState>> GetTests();
}
}

33
src/ZoneCodeGenerator/Parsing/IParsingFileStream.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
namespace ZoneCodeGenerator.Parsing
{
interface IParsingFileStream : IDisposable
{
/// <summary>
/// Whether the end of the stream has been reached.
/// </summary>
bool EndOfStream { get; }
/// <summary>
/// The current filename.
/// </summary>
string Filename { get; }
/// <summary>
/// The current line in the current file.
/// </summary>
int Line { get; }
/// <summary>
/// Reads a line from the current header and returns it.
/// </summary>
/// <returns></returns>
string ReadLine();
/// <summary>
/// Closes this and all underlying streams.
/// </summary>
void Close();
}
}

57
src/ZoneCodeGenerator/Parsing/Impl/CommentProcessor.cs Normal file
View File

@ -0,0 +1,57 @@
using System;
namespace ZoneCodeGenerator.Parsing.Impl
{
class CommentProcessor : ICommentProcessor
{
private const string LineCommentStart = "//";
private const string BlockCommentStart = "/*";
private const string BlockCommentEnd = "*/";
private bool inComment;
public CommentProcessor()
{
inComment = false;
}
public string RemoveComments(string line)
{
if (inComment)
{
var idx = line.IndexOf(BlockCommentEnd, StringComparison.Ordinal);
if(idx == -1)
return "";
line = line.Substring(idx + BlockCommentEnd.Length);
inComment = false;
}
var commentIndex = line.IndexOf(LineCommentStart, StringComparison.Ordinal);
if (commentIndex != -1)
line = line.Remove(commentIndex).Trim();
while (true)
{
var indexBlockCommentBegin = line.IndexOf(BlockCommentStart, StringComparison.Ordinal);
if (indexBlockCommentBegin == -1)
break;
var indexBlockCommentEnd = line.IndexOf(BlockCommentEnd, indexBlockCommentBegin, StringComparison.Ordinal);
if(indexBlockCommentEnd == -1)
{
line = line.Remove(indexBlockCommentBegin).Trim();
inComment = true;
break;
}
line = line.Remove(indexBlockCommentBegin, indexBlockCommentEnd + BlockCommentEnd.Length - indexBlockCommentBegin).Trim();
}
return line;
}
}
}

107
src/ZoneCodeGenerator/Parsing/Impl/IncludingStreamFileSystem.cs Normal file
View File

@ -0,0 +1,107 @@
using System.Collections.Generic;
using System.IO;
namespace ZoneCodeGenerator.Parsing.Impl
{
class IncludingStreamFileSystem : IIncludingParsingStream
{
private class HeaderStreamState
{
private readonly StreamReader reader;
private readonly Stream stream;
public string Cwd { get; }
public string Filename { get; }
public int Line { get; private set; }
public HeaderStreamState(FileStream stream, string filepath)
{
this.stream = stream;
reader = new StreamReader(stream);
Cwd = Path.GetDirectoryName(filepath);
Filename = Path.GetFileName(filepath);
Line = 1;
}
public bool EndOfStream => reader.EndOfStream;
public string ReadLine()
{
Line++;
return reader.ReadLine();
}
public void Close()
{
reader.Close();
stream.Close();
}
}
private readonly Stack<HeaderStreamState> states;
public IncludingStreamFileSystem(string path)
{
states = new Stack<HeaderStreamState>();
var initialState = new HeaderStreamState(new FileStream(path, FileMode.Open), path);
states.Push(initialState);
}
public bool EndOfStream => states.Count == 0;
public string Filename => states.Count == 0 ? "" : State.Filename;
public int Line => states.Count == 0 ? -1 : State.Line;
private HeaderStreamState State => states.Count == 0 ? null : states.Peek();
public string ReadLine()
{
var state = State;
if (state == null)
return "";
var line = state.ReadLine();
if (state.EndOfStream)
{
state.Close();
states.Pop();
}
return line;
}
public void IncludeFile(string filename)
{
var state = State;
if (state == null)
return;
var path = Path.Combine(state.Cwd, filename);
if (!File.Exists(path)) return;
var newState = new HeaderStreamState(new FileStream(path, FileMode.Open), path);
states.Push(newState);
}
public void Close()
{
while (states.Count > 0)
{
states.Pop().Close();
}
}
public void Dispose()
{
Close();
}
}
}

160
src/ZoneCodeGenerator/Parsing/Impl/Lexer.cs Normal file
View File

@ -0,0 +1,160 @@
using System.Collections.Generic;
using System.Linq;
namespace ZoneCodeGenerator.Parsing.Impl
{
class Lexer : ILexer
{
private class LexerToken
{
public string Token { get; set; }
public int Row { get; set; }
public string Filename { get; set; }
}
private static readonly char[] charTokens = {
';',
':',
'{',
'}',
',',
'=',
'(',
')',
'*',
'[',
']',
'<',
'>',
'#',
'&',
'%',
'\"',
'?',
'!',
'+',
'-',
'/',
'\\'
};
private readonly IParsingFileStream fileStream;
private readonly List<LexerToken> tokenCache;
public bool IsEndOfStream => fileStream.EndOfStream && tokenCache.Count == 0;
public int CachedTokenCount => tokenCache.Count;
public Lexer(IParsingFileStream fileStream)
{
this.fileStream = fileStream;
tokenCache = new List<LexerToken>();
}
public int CurrentLine => tokenCache.Count == 0 ? fileStream.Line : tokenCache[0].Row;
public string CurrentFile => tokenCache.Count == 0 ? fileStream.Filename : tokenCache[0].Filename;
private static bool IsCharToken(char c)
{
return charTokens.Contains(c);
}
private void TokenizeLine(string filename, int lineNr, string line)
{
line = line.Trim();
while(line.Length > 0)
{
var token = "";
while (line.Length > 0)
{
var c = line[0];
line = line.Substring(1);
if (char.IsWhiteSpace(c))
break;
if (IsCharToken(c))
{
if(token == "")
{
token += c;
break;
}
else
{
line = c + line;
break;
}
}
token += c;
}
tokenCache.Add(new LexerToken()
{
Token = token,
Row = lineNr,
Filename = filename
});
line = line.TrimStart();
}
}
public string NextToken()
{
while(tokenCache.Count == 0)
{
if (fileStream.EndOfStream)
{
return "";
}
TokenizeLine(fileStream.Filename, fileStream.Line, fileStream.ReadLine());
}
var token = tokenCache[0].Token;
tokenCache.RemoveAt(0);
return token;
}
public string PeekToken(int index = 0)
{
if (index < 0)
return "";
while (tokenCache.Count <= index)
{
if (fileStream.EndOfStream)
{
return "";
}
TokenizeLine(fileStream.Filename, fileStream.Line, fileStream.ReadLine());
}
return tokenCache[index].Token;
}
public void SkipTokens(int count)
{
if (count < 0)
return;
while (tokenCache.Count < count)
{
if (fileStream.EndOfStream)
{
tokenCache.Clear();
return;
}
TokenizeLine(fileStream.Filename, fileStream.Line, fileStream.ReadLine());
}
tokenCache.RemoveRange(0, count);
}
}
}

62
src/ZoneCodeGenerator/Parsing/Matching/BaseMatcher.cs Normal file
View File

@ -0,0 +1,62 @@
namespace ZoneCodeGenerator.Parsing.Matching
{
abstract class BaseMatcher : TokenMatcher
{
protected class MatchingResult
{
public bool Successful { get; }
public int TokensConsumed { get; }
public MatchingResult(bool successful, int tokensConsumed)
{
Successful = successful;
TokensConsumed = tokensConsumed;
}
}
private string name;
private string output;
private bool consuming;
protected BaseMatcher()
{
name = null;
consuming = true;
output = "";
}
protected abstract MatchingResult Matches(MatchingContext context, int tokenOffset);
protected void SetMatcherOutput(string text)
{
output = text;
}
public BaseMatcher WithName(string tokenName)
{
name = tokenName;
return this;
}
public BaseMatcher NonConsuming()
{
consuming = false;
return this;
}
protected override TokenMatchingResult PerformTest(MatchingContext context, int tokenOffset)
{
var match = Matches(context, tokenOffset);
var success = match.Successful;
var consumedTokens = success && consuming ? match.TokensConsumed : 0;
var result = new TokenMatchingResult(success, consumedTokens);
if (name != null && success)
result.AddNamedMatch(name, output);
return result;
}
}
}

17
src/ZoneCodeGenerator/Parsing/Matching/GroupMatcher.cs Normal file
View File

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace ZoneCodeGenerator.Parsing.Matching
{
abstract class GroupMatcher : TokenMatcher
{
public TokenMatcher[] Matchers { get; }
protected GroupMatcher(TokenMatcher[] matchers)
{
Matchers = matchers;
}
protected abstract override TokenMatchingResult PerformTest(MatchingContext context, int tokenOffset);
}
}

47
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherArray.cs Normal file
View File

@ -0,0 +1,47 @@
using System.Globalization;
using System.Text.RegularExpressions;
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherArray : BaseMatcher
{
private static readonly Regex decimalNumberRegex = new Regex(@"^[0-9]+$");
private static readonly Regex hexNumberRegex = new Regex(@"^0x([0-9a-fA-F]+)$");
private static readonly Regex nameRegex = new Regex(@"^[a-zA-Z_$][a-zA-Z0-9_$]*$");
protected override MatchingResult Matches(MatchingContext context, int tokenOffset)
{
var lexer = context.Lexer;
if (lexer.PeekToken(tokenOffset++) != "[")
return new MatchingResult(false, 0);
var token = lexer.PeekToken(tokenOffset++);
if (decimalNumberRegex.IsMatch(token))
{
SetMatcherOutput(token);
}
else if (hexNumberRegex.IsMatch(token))
{
SetMatcherOutput(int.Parse(hexNumberRegex.Match(token).Groups[1].Value, NumberStyles.HexNumber).ToString());
}
else if (nameRegex.IsMatch(token))
{
SetMatcherOutput(token);
}
else
{
return new MatchingResult(false, 0);
}
return lexer.PeekToken(tokenOffset) != "]" ? new MatchingResult(false, 0)
: new MatchingResult(true, 3);
}
protected override string GetIdentifier()
{
return "Array";
}
}
}

40
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherGroupAnd.cs Normal file
View File

@ -0,0 +1,40 @@
using System;
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherGroupAnd : GroupMatcher
{
public MatcherGroupAnd(params TokenMatcher[] matchers) : base(matchers)
{
if(matchers.Length == 0)
throw new ArgumentException("A matcher group is supposed to have matchers.");
}
protected override TokenMatchingResult PerformTest(MatchingContext context, int tokenOffset)
{
var result = new TokenMatchingResult(true, 0);
foreach(var matcher in Matchers)
{
var matcherResult = matcher.Test(context, tokenOffset + result.ConsumedTokenCount);
if (matcherResult.Successful)
{
matcherResult.CopyNamedMatchesTo(result);
result.ConsumedTokenCount += matcherResult.ConsumedTokenCount;
}
else
{
return new TokenMatchingResult(false, 0);
}
}
return result;
}
protected override string GetIdentifier()
{
return "GroupAnd";
}
}
}

68
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherGroupLoop.cs Normal file
View File

@ -0,0 +1,68 @@
using System;
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherGroupLoop : GroupMatcher
{
public enum LoopMode
{
ZeroOneMultiple,
OneMultiple,
Multiple
}
private readonly TokenMatcher matcher;
private readonly LoopMode mode;
public MatcherGroupLoop(LoopMode mode, TokenMatcher matcher) : base(new []{matcher})
{
this.mode = mode;
this.matcher = matcher;
}
protected override TokenMatchingResult PerformTest(MatchingContext context, int tokenOffset)
{
var result = new TokenMatchingResult(false, 0);
var matchedTimes = 0;
while (true)
{
var matcherResult = matcher.Test(context, tokenOffset + result.ConsumedTokenCount);
if (!matcherResult.Successful)
break;
matcherResult.CopyNamedMatchesTo(result);
result.ConsumedTokenCount += matcherResult.ConsumedTokenCount;
matchedTimes++;
}
switch (mode)
{
case LoopMode.ZeroOneMultiple:
result.Successful = true;
break;
case LoopMode.OneMultiple:
result.Successful = matchedTimes > 0;
break;
case LoopMode.Multiple:
result.Successful = matchedTimes > 1;
break;
default:
throw new ArgumentOutOfRangeException();
}
if (!result.Successful)
result.ConsumedTokenCount = 0;
return result;
}
protected override string GetIdentifier()
{
return "GroupLoop";
}
}
}

24
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherGroupOptional.cs Normal file
View File

@ -0,0 +1,24 @@
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherGroupOptional : GroupMatcher
{
private readonly TokenMatcher matcher;
public MatcherGroupOptional(TokenMatcher matcher) : base(new []{matcher})
{
this.matcher = matcher;
}
protected override TokenMatchingResult PerformTest(MatchingContext context, int tokenOffset)
{
var result = matcher.Test(context, tokenOffset);
return !result.Successful ? new TokenMatchingResult(true, 0) : result;
}
protected override string GetIdentifier()
{
return "GroupOptional";
}
}
}

35
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherGroupOr.cs Normal file
View File

@ -0,0 +1,35 @@
using System;
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherGroupOr : GroupMatcher
{
private readonly TokenMatcher[] matchers;
public MatcherGroupOr(params TokenMatcher[] matchers) : base(matchers)
{
if(matchers.Length == 0)
throw new ArgumentException("A matcher group is supposed to have matchers.");
this.matchers = matchers;
}
protected override TokenMatchingResult PerformTest(MatchingContext context, int tokenOffset)
{
foreach(var matcher in matchers)
{
var matcherResult = matcher.Test(context, tokenOffset);
if (matcherResult.Successful)
return matcherResult;
}
return new TokenMatchingResult(false, 0);
}
protected override string GetIdentifier()
{
return "GroupOr";
}
}
}

28
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherLiteral.cs Normal file
View File

@ -0,0 +1,28 @@
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherLiteral : BaseMatcher
{
private readonly string literal;
public MatcherLiteral(string literal)
{
this.literal = literal;
}
protected override MatchingResult Matches(MatchingContext context, int tokenOffset)
{
var token = context.Lexer.PeekToken(tokenOffset);
var isMatch = string.Equals(token, literal);
if (!isMatch) return new MatchingResult(false, 0);
SetMatcherOutput(literal);
return new MatchingResult(true, 1);
}
protected override string GetIdentifier()
{
return $"Literal(\"{literal}\")";
}
}
}

98
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherName.cs Normal file
View File

@ -0,0 +1,98 @@
using System.Linq;
using System.Text.RegularExpressions;
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherName : BaseMatcher
{
private static readonly Regex nameRegex = new Regex(@"^[a-zA-Z_$][a-zA-Z0-9_$]*$");
private static readonly string[] reservedWords = {
"const",
"int",
"char",
"short",
"long",
"unsigned",
"typedef",
"void",
"static",
"class",
"struct",
"enum",
"union",
"namespace",
"using",
"public",
"protected",
"private",
"alignas",
"and",
"auto",
"bool",
"catch",
"concept",
"default",
"double",
"float",
"extern",
"for",
"if",
"mutable",
"noexcept",
"nullptr",
"switch",
"this",
"true",
"false",
"typeid",
"while",
"alignof",
"break",
"continue",
"dynamic_cast",
"static_cast",
"explicit",
"friend",
"not",
"operator",
"return",
"sizeof",
"thread_local",
"try",
"typename",
"volatile",
"asm",
"case",
"constexpr",
"decltype",
"do",
"else",
"export",
"goto",
"inline",
"new",
"reinterpret_cast",
"static",
"template",
"throw",
"virtual"
};
protected override MatchingResult Matches(MatchingContext context, int tokenOffset)
{
var token = context.Lexer.PeekToken(tokenOffset);
var isMatch = nameRegex.IsMatch(token) && !reservedWords.Contains(token);
if (!isMatch) return new MatchingResult(false, 0);
SetMatcherOutput(token);
return new MatchingResult(true, 1);
}
protected override string GetIdentifier()
{
return "Name";
}
}
}

36
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherNumber.cs Normal file
View File

@ -0,0 +1,36 @@
using System.Globalization;
using System.Text.RegularExpressions;
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherNumber : BaseMatcher
{
private static readonly Regex decimalNumber = new Regex(@"^\d+$");
private static readonly Regex hexNumber = new Regex(@"^0x([0-9a-fA-F]+)$");
protected override MatchingResult Matches(MatchingContext context, int tokenOffset)
{
var token = context.Lexer.PeekToken(tokenOffset);
var currentMatch = decimalNumber.Match(token);
if (currentMatch.Success)
{
SetMatcherOutput(token);
return new MatchingResult(true, 1);
}
currentMatch = hexNumber.Match(token);
if (!currentMatch.Success) return new MatchingResult(false, 0);
SetMatcherOutput(int.Parse( currentMatch.Groups[1].Value, NumberStyles.HexNumber).ToString());
return new MatchingResult(true, 1);
}
protected override string GetIdentifier()
{
return "Number";
}
}
}

80
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherTypename.cs Normal file
View File

@ -0,0 +1,80 @@
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherTypename : BaseMatcher
{
private static readonly Regex nameRegex = new Regex(@"^[a-zA-Z_$][a-zA-Z0-9_$]*$");
private static readonly string[] typenamePrefixes = {
"const",
"signed",
"unsigned"
};
private int MatchTypenameExtension(MatchingContext context, int tokenOffset)
{
if (!":".Equals(context.Lexer.PeekToken(tokenOffset++)))
return 0;
if (!":".Equals(context.Lexer.PeekToken(tokenOffset++)))
return 0;
return nameRegex.IsMatch(context.Lexer.PeekToken(tokenOffset)) ? 3 : 0;
}
protected override MatchingResult Matches(MatchingContext context, int tokenOffset)
{
var lexer = context.Lexer;
var typename = new StringBuilder();
var startTokenOffset = tokenOffset;
var currentPart = lexer.PeekToken(tokenOffset++);
if (!nameRegex.IsMatch(currentPart))
return new MatchingResult(false, 0);
typename.Append(currentPart);
while (typenamePrefixes.Contains(currentPart))
{
currentPart = lexer.PeekToken(tokenOffset++);
if (!nameRegex.IsMatch(currentPart))
return new MatchingResult(false, 0);
typename.Append(' ');
typename.Append(currentPart);
}
if ("long".Equals(currentPart))
{
currentPart = lexer.PeekToken(tokenOffset);
if (currentPart.Equals("long"))
{
tokenOffset++;
typename.Append(' ');
typename.Append(currentPart);
}
}
var extensionLength = MatchTypenameExtension(context, tokenOffset);
while (extensionLength > 0)
{
for (var i = 0; i < extensionLength; i++)
{
typename.Append(lexer.PeekToken(tokenOffset + i));
}
tokenOffset += extensionLength;
extensionLength = MatchTypenameExtension(context, tokenOffset);
}
SetMatcherOutput(typename.ToString());
return new MatchingResult(true, tokenOffset - startTokenOffset);
}
protected override string GetIdentifier()
{
return "Typename";
}
}
}

33
src/ZoneCodeGenerator/Parsing/Matching/Matchers/MatcherWithTag.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace ZoneCodeGenerator.Parsing.Matching.Matchers
{
class MatcherWithTag : TokenMatcher
{
private readonly string tag;
public MatcherWithTag(string tag)
{
this.tag = tag;
}
protected override TokenMatchingResult PerformTest(MatchingContext context, int tokenOffset)
{
var matcherWithTag = context.GetMatcherByTag(tag);
if (matcherWithTag == null)
{
throw new Exception($"Could not find matcher with tag '{tag}'");
}
return matcherWithTag.Test(context, tokenOffset);
}
protected override string GetIdentifier()
{
return $"WithTag(\"{tag}\")";
}
}
}

33
src/ZoneCodeGenerator/Parsing/Matching/MatchingContext.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace ZoneCodeGenerator.Parsing.Matching
{
class MatchingContext
{
public ILexer Lexer { get; }
public bool Verbose { get; set; }
private readonly Dictionary<string, TokenMatcher> taggedMatchers;
public MatchingContext(ILexer lexer, Dictionary<string, TokenMatcher> taggedMatchers)
{
Lexer = lexer;
this.taggedMatchers = taggedMatchers;
#if DEBUG
Verbose = true;
#else
Verbose = false;
#endif
}
public TokenMatcher GetMatcherByTag(string tag)
{
return taggedMatchers.ContainsKey(tag) ? taggedMatchers[tag] : null;
}
}
}

66
src/ZoneCodeGenerator/Parsing/Matching/TokenMatcher.cs Normal file
View File

@ -0,0 +1,66 @@
using System;
using System.Collections.Generic;
namespace ZoneCodeGenerator.Parsing.Matching
{
abstract class TokenMatcher
{
private const int DebugPeekMin = 3;
public string Tag { get; private set; }
protected TokenMatcher()
{
Tag = "";
}
public TokenMatcher WithTag(string tag)
{
Tag = tag;
return this;
}
protected abstract string GetIdentifier();
public override string ToString()
{
return string.IsNullOrEmpty(Tag) ? GetIdentifier() : Tag;
}
protected abstract TokenMatchingResult PerformTest(MatchingContext context, int tokenOffset);
public virtual TokenMatchingResult Test(MatchingContext context, int tokenOffset)
{
var result = PerformTest(context, tokenOffset);
if (context.Verbose)
{
if (result.Successful)
{
var matchedTokens = new List<string>();
for (var i = 0; i < result.ConsumedTokenCount; i++)
{
matchedTokens.Add(context.Lexer.PeekToken(tokenOffset + i));
}
Console.WriteLine($"Matcher {ToString()} matched: {string.Join(" ", matchedTokens)}");
}
else
{
var debugPeekCount = Math.Max(context.Lexer.CachedTokenCount - tokenOffset, DebugPeekMin);
var nextTokens = new List<string>();
for (var i = 0; i < debugPeekCount; i++)
{
nextTokens.Add(context.Lexer.PeekToken(tokenOffset + i));
}
Console.WriteLine($"Matcher {ToString()} failed @ {string.Join(" ", nextTokens)}");
}
}
return result;
}
}
}

59
src/ZoneCodeGenerator/Parsing/Matching/TokenMatchingResult.cs Normal file
View File

@ -0,0 +1,59 @@
using System;
using System.Collections;
using System.Collections.Generic;
namespace ZoneCodeGenerator.Parsing.Matching
{
class TokenMatchingResult : IEnumerable<KeyValuePair<string, List<string>>>
{
public bool Successful { get; set; }
public int ConsumedTokenCount { get; set; }
private readonly Dictionary<string, List<string>> namedMatches;
public TokenMatchingResult(bool success, int consumedTokenCount)
{
if(!success && consumedTokenCount > 0)
throw new ArgumentException("Token Match cannot be unsuccessful and consume tokens");
if(consumedTokenCount < 0)
throw new ArgumentException("Consumed token count cannot be negative");
Successful = success;
ConsumedTokenCount = consumedTokenCount;
namedMatches = new Dictionary<string, List<string>>();
}
public void AddNamedMatch(string name, string value)
{
if(!namedMatches.ContainsKey(name))
namedMatches.Add(name, new List<string>());
namedMatches[name].Add(value);
}
public IEnumerator<KeyValuePair<string, List<string>>> GetEnumerator()
{
return namedMatches.GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return namedMatches.GetEnumerator();
}
public List<string> this[string key] => namedMatches.ContainsKey(key) ? namedMatches[key] : new List<string>();
public void CopyNamedMatchesTo(TokenMatchingResult target)
{
foreach(var namedMatchKey in namedMatches)
{
foreach (var namedMatch in namedMatchKey.Value)
{
target.AddNamedMatch(namedMatchKey.Key, namedMatch);
}
}
}
}
}

85
src/ZoneCodeGenerator/Parsing/Parser.cs Normal file
View File

@ -0,0 +1,85 @@
using System;
using ZoneCodeGenerator.Parsing.Testing;
namespace ZoneCodeGenerator.Parsing
{
class Parser<TState> where TState : IParserState<TState>
{
// Amount of consecutive tokens that are output after a parser error to show the region where the error happened.
private const int ParserErrorTokenPeekCount = 3;
private readonly ILexer lexer;
private readonly TState state;
public Parser(TState state, ILexer lexer)
{
this.state = state;
this.lexer = lexer;
}
public bool Parse()
{
while (!lexer.IsEndOfStream)
{
var testSuccessful = false;
try
{
foreach (var test in state.GetTests())
{
switch (test.PerformTest(state, lexer))
{
case TokenTestResult.Match:
lexer.SkipTokens(test.ConsumedTokenCount);
testSuccessful = true;
break;
case TokenTestResult.NoMatch:
break;
default:
throw new ArgumentOutOfRangeException();
}
if (testSuccessful)
break;
}
}
catch (LoadingException e)
{
PrintParserError(e);
return false;
}
if (!testSuccessful)
{
PrintParserError(new LoadingException("Could not parse expression"));
return false;
}
// Peek for the next token here to be able to determine if the stream is at its end or not
lexer.PeekToken();
}
return true;
}
private void PrintParserError(LoadingException e)
{
Console.WriteLine($"Parsing failed in file '{lexer.CurrentFile}' at line {lexer.CurrentLine}:");
Console.WriteLine(e.Message);
Console.Write(" ");
var tokenCountToShow = Math.Max(ParserErrorTokenPeekCount, lexer.CachedTokenCount);
for (var i = 0; i < tokenCountToShow; i++)
{
var token = lexer.PeekToken(i);
if (string.IsNullOrEmpty(token))
break;
Console.Write($"{token} ");
}
Console.WriteLine();
Console.WriteLine();
}
}
}

9
src/ZoneCodeGenerator/Parsing/ParserException.cs Normal file
View File

@ -0,0 +1,9 @@
namespace ZoneCodeGenerator.Parsing
{
class ParserException : LoadingException
{
public ParserException(string message) : base(message)
{
}
}
}

109
src/ZoneCodeGenerator/Parsing/Testing/AbstractTokenTest.cs Normal file
View File

@ -0,0 +1,109 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using ZoneCodeGenerator.Parsing.Matching;
namespace ZoneCodeGenerator.Parsing.Testing
{
abstract class AbstractTokenTest<TState> : ITokenTest<TState> where TState : IParserState<TState>
{
public int ConsumedTokenCount { get; private set; }
private readonly TokenMatcher[] matcherEntryPoint;
private readonly Dictionary<string, List<string>> matchedEntries;
private readonly Dictionary<string, TokenMatcher> taggedMatchers;
private bool tested;
protected AbstractTokenTest(TokenMatcher[] matcherEntryPoint)
{
this.matcherEntryPoint = matcherEntryPoint;
matchedEntries = new Dictionary<string, List<string>>();
taggedMatchers = new Dictionary<string, TokenMatcher>();
tested = false;
BuildTaggedMatcherList(matcherEntryPoint);
}
private void BuildTaggedMatcherList(IEnumerable<TokenMatcher> matchersWithTags)
{
foreach (var tokenMatcher in matchersWithTags)
{
if (!string.IsNullOrEmpty(tokenMatcher.Tag))
{
taggedMatchers[tokenMatcher.Tag] = tokenMatcher;
}
if (tokenMatcher is GroupMatcher groupMatcher)
{
BuildTaggedMatcherList(groupMatcher.Matchers);
}
}
}
protected void AddTaggedMatcher(params TokenMatcher[] taggedMatcher)
{
BuildTaggedMatcherList(taggedMatcher);
}
protected ReadOnlyCollection<string> GetMatcherTokens(string matcherName)
{
return tested && matchedEntries.ContainsKey(matcherName) ? matchedEntries[matcherName].AsReadOnly() : new List<string>().AsReadOnly();
}
protected bool HasMatcherTokens(string matcherName)
{
return tested && matchedEntries.ContainsKey(matcherName);
}
protected abstract void ProcessMatch(TState state);
public TokenTestResult PerformTest(TState state, ILexer lexer)
{
var tokenOffset = 0;
matchedEntries.Clear();
ConsumedTokenCount = 0;
tested = true;
var context = new MatchingContext(lexer, taggedMatchers);
if (context.Verbose)
{
Console.WriteLine($"Test {GetType().Name} start");
}
foreach(var matcher in matcherEntryPoint)
{
var result = matcher.Test(context, tokenOffset);
if (!result.Successful)
{
if (context.Verbose)
{
Console.WriteLine($"Test {GetType().Name} failed");
}
return TokenTestResult.NoMatch;
}
tokenOffset += result.ConsumedTokenCount;
foreach (var entry in result)
{
matchedEntries.Add(entry.Key, entry.Value);
}
}
ConsumedTokenCount = tokenOffset;
ProcessMatch(state);
if (context.Verbose)
{
Console.WriteLine($"Test {GetType().Name} successful");
}
return TokenTestResult.Match;
}
}
}

9
src/ZoneCodeGenerator/Parsing/Testing/ITokenTest.cs Normal file
View File

@ -0,0 +1,9 @@
namespace ZoneCodeGenerator.Parsing.Testing
{
interface ITokenTest<in TState> where TState : IParserState<TState>
{
int ConsumedTokenCount { get; }
TokenTestResult PerformTest(TState state, ILexer lexer);
}
}

9
src/ZoneCodeGenerator/Parsing/Testing/TestFailedException.cs Normal file
View File

@ -0,0 +1,9 @@
namespace ZoneCodeGenerator.Parsing.Testing
{
class TestFailedException : LoadingException
{
public TestFailedException(string message) : base(message)
{
}
}
}

8
src/ZoneCodeGenerator/Parsing/Testing/TokenTestResult.cs Normal file
View File

@ -0,0 +1,8 @@
namespace ZoneCodeGenerator.Parsing.Testing
{
enum TokenTestResult
{
Match,
NoMatch
}
}