2
0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-06-26 06:11:53 +00:00

chore: parse includes and assetlists while parsing zone definition

This commit is contained in:
Jan
2025-01-01 18:13:23 +01:00
parent 9852f52a15
commit aa212e0958
74 changed files with 530 additions and 437 deletions

View File

@ -1,19 +0,0 @@
#include "SequenceZoneDefinitionAssetList.h"
#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
SequenceZoneDefinitionAssetList::SequenceZoneDefinitionAssetList()
{
const ZoneDefinitionMatcherFactory create(this);
AddMatchers({
create.Keyword("assetlist"),
create.Char(','),
create.Field().Capture(CAPTURE_ASSET_LIST_NAME),
});
}
void SequenceZoneDefinitionAssetList::ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const
{
state->m_definition->m_asset_lists.emplace_back(result.NextCapture(CAPTURE_ASSET_LIST_NAME).FieldValue());
}

View File

@ -1,19 +0,0 @@
#include "SequenceZoneDefinitionInclude.h"
#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
SequenceZoneDefinitionInclude::SequenceZoneDefinitionInclude()
{
const ZoneDefinitionMatcherFactory create(this);
AddMatchers({
create.Keyword("include"),
create.Char(','),
create.Field().Capture(CAPTURE_INCLUDE_NAME),
});
}
void SequenceZoneDefinitionInclude::ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const
{
state->m_definition->m_includes.emplace_back(result.NextCapture(CAPTURE_INCLUDE_NAME).FieldValue());
}

View File

@ -0,0 +1,14 @@
#include "AssetListOutputStream.h"
AssetListOutputStream::AssetListOutputStream(std::ostream& stream, const GameId game)
: m_stream(stream),
m_asset_name_resolver(IAssetNameResolver::GetResolverForGame(game))
{
}
void AssetListOutputStream::WriteEntry(const AssetListEntry& entry)
{
m_stream.WriteColumn(*m_asset_name_resolver->GetAssetTypeName(entry.m_type));
m_stream.WriteColumn(entry.m_name);
m_stream.NextRow();
}

View File

@ -1,23 +1,12 @@
#pragma once
#include "AssetList.h"
#include "Csv/CsvStream.h"
#include "Game/IGame.h"
#include "Zone/AssetNameResolver.h"
#include <Zone/AssetNameResolver.h>
#include <iostream>
class AssetListInputStream
{
public:
AssetListInputStream(std::istream& stream, GameId game);
bool NextEntry(AssetListEntry& entry, bool* failure) const;
private:
CsvInputStream m_stream;
const IAssetNameResolver* m_asset_name_resolver;
};
class AssetListOutputStream
{
public:

View File

@ -0,0 +1,96 @@
#include "AssetListReader.h"
#include "Csv/CsvStream.h"
#include "Game/IGame.h"
#include "Zone/AssetNameResolver.h"
#include <format>
namespace
{
class AssetListInputStream
{
public:
AssetListInputStream(std::istream& stream, GameId game)
: m_stream(stream),
m_asset_name_resolver(IAssetNameResolver::GetResolverForGame(game))
{
}
bool NextEntry(AssetListEntry& entry, bool* failure) const
{
std::vector<std::string> row;
if (failure)
*failure = false;
while (true)
{
if (!m_stream.NextRow(row))
return false;
if (row.empty() || (row.size() == 1 && row[0].empty()))
continue;
const auto& typeName = row[0];
const auto maybeType = m_asset_name_resolver->GetAssetTypeByName(typeName);
if (!maybeType)
{
std::cerr << std::format("Unknown asset type name \"{}\"\n", typeName);
if (failure)
*failure = true;
return false;
}
entry.m_type = *maybeType;
if (row.size() >= 3 && row[1].empty())
{
entry.m_name = row[2];
entry.m_is_reference = true;
}
else
{
entry.m_name = row[1];
entry.m_is_reference = false;
}
return true;
}
}
private:
CsvInputStream m_stream;
const IAssetNameResolver* m_asset_name_resolver;
};
} // namespace
AssetListReader::AssetListReader(ISearchPath& searchPath, const GameId game)
: m_search_path(searchPath),
m_game(game)
{
}
std::optional<AssetList> AssetListReader::ReadAssetList(const std::string& zoneName, const bool logMissing) const
{
const auto assetListFileName = std::format("assetlist/{}.csv", zoneName);
const auto assetListStream = m_search_path.Open(assetListFileName);
if (assetListStream.IsOpen())
{
AssetList assetList;
const AssetListInputStream stream(*assetListStream.m_stream, m_game);
AssetListEntry entry;
bool failure;
while (stream.NextEntry(entry, &failure))
{
assetList.m_entries.emplace_back(std::move(entry));
}
if (!failure)
return assetList;
}
else if (logMissing)
std::cerr << std::format("Failed to open file for assetlist: {}\n", assetListFileName);
return std::nullopt;
}

View File

@ -0,0 +1,20 @@
#pragma once
#include "AssetList.h"
#include "Game/IGame.h"
#include "SearchPath/ISearchPath.h"
#include <optional>
#include <string>
class AssetListReader
{
public:
AssetListReader(ISearchPath& searchPath, GameId game);
std::optional<AssetList> ReadAssetList(const std::string& zoneName, bool logMissing = true) const;
private:
ISearchPath& m_search_path;
GameId m_game;
};

View File

@ -1,62 +0,0 @@
#include "AssetListStream.h"
#include <format>
AssetListInputStream::AssetListInputStream(std::istream& stream, const GameId game)
: m_stream(stream),
m_asset_name_resolver(IAssetNameResolver::GetResolverForGame(game))
{
}
bool AssetListInputStream::NextEntry(AssetListEntry& entry, bool* failure) const
{
std::vector<std::string> row;
if (failure)
*failure = false;
while (true)
{
if (!m_stream.NextRow(row))
return false;
if (row.empty())
continue;
const auto& typeName = row[0];
const auto maybeType = m_asset_name_resolver->GetAssetTypeByName(typeName);
if (!maybeType)
{
std::cerr << std::format("Unknown asset type name \"{}\"\n", typeName);
if (failure)
*failure = true;
return false;
}
entry.m_type = *maybeType;
if (row.size() >= 3 && row[1].empty())
{
entry.m_name = row[2];
entry.m_is_reference = true;
}
else
{
entry.m_name = row[1];
entry.m_is_reference = false;
}
return true;
}
}
AssetListOutputStream::AssetListOutputStream(std::ostream& stream, const GameId game)
: m_stream(stream),
m_asset_name_resolver(IAssetNameResolver::GetResolverForGame(game))
{
}
void AssetListOutputStream::WriteEntry(const AssetListEntry& entry)
{
m_stream.WriteColumn(*m_asset_name_resolver->GetAssetTypeName(entry.m_type));
m_stream.WriteColumn(entry.m_name);
m_stream.NextRow();
}

View File

@ -1,7 +1,7 @@
#pragma once
#include "Parsing/Matcher/AbstractMatcher.h"
#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParserValue.h"
class ZoneDefinitionMatcherCharacter final : public AbstractMatcher<ZoneDefinitionParserValue>
{

View File

@ -1,7 +1,7 @@
#pragma once
#include "Parsing/Matcher/AbstractMatcherFactory.h"
#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParserValue.h"
#include <string>

View File

@ -1,7 +1,7 @@
#pragma once
#include "Parsing/Matcher/AbstractMatcher.h"
#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParserValue.h"
#include <string>

View File

@ -1,7 +1,7 @@
#pragma once
#include "Parsing/Matcher/AbstractMatcher.h"
#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParserValue.h"
class ZoneDefinitionMatcherValueType final : public AbstractMatcher<ZoneDefinitionParserValue>
{

View File

@ -0,0 +1,35 @@
#include "SequenceZoneDefinitionAssetList.h"
#include "Zone/AssetList/AssetListReader.h"
#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
SequenceZoneDefinitionAssetList::SequenceZoneDefinitionAssetList()
{
const ZoneDefinitionMatcherFactory create(this);
AddMatchers({
create.Keyword("assetlist").Capture(CAPTURE_ASSET_LIST_KEYWORD),
create.Char(','),
create.Field().Capture(CAPTURE_ASSET_LIST_NAME),
});
}
void SequenceZoneDefinitionAssetList::ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const
{
if (state->m_definition->m_game == GameId::COUNT)
{
const auto& assetListKeywordToken = result.NextCapture(CAPTURE_ASSET_LIST_KEYWORD);
throw ParsingException(assetListKeywordToken.GetPos(), "Must define game before using assetlist");
}
const auto& assetListNameToken = result.NextCapture(CAPTURE_ASSET_LIST_NAME);
AssetListReader assetListReader(state->m_search_path, state->m_definition->m_game);
const auto maybeAssetList = assetListReader.ReadAssetList(assetListNameToken.FieldValue());
if (!maybeAssetList)
throw ParsingException(assetListNameToken.GetPos(), "Failed to read asset list");
for (auto& assetListEntry : maybeAssetList->m_entries)
state->m_definition->m_assets.emplace_back(assetListEntry.m_type, std::move(assetListEntry.m_name), assetListEntry.m_is_reference);
}

View File

@ -1,10 +1,11 @@
#pragma once
#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
class SequenceZoneDefinitionAssetList final : public ZoneDefinitionParser::sequence_t
{
static constexpr auto CAPTURE_ASSET_LIST_NAME = 1;
static constexpr auto CAPTURE_ASSET_LIST_KEYWORD = 1;
static constexpr auto CAPTURE_ASSET_LIST_NAME = 2;
protected:
void ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const override;

View File

@ -1,6 +1,6 @@
#include "SequenceZoneDefinitionBuild.h"
#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
SequenceZoneDefinitionBuild::SequenceZoneDefinitionBuild()
{

View File

@ -1,6 +1,6 @@
#pragma once
#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
class SequenceZoneDefinitionBuild final : public ZoneDefinitionParser::sequence_t
{

View File

@ -1,6 +1,6 @@
#include "SequenceZoneDefinitionEntry.h"
#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
SequenceZoneDefinitionEntry::SequenceZoneDefinitionEntry()
{

View File

@ -1,6 +1,6 @@
#pragma once
#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
class SequenceZoneDefinitionEntry final : public ZoneDefinitionParser::sequence_t
{

View File

@ -1,6 +1,6 @@
#include "SequenceZoneDefinitionIgnore.h"
#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
SequenceZoneDefinitionIgnore::SequenceZoneDefinitionIgnore()
{

View File

@ -1,6 +1,6 @@
#pragma once
#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
class SequenceZoneDefinitionIgnore final : public ZoneDefinitionParser::sequence_t
{

View File

@ -0,0 +1,32 @@
#include "SequenceZoneDefinitionInclude.h"
#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
#include <format>
SequenceZoneDefinitionInclude::SequenceZoneDefinitionInclude()
{
const ZoneDefinitionMatcherFactory create(this);
AddMatchers({
create.Keyword("include"),
create.Char(','),
create.Field().Capture(CAPTURE_INCLUDE_NAME),
});
}
void SequenceZoneDefinitionInclude::ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const
{
const auto& inclusionNameToken = result.NextCapture(CAPTURE_INCLUDE_NAME);
const auto inclusionName = inclusionNameToken.FieldValue();
const auto existingInclusion = state->m_inclusions.find(inclusionName);
if (existingInclusion != state->m_inclusions.end())
throw ParsingException(inclusionNameToken.GetPos(), "Zone definition has already been included");
const auto zoneDefinitionFileNameToInclude = std::format("{}.zone", inclusionName);
if (!state->m_underlying_stream.IncludeFile(zoneDefinitionFileNameToInclude))
throw ParsingException(inclusionNameToken.GetPos(), "Could not find zone definition with this filename");
state->m_inclusions.emplace(inclusionName);
}

View File

@ -1,6 +1,6 @@
#pragma once
#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
class SequenceZoneDefinitionInclude final : public ZoneDefinitionParser::sequence_t
{

View File

@ -1,7 +1,7 @@
#include "SequenceZoneDefinitionMetaData.h"
#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
#include "Utils/StringUtils.h"
#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
#include <cstdint>
#include <format>

View File

@ -1,6 +1,6 @@
#pragma once
#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
class SequenceZoneDefinitionMetaData final : public ZoneDefinitionParser::sequence_t
{

View File

@ -7,8 +7,9 @@
#include "Sequence/SequenceZoneDefinitionInclude.h"
#include "Sequence/SequenceZoneDefinitionMetaData.h"
ZoneDefinitionParser::ZoneDefinitionParser(ZoneDefinitionLexer* lexer, std::string targetName, const std::optional<GameId> maybeGame)
: AbstractParser(lexer, std::make_unique<ZoneDefinitionParserState>(std::move(targetName)))
ZoneDefinitionParser::ZoneDefinitionParser(
ZoneDefinitionLexer* lexer, std::string targetName, ISearchPath& searchPath, IParserLineStream& underlyingStream, const std::optional<GameId> maybeGame)
: AbstractParser(lexer, std::make_unique<ZoneDefinitionParserState>(std::move(targetName), searchPath, underlyingStream))
{
if (maybeGame)
m_state->SetGame(*maybeGame);

View File

@ -2,6 +2,7 @@
#include "Game/IGame.h"
#include "Parsing/Impl/AbstractParser.h"
#include "SearchPath/ISearchPath.h"
#include "Zone/Definition/ZoneDefinition.h"
#include "ZoneDefinitionLexer.h"
#include "ZoneDefinitionParserState.h"
@ -11,10 +12,14 @@
class ZoneDefinitionParser final : public AbstractParser<ZoneDefinitionParserValue, ZoneDefinitionParserState>
{
public:
ZoneDefinitionParser(ZoneDefinitionLexer* lexer,
std::string targetName,
ISearchPath& searchPath,
IParserLineStream& underlyingStream,
std::optional<GameId> maybeGame = std::nullopt);
std::unique_ptr<ZoneDefinition> GetParsedValue();
protected:
const std::vector<sequence_t*>& GetTestsForState() override;
public:
ZoneDefinitionParser(ZoneDefinitionLexer* lexer, std::string targetName, std::optional<GameId> maybeGame = std::nullopt);
std::unique_ptr<ZoneDefinition> GetParsedValue();
};

View File

@ -2,10 +2,14 @@
#include <algorithm>
ZoneDefinitionParserState::ZoneDefinitionParserState(std::string targetName)
: m_asset_name_resolver(nullptr),
ZoneDefinitionParserState::ZoneDefinitionParserState(std::string targetName, ISearchPath& searchPath, IParserLineStream& underlyingStream)
: m_search_path(searchPath),
m_underlying_stream(underlyingStream),
m_asset_name_resolver(nullptr),
m_definition(std::make_unique<ZoneDefinition>())
{
m_inclusions.emplace(targetName);
m_definition->m_name = std::move(targetName);
}

View File

@ -1,15 +1,19 @@
#pragma once
#include "Parsing/IParserLineStream.h"
#include "SearchPath/ISearchPath.h"
#include "Zone/AssetNameResolver.h"
#include "Zone/Definition/ZoneDefinition.h"
#include <memory>
#include <optional>
#include <string>
#include <unordered_set>
class ZoneDefinitionParserState
{
public:
explicit ZoneDefinitionParserState(std::string targetName);
ZoneDefinitionParserState(std::string targetName, ISearchPath& searchPath, IParserLineStream& underlyingStream);
void SetGame(GameId game);
@ -18,6 +22,10 @@ public:
void Finalize();
ISearchPath& m_search_path;
IParserLineStream& m_underlying_stream;
std::unordered_set<std::string> m_inclusions;
const IAssetNameResolver* m_asset_name_resolver;
std::optional<ZoneDefinitionObjContainer> m_current_ipak;

View File

@ -32,20 +32,3 @@ ZoneDefinition::ZoneDefinition()
: m_game(GameId::COUNT)
{
}
void ZoneDefinition::Include(const AssetList& assetListToInclude)
{
for (const auto& entry : assetListToInclude.m_entries)
m_assets.emplace_back(entry.m_type, entry.m_name, false);
}
void ZoneDefinition::Include(const ZoneDefinition& definitionToInclude)
{
m_properties.Include(definitionToInclude.m_properties);
for (const auto& ignore : definitionToInclude.m_ignores)
m_ignores.emplace_back(ignore);
for (const auto& asset : definitionToInclude.m_assets)
m_assets.emplace_back(asset);
}

View File

@ -60,14 +60,9 @@ class ZoneDefinition
public:
ZoneDefinition();
void Include(const AssetList& assetListToInclude);
void Include(const ZoneDefinition& definitionToInclude);
std::string m_name;
GameId m_game;
ZoneDefinitionProperties m_properties;
std::vector<std::string> m_includes;
std::vector<std::string> m_asset_lists;
std::vector<std::string> m_ignores;
std::vector<std::string> m_targets_to_build;
std::vector<std::string> m_gdts;

View File

@ -2,17 +2,18 @@
#include "Parsing/Impl/CommentRemovingStreamProxy.h"
#include "Parsing/Impl/DefinesStreamProxy.h"
#include "Parsing/Impl/IncludingStreamProxy.h"
#include "Parsing/Impl/ParserSingleInputStream.h"
#include "Parsing/ZoneDefinition/ZoneDefinitionLexer.h"
#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
#include "Zone/Definition/Parsing/ZoneDefinitionLexer.h"
#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
#include <chrono>
#include <format>
ZoneDefinitionInputStream::ZoneDefinitionInputStream(std::istream& stream, std::string targetName, std::string fileName, const bool verbose)
: m_target_name(std::move(targetName)),
ZoneDefinitionInputStream::ZoneDefinitionInputStream(std::istream& stream, std::string targetName, std::string fileName, ISearchPath& searchPath)
: SearchPathMultiInputStream(searchPath),
m_target_name(std::move(targetName)),
m_file_name(std::move(fileName)),
m_verbose(verbose),
m_stream(nullptr),
m_previously_set_game(std::nullopt)
{
@ -23,15 +24,15 @@ ZoneDefinitionInputStream::ZoneDefinitionInputStream(std::istream& stream, std::
bool ZoneDefinitionInputStream::OpenBaseStream(std::istream& stream)
{
m_open_streams.emplace_back(std::make_unique<ParserSingleInputStream>(stream, m_file_name));
m_open_streams.emplace_back(std::make_unique<ParserMultiInputStream>(stream, m_file_name, *this));
return true;
}
void ZoneDefinitionInputStream::SetupStreamProxies()
{
m_open_streams.emplace_back(std::make_unique<CommentRemovingStreamProxy>(m_open_streams.back().get()));
auto definesProxy = std::make_unique<DefinesStreamProxy>(m_open_streams.back().get());
m_open_streams.emplace_back(std::move(definesProxy));
m_open_streams.emplace_back(std::make_unique<IncludingStreamProxy>(m_open_streams.back().get()));
m_open_streams.emplace_back(std::make_unique<DefinesStreamProxy>(m_open_streams.back().get()));
m_stream = m_open_streams.back().get();
}
@ -43,11 +44,10 @@ void ZoneDefinitionInputStream::SetPreviouslySetGame(GameId game)
std::unique_ptr<ZoneDefinition> ZoneDefinitionInputStream::ReadDefinition()
{
if (m_verbose)
std::cout << std::format("Reading zone definition file: {}\n", m_file_name);
std::cout << std::format("Reading zone definition file: {}\n", m_file_name);
const auto lexer = std::make_unique<ZoneDefinitionLexer>(m_stream);
const auto parser = std::make_unique<ZoneDefinitionParser>(lexer.get(), m_target_name, m_previously_set_game);
const auto parser = std::make_unique<ZoneDefinitionParser>(lexer.get(), m_target_name, m_search_path, *m_stream, m_previously_set_game);
const auto start = std::chrono::steady_clock::now();
std::unique_ptr<ZoneDefinition> definition;
@ -55,8 +55,7 @@ std::unique_ptr<ZoneDefinition> ZoneDefinitionInputStream::ReadDefinition()
definition = parser->GetParsedValue();
const auto end = std::chrono::steady_clock::now();
if (m_verbose)
std::cout << std::format("Processing zone definition took {}ms\n", std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count());
std::cout << std::format("Processing zone definition took {}ms\n", std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count());
return std::move(definition);
}

View File

@ -2,16 +2,18 @@
#include "Game/IGame.h"
#include "Parsing/IParserLineStream.h"
#include "SearchPath/ISearchPath.h"
#include "SearchPath/SearchPathMultiInputStream.h"
#include "ZoneDefinition.h"
#include <iostream>
#include <memory>
#include <optional>
class ZoneDefinitionInputStream
class ZoneDefinitionInputStream : public SearchPathMultiInputStream
{
public:
ZoneDefinitionInputStream(std::istream& stream, std::string targetName, std::string fileName, bool verbose);
ZoneDefinitionInputStream(std::istream& stream, std::string targetName, std::string fileName, ISearchPath& searchPath);
void SetPreviouslySetGame(GameId game);
std::unique_ptr<ZoneDefinition> ReadDefinition();
@ -22,7 +24,6 @@ private:
std::string m_target_name;
std::string m_file_name;
bool m_verbose;
IParserLineStream* m_stream;
std::vector<std::unique_ptr<IParserLineStream>> m_open_streams;
std::optional<GameId> m_previously_set_game;