Parse techset files for IW4

src/Common/Game/IW4/IW4_Assets.h

@@ -1252,9 +1252,9 @@ namespace IW4
     struct MaterialTechniqueSet
     {
         const char* name;
-        char worldVertFormat;
+        unsigned char worldVertFormat;
         bool hasBeenUploaded;
-        char unused[1];
+        unsigned char unused[1];
         MaterialTechniqueSet* remappedTechniqueSet;
         MaterialTechnique* techniques[48];
     };

src/ObjLoading/Game/IW4/AssetLoaders/AssetLoaderTechniqueSet.cpp

@@ -1,10 +1,14 @@
 #include "AssetLoaderTechniqueSet.h"
 
 #include <cstring>
+#include <sstream>
+#include <type_traits>
 
 #include "ObjLoading.h"
 #include "Game/IW4/IW4.h"
+#include "Game/IW4/TechsetConstantsIW4.h"
 #include "Pool/GlobalAssetPool.h"
+#include "Techset/TechsetFileReader.h"
 
 using namespace IW4;
 
@@ -15,3 +19,68 @@ void* AssetLoaderTechniqueSet::CreateEmptyAsset(const std::string& assetName, Me
     techset->name = memory->Dup(assetName.c_str());
     return techset;
 }
+
+std::string AssetLoaderTechniqueSet::GetTechniqueFileName(const std::string& techniqueName)
+{
+    std::ostringstream ss;
+    ss << "techniques/" << techniqueName << ".tech";
+    return ss.str();
+}
+
+std::string AssetLoaderTechniqueSet::GetTechsetFileName(const std::string& techsetAssetName)
+{
+    std::ostringstream ss;
+    ss << "techsets/" << techsetAssetName << ".techset";
+    return ss.str();
+}
+
+MaterialTechnique* AssetLoaderTechniqueSet::LoadTechniqueWithName(const std::string& techniqueName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager)
+{
+    // TODO: Load technique or use previously loaded one
+    return nullptr;
+}
+
+bool AssetLoaderTechniqueSet::CreateTechsetFromDefinition(const std::string& assetName, const techset::TechsetDefinition& definition, ISearchPath* searchPath, MemoryManager* memory,
+                                                          IAssetLoadingManager* manager)
+{
+    auto* techset = memory->Create<MaterialTechniqueSet>();
+    memset(techset, 0, sizeof(MaterialTechniqueSet));
+    techset->name = memory->Dup(assetName.c_str());
+
+    for(auto i = 0u; i < std::extent_v<decltype(MaterialTechniqueSet::techniques)>; i++)
+    {
+        std::string techniqueName;
+        if(definition.GetTechniqueByIndex(i, techniqueName))
+        {
+            auto* technique = LoadTechniqueWithName(techniqueName, searchPath, memory, manager);
+            if (technique == nullptr)
+                return false;
+            techset->techniques[i] = technique;
+        }
+    }
+
+    manager->AddAsset(ASSET_TYPE_TECHNIQUE_SET, assetName, techset);
+
+    return true;
+}
+
+bool AssetLoaderTechniqueSet::CanLoadFromRaw() const
+{
+    return true;
+}
+
+bool AssetLoaderTechniqueSet::LoadFromRaw(const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const
+{
+    const auto techsetFileName = GetTechsetFileName(assetName);
+    const auto file = searchPath->Open(techsetFileName);
+    if (!file.IsOpen())
+        return false;
+
+    const TechsetFileReader reader(*file.m_stream, techsetFileName, techniqueTypeNames, std::extent_v<decltype(techniqueTypeNames)>);
+    const auto techsetDefinition = reader.ReadTechsetDefinition();
+
+    if(techsetDefinition)
+        return CreateTechsetFromDefinition(assetName, *techsetDefinition, searchPath, memory, manager);
+
+    return false;
+}

src/ObjLoading/Game/IW4/AssetLoaders/AssetLoaderTechniqueSet.h

@@ -3,12 +3,21 @@
 #include "Game/IW4/IW4.h"
 #include "AssetLoading/BasicAssetLoader.h"
 #include "SearchPath/ISearchPath.h"
+#include "Techset/TechsetDefinition.h"
 
 namespace IW4
 {
     class AssetLoaderTechniqueSet final : public BasicAssetLoader<ASSET_TYPE_TECHNIQUE_SET, MaterialTechniqueSet>
     {
+        static std::string GetTechniqueFileName(const std::string& techniqueName);
+        static std::string GetTechsetFileName(const std::string& techsetAssetName);
+        static MaterialTechnique* LoadTechniqueWithName(const std::string& techniqueName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager);
+        static bool CreateTechsetFromDefinition(const std::string& assetName, const techset::TechsetDefinition& definition, ISearchPath* searchPath, MemoryManager* memory,
+                                                IAssetLoadingManager* manager);
+
     public:
         _NODISCARD void* CreateEmptyAsset(const std::string& assetName, MemoryManager* memory) override;
+        _NODISCARD bool CanLoadFromRaw() const override;
+        bool LoadFromRaw(const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const override;
     };
 }

src/ObjLoading/Techset/Parsing/TechsetFileParser.cpp

@@ -0,0 +1,93 @@
+#include "TechsetFileParser.h"
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+using namespace techset;
+
+namespace techset
+{
+    class SequenceTechniqueTypeName final : public Parser::sequence_t
+    {
+        static constexpr auto CAPTURE_TYPE_NAME = 1;
+
+    public:
+        SequenceTechniqueTypeName()
+        {
+            const SimpleMatcherFactory create(this);
+
+            AddMatchers({
+                create.String().Capture(CAPTURE_TYPE_NAME),
+                create.Char(':')
+            });
+        }
+
+    protected:
+        void ProcessMatch(ParserState* state, SequenceResult<SimpleParserValue>& result) const override
+        {
+            const auto& typeNameToken = result.NextCapture(CAPTURE_TYPE_NAME);
+
+            size_t techniqueTypeIndex;
+            if (!state->FindTechniqueTypeIndex(typeNameToken.StringValue(), techniqueTypeIndex))
+                throw ParsingException(typeNameToken.GetPos(), "Unknown technique type name");
+
+            state->m_current_technique_types.push_back(techniqueTypeIndex);
+        }
+    };
+
+    class SequenceTechniqueName final : public Parser::sequence_t
+    {
+        static constexpr auto CAPTURE_NAME = 1;
+
+    public:
+        SequenceTechniqueName()
+        {
+            const SimpleMatcherFactory create(this);
+
+            AddMatchers({
+                create.Or({
+                    create.Identifier(),
+                    create.String()
+                }).Capture(CAPTURE_NAME),
+                create.Char(';')
+            });
+        }
+
+    protected:
+        void ProcessMatch(ParserState* state, SequenceResult<SimpleParserValue>& result) const override
+        {
+            assert(!state->m_current_technique_types.empty());
+
+            const auto& techniqueNameToken = result.NextCapture(CAPTURE_NAME);
+            const auto& techniqueName = techniqueNameToken.m_type == SimpleParserValueType::STRING
+                                            ? techniqueNameToken.StringValue()
+                                            : techniqueNameToken.IdentifierValue();
+
+            for (const auto techniqueTypeIndex : state->m_current_technique_types)
+                state->m_definition->SetTechniqueByIndex(techniqueTypeIndex, techniqueName);
+            state->m_current_technique_types.clear();
+        }
+    };
+}
+
+Parser::Parser(SimpleLexer* lexer, const char** validTechniqueTypeNames, const size_t validTechniqueTypeNameCount)
+    : AbstractParser(lexer, std::make_unique<ParserState>(validTechniqueTypeNames, validTechniqueTypeNameCount))
+{
+}
+
+const std::vector<Parser::sequence_t*>& Parser::GetTestsForState()
+{
+    static std::vector<sequence_t*> allTests({
+        new SequenceTechniqueTypeName(),
+        new SequenceTechniqueName()
+    });
+    static std::vector<sequence_t*> techniqueTypeNameOnlyTests({
+        new SequenceTechniqueTypeName()
+    });
+
+    return m_state->m_current_technique_types.empty() ? techniqueTypeNameOnlyTests : allTests;
+}
+
+std::unique_ptr<TechsetDefinition> Parser::GetTechsetDefinition() const
+{
+    return std::move(m_state->m_definition);
+}

src/ObjLoading/Techset/Parsing/TechsetFileParser.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "Utils/ClassUtils.h"
+#include "TechsetFileParserState.h"
+#include "Techset/TechsetDefinition.h"
+#include "Parsing/Simple/SimpleLexer.h"
+#include "Parsing/Simple/SimpleParserValue.h"
+#include "Parsing/Impl/AbstractParser.h"
+
+namespace techset
+{
+    class Parser final : public AbstractParser<SimpleParserValue, ParserState>
+    {
+    protected:
+        const std::vector<sequence_t*>& GetTestsForState() override;
+
+    public:
+        Parser(SimpleLexer* lexer, const char** validTechniqueTypeNames, size_t validTechniqueTypeNameCount);
+        _NODISCARD std::unique_ptr<TechsetDefinition> GetTechsetDefinition() const;
+    };
+}

src/ObjLoading/Techset/Parsing/TechsetFileParserState.cpp

@@ -0,0 +1,25 @@
+#include "TechsetFileParserState.h"
+
+using namespace techset;
+
+ParserState::ParserState(const char** validTechniqueTypeNames, size_t validTechniqueTypeNameCount)
+    : m_definition(std::make_unique<TechsetDefinition>(validTechniqueTypeNameCount))
+{
+    for(auto i = 0u; i < validTechniqueTypeNameCount; i++)
+    {
+        m_valid_technique_type_names.emplace(std::make_pair(std::string(validTechniqueTypeNames[i]), i));
+    }
+}
+
+bool ParserState::FindTechniqueTypeIndex(const std::string& techniqueTypeName, size_t& techniqueTypeIndex) const
+{
+    const auto foundTechniqueType = m_valid_technique_type_names.find(techniqueTypeName);
+
+    if(foundTechniqueType != m_valid_technique_type_names.end())
+    {
+        techniqueTypeIndex = foundTechniqueType->second;
+        return true;
+    }
+
+    return false;
+}

src/ObjLoading/Techset/Parsing/TechsetFileParserState.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "Techset/TechsetDefinition.h"
+
+namespace techset
+{
+    class ParserState
+    {
+    public:
+        std::map<std::string, size_t> m_valid_technique_type_names;
+        std::unique_ptr<TechsetDefinition> m_definition;
+        std::vector<size_t> m_current_technique_types;
+
+        ParserState(const char** validTechniqueTypeNames, size_t validTechniqueTypeNameCount);
+
+        bool FindTechniqueTypeIndex(const std::string& techniqueTypeName, size_t& techniqueTypeIndex) const;
+    };
+}

src/ObjLoading/Techset/TechsetDefinition.cpp

@@ -0,0 +1,31 @@
+#include "TechsetDefinition.h"
+
+#include <cassert>
+
+using namespace techset;
+
+TechsetDefinition::TechsetDefinition(const size_t techniqueTypeCount)
+    : m_has_technique(techniqueTypeCount),
+      m_technique_names(techniqueTypeCount)
+{
+}
+
+bool TechsetDefinition::GetTechniqueByIndex(const size_t index, std::string& techniqueName) const
+{
+    assert(index < m_has_technique.size());
+    if (index >= m_has_technique.size() || !m_has_technique[index])
+        return false;
+
+    techniqueName = m_technique_names[index];
+    return true;
+}
+
+void TechsetDefinition::SetTechniqueByIndex(const size_t index, std::string techniqueName)
+{
+    assert(index < m_has_technique.size());
+    if (index >= m_has_technique.size())
+        return;
+
+    m_has_technique[index] = true;
+    m_technique_names[index] = std::move(techniqueName);
+}

src/ObjLoading/Techset/TechsetDefinition.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace techset
+{
+    class TechsetDefinition
+    {
+        std::vector<bool> m_has_technique;
+        std::vector<std::string> m_technique_names;
+
+    public:
+        explicit TechsetDefinition(size_t techniqueTypeCount);
+        bool GetTechniqueByIndex(size_t index, std::string& techniqueName) const;
+        void SetTechniqueByIndex(size_t index, std::string techniqueName);
+    };
+}

src/ObjLoading/Techset/TechsetFileReader.cpp

@@ -0,0 +1,33 @@
+#include "TechsetFileReader.h"
+
+#include "Parsing/TechsetFileParser.h"
+#include "StructuredDataDef/Parsing/StructuredDataDefParser.h"
+#include "Parsing/Impl/CommentRemovingStreamProxy.h"
+#include "Parsing/Impl/ParserSingleInputStream.h"
+
+TechsetFileReader::TechsetFileReader(std::istream& stream, std::string fileName, const char** validTechniqueTypeNames, const size_t validTechniqueTypeNameCount)
+    : m_file_name(std::move(fileName)),
+      m_valid_technique_type_names(validTechniqueTypeNames),
+      m_valid_technique_type_name_count(validTechniqueTypeNameCount)
+{
+    m_base_stream = std::make_unique<ParserSingleInputStream>(stream, m_file_name);
+    m_comment_proxy = std::make_unique<CommentRemovingStreamProxy>(m_base_stream.get());
+}
+
+std::unique_ptr<techset::TechsetDefinition> TechsetFileReader::ReadTechsetDefinition() const
+{
+    SimpleLexer::Config lexerConfig;
+    lexerConfig.m_emit_new_line_tokens = false;
+    lexerConfig.m_read_strings = true;
+    lexerConfig.m_read_numbers = false;
+    const auto lexer = std::make_unique<SimpleLexer>(m_comment_proxy.get(), std::move(lexerConfig));
+
+    const auto parser = std::make_unique<techset::Parser>(lexer.get(), m_valid_technique_type_names, m_valid_technique_type_name_count);
+
+    const auto success = parser->Parse();
+    if (success)
+        return parser->GetTechsetDefinition();
+
+    std::cout << "Parsing techset file \"" << m_file_name << "\" failed!\n";
+    return {};
+}

src/ObjLoading/Techset/TechsetFileReader.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "Utils/ClassUtils.h"
+#include "TechsetDefinition.h"
+#include "Parsing/IParserLineStream.h"
+
+class TechsetFileReader
+{
+    std::string m_file_name;
+    const char** m_valid_technique_type_names;
+    size_t m_valid_technique_type_name_count;
+    std::unique_ptr<IParserLineStream> m_base_stream;
+    std::unique_ptr<IParserLineStream> m_comment_proxy;
+
+public:
+    TechsetFileReader(std::istream& stream, std::string fileName, const char** validTechniqueTypeNames, size_t validTechniqueTypeNameCount);
+
+    _NODISCARD std::unique_ptr<techset::TechsetDefinition> ReadTechsetDefinition() const;
+};