Read assetlists for ignoring assets

src/Linker/Linker.cpp

@@ -4,6 +4,7 @@
 #include <regex>
 #include <filesystem>
 #include <fstream>
+#include <deque>
 
 #include "Utils/ClassUtils.h"
 #include "Utils/Arguments/ArgumentParser.h"
@@ -16,6 +17,8 @@
 #include "LinkerArgs.h"
 
 #include "Utils/ObjFileStream.h"
+#include "Zone/AssetList/AssetList.h"
+#include "Zone/AssetList/AssetListStream.h"
 #include "Zone/Definition/ZoneDefinitionStream.h"
 
 namespace fs = std::filesystem;
@@ -204,32 +207,175 @@ class Linker::Impl
         return true;
     }
 
-    bool BuildZone(const std::string& zoneName)
+    bool IncludeAdditionalZoneDefinitions(const std::string& initialFileName, ZoneDefinition& zoneDefinition, ISearchPath* sourceSearchPath) const
     {
-        auto assetSearchPaths = GetAssetSearchPathsForZone(zoneName);
+        std::set<std::string> sourceNames;
-        auto gdtSearchPaths = GetGdtSearchPathsForZone(zoneName);
+        sourceNames.emplace(initialFileName);
-        auto sourceSearchPaths = GetSourceSearchPathsForZone(zoneName);
 
+        std::deque<std::string> toIncludeQueue;
+        for (const auto& include : zoneDefinition.m_includes)
+            toIncludeQueue.emplace_back(include);
+
+        while(!toIncludeQueue.empty())
+        {
+            const auto& source = toIncludeQueue.front();
+
+            if(sourceNames.find(source) == sourceNames.end())
+            {
+                sourceNames.emplace(source);
+
+                std::unique_ptr<ZoneDefinition> includeDefinition;
+                {
+                    const auto definitionFileName = source + ".zone";
+                    const auto definitionStream = sourceSearchPath->Open(definitionFileName);
+                    if (!definitionStream)
+                    {
+                        std::cout << "Could not find zone definition file for zone \"" << source << "\"." << std::endl;
+                        return false;
+                    }
+
+                    ZoneDefinitionInputStream zoneDefinitionInputStream(*definitionStream, definitionFileName, m_args.m_verbose);
+                    includeDefinition = zoneDefinitionInputStream.ReadDefinition();
+                }
+
+                if (!includeDefinition)
+                {
+                    std::cout << "Failed to read zone definition file for zone \"" << source << "\"." << std::endl;
+                    return false;
+                }
+
+                for (const auto& include : includeDefinition->m_includes)
+                    toIncludeQueue.emplace_back(include);
+
+                zoneDefinition.Include(*includeDefinition);
+            }
+
+            toIncludeQueue.pop_front();
+        }
+
+        return true;
+    }
+
+    std::unique_ptr<ZoneDefinition> ReadZoneDefinition(const std::string& zoneName, ISearchPath* sourceSearchPath) const
+    {
         std::unique_ptr<ZoneDefinition> zoneDefinition;
         {
             const auto definitionFileName = zoneName + ".zone";
-            const auto definitionStream = sourceSearchPaths.Open(definitionFileName);
+            const auto definitionStream = sourceSearchPath->Open(definitionFileName);
             if (!definitionStream)
             {
                 std::cout << "Could not find zone definition file for zone \"" << zoneName << "\"." << std::endl;
-                return false;
+                return nullptr;
             }
 
             ZoneDefinitionInputStream zoneDefinitionInputStream(*definitionStream, definitionFileName, m_args.m_verbose);
             zoneDefinition = zoneDefinitionInputStream.ReadDefinition();
         }
 
-        if(!zoneDefinition)
+        if (!zoneDefinition)
         {
             std::cout << "Failed to read zone definition file for zone \"" << zoneName << "\"." << std::endl;
+            return nullptr;
+        }
+
+        if (!IncludeAdditionalZoneDefinitions(zoneName, *zoneDefinition, sourceSearchPath))
+            return nullptr;
+
+        return zoneDefinition;
+    }
+
+    bool ReadAssetList(const std::string& zoneName, std::vector<AssetListEntry>& assetList, ISearchPath* sourceSearchPath) const
+    {
+        {
+            const auto assetListFileName = "assetlist/" + zoneName + ".csv";
+            const auto assetListStream = sourceSearchPath->Open(assetListFileName);
+
+            if (assetListStream)
+            {
+                const AssetListInputStream stream(*assetListStream);
+                AssetListEntry entry;
+
+                while (stream.NextEntry(entry))
+                {
+                    assetList.emplace_back(std::move(entry));
+                }
+                return true;
+            }
+        }
+
+        {
+            const auto zoneDefinition = ReadZoneDefinition(zoneName, sourceSearchPath);
+
+            if(zoneDefinition)
+            {
+                for(const auto& entry : zoneDefinition->m_assets)
+                {
+                    assetList.emplace_back(entry.m_asset_type, entry.m_asset_name);
+                }
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    bool ProcessZoneDefinitionIgnores(const std::string& zoneName, ZoneDefinition& zoneDefinition, ISearchPath* sourceSearchPath) const
+    {
+        if (zoneDefinition.m_ignores.empty())
+            return true;
+
+        std::map<std::string, std::reference_wrapper<ZoneDefinitionEntry>> zoneDefinitionAssetsByName;
+        for (auto& entry : zoneDefinition.m_assets)
+        {
+            zoneDefinitionAssetsByName.try_emplace(entry.m_asset_name, entry);
+        }
+
+        for(const auto& ignore : zoneDefinition.m_ignores)
+        {
+            if(ignore == zoneName)
+                continue;
+
+            std::vector<AssetListEntry> assetList;
+            if(!ReadAssetList(ignore, assetList, sourceSearchPath))
+            {
+                std::cout << "Failed to read asset listing for ignoring assets of zone \"" << ignore << "\"." << std::endl;
+                return false;
+            }
+
+            for(const auto& assetListEntry : assetList)
+            {
+                const auto foundAsset = zoneDefinitionAssetsByName.find(assetListEntry.m_name);
+
+                if(foundAsset != zoneDefinitionAssetsByName.end()
+                    && foundAsset->second.get().m_asset_type == assetListEntry.m_type)
+                {
+                    foundAsset->second.get().m_is_reference = true;
+                }
+            }
+        }
+        return true;
+    }
+
+    std::unique_ptr<Zone> CreateZoneForDefinition(const std::string& zoneName, ZoneDefinition& zoneDefinition)
+    {
+        return nullptr;
+    }
+    
+    bool BuildZone(const std::string& zoneName)
+    {
+        auto assetSearchPaths = GetAssetSearchPathsForZone(zoneName);
+        auto gdtSearchPaths = GetGdtSearchPathsForZone(zoneName);
+        auto sourceSearchPaths = GetSourceSearchPathsForZone(zoneName);
+
+        const auto zoneDefinition = ReadZoneDefinition(zoneName, &sourceSearchPaths);
+        if (!zoneDefinition
+            || !ProcessZoneDefinitionIgnores(zoneName, *zoneDefinition, &sourceSearchPaths))
+        {
             return false;
         }
 
+        const auto zone = CreateZoneForDefinition(zoneName, *zoneDefinition);
+
         for(const auto& loadedSearchPath : m_loaded_zone_search_paths)
         {
             UnloadSearchPath(loadedSearchPath.get());

src/ZoneCommon/Zone/AssetList/AssetList.cpp

@@ -0,0 +1,10 @@
+#include "AssetList.h"
+
+AssetListEntry::AssetListEntry()
+= default;
+
+AssetListEntry::AssetListEntry(std::string type, std::string name)
+    : m_type(std::move(type)),
+      m_name(std::move(name))
+{
+}

src/ZoneCommon/Zone/AssetList/AssetList.h

@@ -0,0 +1,12 @@
+#pragma once
+#include <string>
+
+class AssetListEntry
+{
+public:
+    std::string m_type;
+    std::string m_name;
+
+    AssetListEntry();
+    AssetListEntry(std::string type, std::string name);
+};

src/ZoneCommon/Zone/AssetList/AssetListStream.cpp

@@ -0,0 +1,37 @@
+#include "AssetListStream.h"
+
+AssetListInputStream::AssetListInputStream(std::istream& stream)
+    : m_stream(stream)
+{
+}
+
+bool AssetListInputStream::NextEntry(AssetListEntry& entry) const
+{
+    std::vector<std::string> row;
+
+    while(true)
+    {
+        if (!m_stream.NextRow(row))
+            return false;
+
+        if (row.empty())
+            continue;
+
+        entry.m_type = row[0];
+        if (row.size() >= 2)
+            entry.m_name = row[1];
+        return true;
+    }
+}
+
+AssetListOutputStream::AssetListOutputStream(std::ostream& stream)
+    : m_stream(stream)
+{
+}
+
+void AssetListOutputStream::WriteEntry(const AssetListEntry& entry)
+{
+    m_stream.WriteColumn(entry.m_type);
+    m_stream.WriteColumn(entry.m_name);
+    m_stream.NextRow();
+}

src/ZoneCommon/Zone/AssetList/AssetListStream.h

@@ -0,0 +1,25 @@
+#pragma once
+#include <iostream>
+
+#include "AssetList.h"
+#include "Csv/CsvStream.h"
+
+class AssetListInputStream
+{
+    CsvInputStream m_stream;
+
+public:
+    explicit AssetListInputStream(std::istream& stream);
+
+    bool NextEntry(AssetListEntry& entry) const;
+};
+
+class AssetListOutputStream
+{
+    CsvOutputStream m_stream;
+
+public:
+    explicit AssetListOutputStream(std::ostream& stream);
+
+    void WriteEntry(const AssetListEntry& entry);
+};

src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp

@@ -11,3 +11,24 @@ ZoneDefinitionEntry::ZoneDefinitionEntry(std::string type, std::string name, boo
       m_is_reference(isReference)
 {
 }
+
+void ZoneDefinition::Include(ZoneDefinition& definitionToInclude)
+{
+    for(const auto& [key, value] : definitionToInclude.m_metadata)
+    {
+        if(m_metadata.find(key) == m_metadata.end())
+        {
+            m_metadata.emplace(std::make_pair(key, value));
+        }
+    }
+
+    for(const auto& ignore : definitionToInclude.m_ignores)
+    {
+        m_ignores.emplace_back(ignore);
+    }
+
+    for(const auto& asset : definitionToInclude.m_assets)
+    {
+        m_assets.emplace_back(asset);
+    }
+}

src/ZoneCommon/Zone/Definition/ZoneDefinition.h

@@ -21,4 +21,6 @@ public:
     std::vector<std::string> m_includes;
     std::vector<std::string> m_ignores;
     std::vector<ZoneDefinitionEntry> m_assets;
+
+    void Include(ZoneDefinition& definitionToInclude);
 };