chore: parse includes and assetlists while parsing zone definition

premake5.lua

@@ -170,6 +170,7 @@ group ""
 -- ========================
 -- Tests
 -- ========================
+include "test/ObjCommonTestUtils.lua"
 include "test/ObjCommonTests.lua"
 include "test/ObjLoadingTests.lua"
 include "test/ParserTestUtils.lua"
@@ -179,6 +180,7 @@ include "test/ZoneCommonTests.lua"
 
 -- Tests group: Unit test and other tests projects
 group "Tests"
+    ObjCommonTestUtils:project()
     ObjCommonTests:project()
     ObjLoadingTests:project()
     ParserTestUtils:project()

src/Linker/Linker.cpp

@@ -8,7 +8,7 @@
 #include "SearchPath/SearchPaths.h"
 #include "Utils/ObjFileStream.h"
 #include "Zone/AssetList/AssetList.h"
-#include "Zone/AssetList/AssetListStream.h"
+#include "Zone/AssetList/AssetListReader.h"
 #include "Zone/Definition/ZoneDefinitionStream.h"
 #include "ZoneCreation/ZoneCreationContext.h"
 #include "ZoneCreation/ZoneCreator.h"
@@ -155,111 +155,7 @@ namespace
 
 class LinkerImpl final : public Linker
 {
-    bool IncludeAdditionalZoneDefinitions(const std::string& initialFileName, ZoneDefinition& zoneDefinition, ISearchPath& sourceSearchPath) const
+    std::unique_ptr<ZoneDefinition> ReadZoneDefinition(LinkerPathManager& paths, const std::string& targetName, bool logMissing = true) const
-    {
-        std::set<std::string> sourceNames;
-        sourceNames.emplace(initialFileName);
-
-        std::deque<std::string> toIncludeQueue;
-        for (const auto& include : zoneDefinition.m_includes)
-            toIncludeQueue.emplace_back(include);
-
-        while (!toIncludeQueue.empty())
-        {
-            const auto& source = toIncludeQueue.front();
-
-            if (sourceNames.find(source) == sourceNames.end())
-            {
-                sourceNames.emplace(source);
-
-                std::unique_ptr<ZoneDefinition> includeDefinition;
-                {
-                    const auto definitionFileName = std::format("{}.zone", source);
-                    const auto definitionStream = sourceSearchPath.Open(definitionFileName);
-                    if (!definitionStream.IsOpen())
-                    {
-                        std::cerr << std::format("Could not find zone definition file for project \"{}\".\n", source);
-                        return false;
-                    }
-
-                    ZoneDefinitionInputStream zoneDefinitionInputStream(*definitionStream.m_stream, source, definitionFileName, m_args.m_verbose);
-                    zoneDefinitionInputStream.SetPreviouslySetGame(zoneDefinition.m_game);
-                    includeDefinition = zoneDefinitionInputStream.ReadDefinition();
-                }
-
-                if (!includeDefinition)
-                {
-                    std::cerr << std::format("Failed to read zone definition file for project \"{}\".\n", source);
-                    return false;
-                }
-
-                for (const auto& include : includeDefinition->m_includes)
-                    toIncludeQueue.emplace_back(include);
-
-                zoneDefinition.Include(*includeDefinition);
-            }
-
-            toIncludeQueue.pop_front();
-        }
-
-        return true;
-    }
-
-    bool ReadAssetList(LinkerPathManager& paths, const std::string& zoneName, const GameId game, AssetList& assetList) const
-    {
-        {
-            const auto assetListFileName = std::format("assetlist/{}.csv", zoneName);
-            const auto assetListStream = paths.m_source_paths.GetSearchPaths().Open(assetListFileName);
-
-            if (assetListStream.IsOpen())
-            {
-                const AssetListInputStream stream(*assetListStream.m_stream, game);
-                AssetListEntry entry;
-
-                bool failure;
-                while (stream.NextEntry(entry, &failure))
-                {
-                    assetList.m_entries.emplace_back(std::move(entry));
-                }
-
-                return !failure;
-            }
-        }
-
-        {
-            const auto zoneDefinition = ReadZoneDefinition(paths, zoneName);
-
-            if (zoneDefinition)
-            {
-                for (const auto& entry : zoneDefinition->m_assets)
-                {
-                    assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name, entry.m_is_reference);
-                }
-                return true;
-            }
-        }
-
-        return false;
-    }
-
-    bool IncludeAssetLists(LinkerPathManager& paths, ZoneDefinition& zoneDefinition) const
-    {
-        for (const auto& assetListName : zoneDefinition.m_asset_lists)
-        {
-            AssetList assetList;
-            if (!ReadAssetList(paths, assetListName, zoneDefinition.m_game, assetList))
-            {
-                std::cerr << std::format("Failed to read asset list \"{}\"\n", assetListName);
-                return false;
-            }
-
-            zoneDefinition.Include(assetList);
-        }
-
-        return true;
-    }
-
-    std::unique_ptr<ZoneDefinition> ReadZoneDefinition(LinkerPathManager& paths, const std::string& targetName) const
     {
         auto& sourceSearchPath = paths.m_source_paths.GetSearchPaths();
         std::unique_ptr<ZoneDefinition> zoneDefinition;
@@ -268,11 +164,12 @@ class LinkerImpl final : public Linker
             const auto definitionStream = sourceSearchPath.Open(definitionFileName);
             if (!definitionStream.IsOpen())
             {
-                std::cerr << std::format("Could not find zone definition file for target \"{}\".\n", targetName);
+                if (logMissing)
+                    std::cerr << std::format("Could not find zone definition file for target \"{}\".\n", targetName);
                 return nullptr;
             }
 
-            ZoneDefinitionInputStream zoneDefinitionInputStream(*definitionStream.m_stream, targetName, definitionFileName, m_args.m_verbose);
+            ZoneDefinitionInputStream zoneDefinitionInputStream(*definitionStream.m_stream, targetName, definitionFileName, sourceSearchPath);
             zoneDefinition = zoneDefinitionInputStream.ReadDefinition();
         }
 
@@ -282,15 +179,40 @@ class LinkerImpl final : public Linker
             return nullptr;
         }
 
-        if (!IncludeAdditionalZoneDefinitions(targetName, *zoneDefinition, sourceSearchPath))
-            return nullptr;
-
-        if (!IncludeAssetLists(paths, *zoneDefinition))
-            return nullptr;
-
         return zoneDefinition;
     }
 
+    bool ReadIgnoreEntries(LinkerPathManager& paths, const std::string& zoneName, const GameId game, AssetList& assetList) const
+    {
+        {
+            AssetListReader assetListReader(paths.m_source_paths.GetSearchPaths(), game);
+            const auto maybeReadAssetList = assetListReader.ReadAssetList(zoneName, false);
+            if (maybeReadAssetList)
+            {
+                assetList.m_entries.reserve(assetList.m_entries.size() + maybeReadAssetList->m_entries.size());
+                for (auto& entry : maybeReadAssetList->m_entries)
+                    assetList.m_entries.emplace_back(std::move(entry));
+
+                return true;
+            }
+        }
+
+        {
+            const auto zoneDefinition = ReadZoneDefinition(paths, zoneName, false);
+
+            if (zoneDefinition)
+            {
+                assetList.m_entries.reserve(assetList.m_entries.size() + zoneDefinition->m_assets.size());
+                for (const auto& entry : zoneDefinition->m_assets)
+                    assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name, entry.m_is_reference);
+
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     bool ProcessZoneDefinitionIgnores(LinkerPathManager& paths, const std::string& targetName, ZoneCreationContext& context) const
     {
         if (context.m_definition->m_ignores.empty())
@@ -301,8 +223,7 @@ class LinkerImpl final : public Linker
             if (ignore == targetName)
                 continue;
 
-            std::vector<AssetListEntry> assetList;
+            if (!ReadIgnoreEntries(paths, ignore, context.m_definition->m_game, context.m_ignored_assets))
-            if (!ReadAssetList(paths, ignore, context.m_definition->m_game, context.m_ignored_assets))
             {
                 std::cerr << std::format("Failed to read asset listing for ignoring assets of project \"{}\".\n", ignore);
                 return false;

src/ObjCommon.lua

@@ -5,6 +5,7 @@ function ObjCommon:include(includes)
 		Common:include(includes)
 		json:include(includes)
 		minizip:include(includes)
+		Parser:include(includes)
 		includedirs {
 			path.join(ProjectFolder(), "ObjCommon")
 		}
@@ -16,6 +17,7 @@ function ObjCommon:link(links)
 	links:linkto(Utils)
 	links:linkto(Common)
 	links:linkto(minizip)
+	links:linkto(Parser)
 end
 
 function ObjCommon:use()

src/ObjCommon/SearchPath/SearchPathMultiInputStream.cpp

@@ -0,0 +1,15 @@
+#include "SearchPathMultiInputStream.h"
+
+SearchPathMultiInputStream::SearchPathMultiInputStream(ISearchPath& searchPath)
+    : m_search_path(searchPath)
+{
+}
+
+std::unique_ptr<std::istream> SearchPathMultiInputStream::OpenIncludedFile(const std::string& filename, const std::string& sourceFile)
+{
+    auto foundFileToInclude = m_search_path.Open(filename);
+    if (!foundFileToInclude.IsOpen() || !foundFileToInclude.m_stream)
+        return nullptr;
+
+    return std::move(foundFileToInclude.m_stream);
+}

src/ObjCommon/SearchPath/SearchPathMultiInputStream.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "Parsing/Impl/ParserMultiInputStream.h"
+#include "SearchPath/ISearchPath.h"
+
+class SearchPathMultiInputStream : public IInclusionCallback
+{
+public:
+    explicit SearchPathMultiInputStream(ISearchPath& searchPath);
+
+    std::unique_ptr<std::istream> OpenIncludedFile(const std::string& filename, const std::string& sourceFile) override;
+
+protected:
+    ISearchPath& m_search_path;
+};

src/ObjLoading/Game/IW4/Menu/LoaderMenuListIW4.cpp

@@ -198,17 +198,7 @@ namespace
 
         std::unique_ptr<menu::ParsingResult> ParseMenuFile(std::istream& stream, const std::string& menuFileName, const menu::MenuAssetZoneState* zoneState)
         {
-            menu::MenuFileReader reader(stream,
+            menu::MenuFileReader reader(stream, menuFileName, menu::FeatureLevel::IW4, m_search_path);
-                                        menuFileName,
-                                        menu::FeatureLevel::IW4,
-                                        [this](const std::string& filename, const std::string& sourceFile) -> std::unique_ptr<std::istream>
-                                        {
-                                            auto foundFileToInclude = m_search_path.Open(filename);
-                                            if (!foundFileToInclude.IsOpen() || !foundFileToInclude.m_stream)
-                                                return nullptr;
-
-                                            return std::move(foundFileToInclude.m_stream);
-                                        });
 
             reader.IncludeZoneState(zoneState);
             reader.SetPermissiveMode(ObjLoading::Configuration.MenuPermissiveParsing);

src/ObjLoading/Game/IW4/StructuredDataDef/LoaderStructuredDataDefIW4.cpp

@@ -26,16 +26,7 @@ namespace
             if (!file.IsOpen())
                 return AssetCreationResult::NoAction();
 
-            StructuredDataDefReader reader(*file.m_stream,
+            StructuredDataDefReader reader(*file.m_stream, assetName, m_search_path);
-                                           assetName,
-                                           [this](const std::string& filename, const std::string& sourceFile) -> std::unique_ptr<std::istream>
-                                           {
-                                               auto foundFileToInclude = m_search_path.Open(filename);
-                                               if (!foundFileToInclude.IsOpen() || !foundFileToInclude.m_stream)
-                                                   return nullptr;
-
-                                               return std::move(foundFileToInclude.m_stream);
-                                           });
 
             bool readingDefsSuccessful;
             const auto defs = reader.ReadStructureDataDefs(readingDefsSuccessful);

src/ObjLoading/Game/IW5/Menu/LoaderMenuListIW5.cpp

@@ -198,17 +198,7 @@ namespace
 
         std::unique_ptr<menu::ParsingResult> ParseMenuFile(std::istream& stream, const std::string& menuFileName, const menu::MenuAssetZoneState* zoneState)
         {
-            menu::MenuFileReader reader(stream,
+            menu::MenuFileReader reader(stream, menuFileName, menu::FeatureLevel::IW5, m_search_path);
-                                        menuFileName,
-                                        menu::FeatureLevel::IW4,
-                                        [this](const std::string& filename, const std::string& sourceFile) -> std::unique_ptr<std::istream>
-                                        {
-                                            auto foundFileToInclude = m_search_path.Open(filename);
-                                            if (!foundFileToInclude.IsOpen() || !foundFileToInclude.m_stream)
-                                                return nullptr;
-
-                                            return std::move(foundFileToInclude.m_stream);
-                                        });
 
             reader.IncludeZoneState(zoneState);
             reader.SetPermissiveMode(ObjLoading::Configuration.MenuPermissiveParsing);

src/ObjLoading/Parsing/Menu/MenuFileReader.cpp

@@ -11,36 +11,22 @@
 
 using namespace menu;
 
-MenuFileReader::MenuFileReader(std::istream& stream, std::string fileName, const FeatureLevel featureLevel, include_callback_t includeCallback)
+MenuFileReader::MenuFileReader(std::istream& stream, std::string fileName, const FeatureLevel featureLevel, ISearchPath& searchPath)
-    : m_feature_level(featureLevel),
+    : SearchPathMultiInputStream(searchPath),
+      m_feature_level(featureLevel),
       m_file_name(std::move(fileName)),
       m_stream(nullptr),
       m_zone_state(nullptr),
       m_permissive_mode(false)
 {
-    OpenBaseStream(stream, std::move(includeCallback));
+    OpenBaseStream(stream);
     SetupStreamProxies();
     m_stream = m_open_streams.back().get();
 }
 
-MenuFileReader::MenuFileReader(std::istream& stream, std::string fileName, const FeatureLevel featureLevel)
+bool MenuFileReader::OpenBaseStream(std::istream& stream)
-    : m_feature_level(featureLevel),
-      m_file_name(std::move(fileName)),
-      m_stream(nullptr),
-      m_zone_state(nullptr),
-      m_permissive_mode(false)
 {
-    OpenBaseStream(stream, nullptr);
+    m_open_streams.emplace_back(std::make_unique<ParserMultiInputStream>(stream, m_file_name, *this));
-    SetupStreamProxies();
-    m_stream = m_open_streams.back().get();
-}
-
-bool MenuFileReader::OpenBaseStream(std::istream& stream, include_callback_t includeCallback)
-{
-    if (includeCallback)
-        m_open_streams.emplace_back(std::make_unique<ParserMultiInputStream>(stream, m_file_name, std::move(includeCallback)));
-    else
-        m_open_streams.emplace_back(std::make_unique<ParserSingleInputStream>(stream, m_file_name));
 
     return true;
 }

src/ObjLoading/Parsing/Menu/MenuFileReader.h

@@ -5,6 +5,8 @@
 #include "MenuAssetZoneState.h"
 #include "MenuFileParserState.h"
 #include "Parsing/IParserLineStream.h"
+#include "SearchPath/ISearchPath.h"
+#include "SearchPath/SearchPathMultiInputStream.h"
 
 #include <memory>
 #include <string>
@@ -12,12 +14,24 @@
 
 namespace menu
 {
-    class MenuFileReader
+    class MenuFileReader : public SearchPathMultiInputStream
     {
     public:
-        using include_callback_t = std::function<std::unique_ptr<std::istream>(const std::string& filename, const std::string& sourceFile)>;
+        MenuFileReader(std::istream& stream, std::string fileName, FeatureLevel featureLevel, ISearchPath& searchPath);
+
+        void IncludeZoneState(const MenuAssetZoneState* zoneState);
+        void SetPermissiveMode(bool usePermissiveMode);
+
+        std::unique_ptr<ParsingResult> ReadMenuFile();
 
     private:
+        bool OpenBaseStream(std::istream& stream);
+        void SetupDefinesProxy();
+        void SetupStreamProxies();
+
+        bool IsValidEndState(const MenuFileParserState* state) const;
+        std::unique_ptr<ParsingResult> CreateParsingResult(MenuFileParserState* state) const;
+
         const FeatureLevel m_feature_level;
         const std::string m_file_name;
 
@@ -26,21 +40,5 @@ namespace menu
 
         const MenuAssetZoneState* m_zone_state;
         bool m_permissive_mode;
-
-        bool OpenBaseStream(std::istream& stream, include_callback_t includeCallback);
-        void SetupDefinesProxy();
-        void SetupStreamProxies();
-
-        bool IsValidEndState(const MenuFileParserState* state) const;
-        std::unique_ptr<ParsingResult> CreateParsingResult(MenuFileParserState* state) const;
-
-    public:
-        MenuFileReader(std::istream& stream, std::string fileName, FeatureLevel featureLevel);
-        MenuFileReader(std::istream& stream, std::string fileName, FeatureLevel featureLevel, include_callback_t includeCallback);
-
-        void IncludeZoneState(const MenuAssetZoneState* zoneState);
-        void SetPermissiveMode(bool usePermissiveMode);
-
-        std::unique_ptr<ParsingResult> ReadMenuFile();
     };
 } // namespace menu

src/ObjLoading/StructuredDataDef/StructuredDataDefReader.cpp

@@ -7,28 +7,24 @@
 #include "Parsing/Impl/ParserSingleInputStream.h"
 #include "StructuredDataDef/Parsing/StructuredDataDefParser.h"
 
+#include <format>
+#include <iostream>
+
 using namespace sdd;
 
-StructuredDataDefReader::StructuredDataDefReader(std::istream& stream, std::string fileName)
+StructuredDataDefReader::StructuredDataDefReader(std::istream& stream, std::string fileName, ISearchPath& searchPath)
-    : StructuredDataDefReader(stream, std::move(fileName), nullptr)
+    : SearchPathMultiInputStream(searchPath),
-{
+      m_file_name(std::move(fileName)),
-}
-
-StructuredDataDefReader::StructuredDataDefReader(std::istream& stream, std::string fileName, include_callback_t includeCallback)
-    : m_file_name(std::move(fileName)),
       m_stream(nullptr)
 {
-    OpenBaseStream(stream, std::move(includeCallback));
+    OpenBaseStream(stream);
     SetupStreamProxies();
     m_stream = m_open_streams.back().get();
 }
 
-bool StructuredDataDefReader::OpenBaseStream(std::istream& stream, include_callback_t includeCallback)
+bool StructuredDataDefReader::OpenBaseStream(std::istream& stream)
 {
-    if (includeCallback)
+    m_open_streams.emplace_back(std::make_unique<ParserMultiInputStream>(stream, m_file_name, *this));
-        m_open_streams.emplace_back(std::make_unique<ParserMultiInputStream>(stream, m_file_name, std::move(includeCallback)));
-    else
-        m_open_streams.emplace_back(std::make_unique<ParserSingleInputStream>(stream, m_file_name));
 
     return true;
 }
@@ -58,6 +54,7 @@ std::vector<std::unique_ptr<CommonStructuredDataDef>> StructuredDataDefReader::R
     if (success)
         return parser->GetDefs();
 
-    std::cout << "Parsing structured data def file \"" << m_file_name << "\" failed!\n";
+    std::cerr << std::format("Parsing structured data def file \"{}\" failed!\n", m_file_name);
+
     return {};
 }

src/ObjLoading/StructuredDataDef/StructuredDataDefReader.h

@@ -1,28 +1,26 @@
 #pragma once
 
 #include "Parsing/IParserLineStream.h"
+#include "SearchPath/ISearchPath.h"
+#include "SearchPath/SearchPathMultiInputStream.h"
 #include "StructuredDataDef/CommonStructuredDataDef.h"
 
 #include <memory>
 #include <string>
 #include <vector>
 
-class StructuredDataDefReader
+class StructuredDataDefReader : public SearchPathMultiInputStream
 {
 public:
-    using include_callback_t = std::function<std::unique_ptr<std::istream>(const std::string& filename, const std::string& sourceFile)>;
+    StructuredDataDefReader(std::istream& stream, std::string fileName, ISearchPath& searchPath);
+
+    std::vector<std::unique_ptr<CommonStructuredDataDef>> ReadStructureDataDefs(bool& success);
 
 private:
+    bool OpenBaseStream(std::istream& stream);
+    void SetupStreamProxies();
+
     std::string m_file_name;
     IParserLineStream* m_stream;
     std::vector<std::unique_ptr<IParserLineStream>> m_open_streams;
-
-    bool OpenBaseStream(std::istream& stream, include_callback_t includeCallback);
-    void SetupStreamProxies();
-
-public:
-    StructuredDataDefReader(std::istream& stream, std::string fileName);
-    StructuredDataDefReader(std::istream& stream, std::string fileName, include_callback_t includeCallback);
-
-    std::vector<std::unique_ptr<CommonStructuredDataDef>> ReadStructureDataDefs(bool& success);
 };

src/Parser/Parsing/Impl/IncludingStreamProxy.cpp

@@ -3,7 +3,7 @@
 #include "Parsing/ParsingException.h"
 
 #include <filesystem>
-#include <sstream>
+#include <format>
 
 namespace fs = std::filesystem;
 
@@ -81,11 +81,7 @@ bool IncludingStreamProxy::MatchIncludeDirective(const ParserLine& line, const u
     const auto filename = line.m_line.substr(filenameStart, filenameEnd - filenameStart);
 
     if (!m_stream->IncludeFile(filename))
-    {
+        throw ParsingException(CreatePos(line, currentPos), std::format("Could not include file \"{}\"", filename));
-        std::ostringstream errorStr;
-        errorStr << "Could not include file \"" << filename << "\"";
-        throw ParsingException(CreatePos(line, currentPos), errorStr.str());
-    }
 
     return true;
 }

src/Parser/Parsing/Impl/ParserMultiInputStream.cpp

@@ -17,14 +17,14 @@ ParserMultiInputStream::FileInfo::FileInfo(std::istream& stream, std::string fil
 {
 }
 
-ParserMultiInputStream::ParserMultiInputStream(std::unique_ptr<std::istream> stream, std::string fileName, include_callback_t includeCallback)
+ParserMultiInputStream::ParserMultiInputStream(std::unique_ptr<std::istream> stream, std::string fileName, IInclusionCallback& includeCallback)
-    : m_include_callback(std::move(includeCallback))
+    : m_include_callback(includeCallback)
 {
     m_files.emplace(std::move(stream), std::move(fileName));
 }
 
-ParserMultiInputStream::ParserMultiInputStream(std::istream& stream, std::string fileName, include_callback_t includeCallback)
+ParserMultiInputStream::ParserMultiInputStream(std::istream& stream, std::string fileName, IInclusionCallback& includeCallback)
-    : m_include_callback(std::move(includeCallback))
+    : m_include_callback(includeCallback)
 {
     m_files.emplace(stream, std::move(fileName));
 }
@@ -76,10 +76,7 @@ ParserLine ParserMultiInputStream::NextLine()
 
 bool ParserMultiInputStream::IncludeFile(const std::string& filename)
 {
-    if (!m_include_callback)
+    auto newFile = m_include_callback.OpenIncludedFile(filename, m_files.empty() ? "" : *m_files.top().m_file_path);
-        return false;
-
-    auto newFile = m_include_callback(filename, m_files.empty() ? "" : *m_files.top().m_file_path);
     if (!newFile)
         return false;
 

src/Parser/Parsing/Impl/ParserMultiInputStream.h

@@ -7,10 +7,25 @@
 #include <memory>
 #include <stack>
 
+class IInclusionCallback
+{
+public:
+    virtual ~IInclusionCallback() = default;
+
+    virtual std::unique_ptr<std::istream> OpenIncludedFile(const std::string& filename, const std::string& sourceFile) = 0;
+};
+
 class ParserMultiInputStream final : public IParserLineStream
 {
 public:
-    using include_callback_t = std::function<std::unique_ptr<std::istream>(const std::string& filename, const std::string& sourceFile)>;
+    ParserMultiInputStream(std::unique_ptr<std::istream> stream, std::string fileName, IInclusionCallback& includeCallback);
+    ParserMultiInputStream(std::istream& stream, std::string fileName, IInclusionCallback& includeCallback);
+
+    ParserLine NextLine() override;
+    bool IncludeFile(const std::string& filename) override;
+    void PopCurrentFile() override;
+    _NODISCARD bool IsOpen() const override;
+    _NODISCARD bool Eof() const override;
 
 private:
     class FileInfo
@@ -25,16 +40,6 @@ private:
         FileInfo(std::istream& stream, std::string filePath);
     };
 
-    const include_callback_t m_include_callback;
+    IInclusionCallback& m_include_callback;
     std::stack<FileInfo> m_files;
-
-public:
-    ParserMultiInputStream(std::unique_ptr<std::istream> stream, std::string fileName, include_callback_t includeCallback);
-    ParserMultiInputStream(std::istream& stream, std::string fileName, include_callback_t includeCallback);
-
-    ParserLine NextLine() override;
-    bool IncludeFile(const std::string& filename) override;
-    void PopCurrentFile() override;
-    _NODISCARD bool IsOpen() const override;
-    _NODISCARD bool Eof() const override;
 };

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.cpp

@@ -1,19 +0,0 @@
-#include "SequenceZoneDefinitionAssetList.h"
-
-#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
-
-SequenceZoneDefinitionAssetList::SequenceZoneDefinitionAssetList()
-{
-    const ZoneDefinitionMatcherFactory create(this);
-
-    AddMatchers({
-        create.Keyword("assetlist"),
-        create.Char(','),
-        create.Field().Capture(CAPTURE_ASSET_LIST_NAME),
-    });
-}
-
-void SequenceZoneDefinitionAssetList::ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const
-{
-    state->m_definition->m_asset_lists.emplace_back(result.NextCapture(CAPTURE_ASSET_LIST_NAME).FieldValue());
-}

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionInclude.cpp

@@ -1,19 +0,0 @@
-#include "SequenceZoneDefinitionInclude.h"
-
-#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
-
-SequenceZoneDefinitionInclude::SequenceZoneDefinitionInclude()
-{
-    const ZoneDefinitionMatcherFactory create(this);
-
-    AddMatchers({
-        create.Keyword("include"),
-        create.Char(','),
-        create.Field().Capture(CAPTURE_INCLUDE_NAME),
-    });
-}
-
-void SequenceZoneDefinitionInclude::ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const
-{
-    state->m_definition->m_includes.emplace_back(result.NextCapture(CAPTURE_INCLUDE_NAME).FieldValue());
-}

src/ZoneCommon/Zone/AssetList/AssetListOutputStream.cpp

@@ -0,0 +1,14 @@
+#include "AssetListOutputStream.h"
+
+AssetListOutputStream::AssetListOutputStream(std::ostream& stream, const GameId game)
+    : m_stream(stream),
+      m_asset_name_resolver(IAssetNameResolver::GetResolverForGame(game))
+{
+}
+
+void AssetListOutputStream::WriteEntry(const AssetListEntry& entry)
+{
+    m_stream.WriteColumn(*m_asset_name_resolver->GetAssetTypeName(entry.m_type));
+    m_stream.WriteColumn(entry.m_name);
+    m_stream.NextRow();
+}

src/ZoneCommon/Zone/AssetList/AssetListOutputStream.h

@@ -1,23 +1,12 @@
 #pragma once
+
 #include "AssetList.h"
 #include "Csv/CsvStream.h"
 #include "Game/IGame.h"
+#include "Zone/AssetNameResolver.h"
 
-#include <Zone/AssetNameResolver.h>
 #include <iostream>
 
-class AssetListInputStream
-{
-public:
-    AssetListInputStream(std::istream& stream, GameId game);
-
-    bool NextEntry(AssetListEntry& entry, bool* failure) const;
-
-private:
-    CsvInputStream m_stream;
-    const IAssetNameResolver* m_asset_name_resolver;
-};
-
 class AssetListOutputStream
 {
 public:

src/ZoneCommon/Zone/AssetList/AssetListReader.cpp

@@ -0,0 +1,96 @@
+#include "AssetListReader.h"
+
+#include "Csv/CsvStream.h"
+#include "Game/IGame.h"
+#include "Zone/AssetNameResolver.h"
+
+#include <format>
+
+namespace
+{
+    class AssetListInputStream
+    {
+    public:
+        AssetListInputStream(std::istream& stream, GameId game)
+            : m_stream(stream),
+              m_asset_name_resolver(IAssetNameResolver::GetResolverForGame(game))
+        {
+        }
+
+        bool NextEntry(AssetListEntry& entry, bool* failure) const
+        {
+            std::vector<std::string> row;
+            if (failure)
+                *failure = false;
+
+            while (true)
+            {
+                if (!m_stream.NextRow(row))
+                    return false;
+
+                if (row.empty() || (row.size() == 1 && row[0].empty()))
+                    continue;
+
+                const auto& typeName = row[0];
+                const auto maybeType = m_asset_name_resolver->GetAssetTypeByName(typeName);
+                if (!maybeType)
+                {
+                    std::cerr << std::format("Unknown asset type name \"{}\"\n", typeName);
+                    if (failure)
+                        *failure = true;
+                    return false;
+                }
+
+                entry.m_type = *maybeType;
+                if (row.size() >= 3 && row[1].empty())
+                {
+                    entry.m_name = row[2];
+                    entry.m_is_reference = true;
+                }
+                else
+                {
+                    entry.m_name = row[1];
+                    entry.m_is_reference = false;
+                }
+
+                return true;
+            }
+        }
+
+    private:
+        CsvInputStream m_stream;
+        const IAssetNameResolver* m_asset_name_resolver;
+    };
+} // namespace
+
+AssetListReader::AssetListReader(ISearchPath& searchPath, const GameId game)
+    : m_search_path(searchPath),
+      m_game(game)
+{
+}
+
+std::optional<AssetList> AssetListReader::ReadAssetList(const std::string& zoneName, const bool logMissing) const
+{
+    const auto assetListFileName = std::format("assetlist/{}.csv", zoneName);
+    const auto assetListStream = m_search_path.Open(assetListFileName);
+
+    if (assetListStream.IsOpen())
+    {
+        AssetList assetList;
+        const AssetListInputStream stream(*assetListStream.m_stream, m_game);
+        AssetListEntry entry;
+
+        bool failure;
+        while (stream.NextEntry(entry, &failure))
+        {
+            assetList.m_entries.emplace_back(std::move(entry));
+        }
+
+        if (!failure)
+            return assetList;
+    }
+    else if (logMissing)
+        std::cerr << std::format("Failed to open file for assetlist: {}\n", assetListFileName);
+
+    return std::nullopt;
+}

src/ZoneCommon/Zone/AssetList/AssetListReader.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "AssetList.h"
+#include "Game/IGame.h"
+#include "SearchPath/ISearchPath.h"
+
+#include <optional>
+#include <string>
+
+class AssetListReader
+{
+public:
+    AssetListReader(ISearchPath& searchPath, GameId game);
+
+    std::optional<AssetList> ReadAssetList(const std::string& zoneName, bool logMissing = true) const;
+
+private:
+    ISearchPath& m_search_path;
+    GameId m_game;
+};

src/ZoneCommon/Zone/AssetList/AssetListStream.cpp

@@ -1,62 +0,0 @@
-#include "AssetListStream.h"
-
-#include <format>
-
-AssetListInputStream::AssetListInputStream(std::istream& stream, const GameId game)
-    : m_stream(stream),
-      m_asset_name_resolver(IAssetNameResolver::GetResolverForGame(game))
-{
-}
-
-bool AssetListInputStream::NextEntry(AssetListEntry& entry, bool* failure) const
-{
-    std::vector<std::string> row;
-    if (failure)
-        *failure = false;
-
-    while (true)
-    {
-        if (!m_stream.NextRow(row))
-            return false;
-
-        if (row.empty())
-            continue;
-
-        const auto& typeName = row[0];
-        const auto maybeType = m_asset_name_resolver->GetAssetTypeByName(typeName);
-        if (!maybeType)
-        {
-            std::cerr << std::format("Unknown asset type name \"{}\"\n", typeName);
-            if (failure)
-                *failure = true;
-            return false;
-        }
-
-        entry.m_type = *maybeType;
-        if (row.size() >= 3 && row[1].empty())
-        {
-            entry.m_name = row[2];
-            entry.m_is_reference = true;
-        }
-        else
-        {
-            entry.m_name = row[1];
-            entry.m_is_reference = false;
-        }
-
-        return true;
-    }
-}
-
-AssetListOutputStream::AssetListOutputStream(std::ostream& stream, const GameId game)
-    : m_stream(stream),
-      m_asset_name_resolver(IAssetNameResolver::GetResolverForGame(game))
-{
-}
-
-void AssetListOutputStream::WriteEntry(const AssetListEntry& entry)
-{
-    m_stream.WriteColumn(*m_asset_name_resolver->GetAssetTypeName(entry.m_type));
-    m_stream.WriteColumn(entry.m_name);
-    m_stream.NextRow();
-}

src/ZoneCommon/Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherCharacter.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "Parsing/Matcher/AbstractMatcher.h"
-#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParserValue.h"
 
 class ZoneDefinitionMatcherCharacter final : public AbstractMatcher<ZoneDefinitionParserValue>
 {

src/ZoneCommon/Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "Parsing/Matcher/AbstractMatcherFactory.h"
-#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParserValue.h"
 
 #include <string>
 

src/ZoneCommon/Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherKeyword.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "Parsing/Matcher/AbstractMatcher.h"
-#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParserValue.h"
 
 #include <string>
 

src/ZoneCommon/Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherValueType.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "Parsing/Matcher/AbstractMatcher.h"
-#include "Parsing/ZoneDefinition/ZoneDefinitionParserValue.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParserValue.h"
 
 class ZoneDefinitionMatcherValueType final : public AbstractMatcher<ZoneDefinitionParserValue>
 {

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionAssetList.cpp

@@ -0,0 +1,35 @@
+#include "SequenceZoneDefinitionAssetList.h"
+
+#include "Zone/AssetList/AssetListReader.h"
+#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
+
+SequenceZoneDefinitionAssetList::SequenceZoneDefinitionAssetList()
+{
+    const ZoneDefinitionMatcherFactory create(this);
+
+    AddMatchers({
+        create.Keyword("assetlist").Capture(CAPTURE_ASSET_LIST_KEYWORD),
+        create.Char(','),
+        create.Field().Capture(CAPTURE_ASSET_LIST_NAME),
+    });
+}
+
+void SequenceZoneDefinitionAssetList::ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const
+{
+    if (state->m_definition->m_game == GameId::COUNT)
+    {
+        const auto& assetListKeywordToken = result.NextCapture(CAPTURE_ASSET_LIST_KEYWORD);
+        throw ParsingException(assetListKeywordToken.GetPos(), "Must define game before using assetlist");
+    }
+
+    const auto& assetListNameToken = result.NextCapture(CAPTURE_ASSET_LIST_NAME);
+
+    AssetListReader assetListReader(state->m_search_path, state->m_definition->m_game);
+    const auto maybeAssetList = assetListReader.ReadAssetList(assetListNameToken.FieldValue());
+
+    if (!maybeAssetList)
+        throw ParsingException(assetListNameToken.GetPos(), "Failed to read asset list");
+
+    for (auto& assetListEntry : maybeAssetList->m_entries)
+        state->m_definition->m_assets.emplace_back(assetListEntry.m_type, std::move(assetListEntry.m_name), assetListEntry.m_is_reference);
+}

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionAssetList.h

@@ -1,10 +1,11 @@
 #pragma once
 
-#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
 
 class SequenceZoneDefinitionAssetList final : public ZoneDefinitionParser::sequence_t
 {
-    static constexpr auto CAPTURE_ASSET_LIST_NAME = 1;
+    static constexpr auto CAPTURE_ASSET_LIST_KEYWORD = 1;
+    static constexpr auto CAPTURE_ASSET_LIST_NAME = 2;
 
 protected:
     void ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const override;

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionBuild.cpp

@@ -1,6 +1,6 @@
 #include "SequenceZoneDefinitionBuild.h"
 
-#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
+#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
 
 SequenceZoneDefinitionBuild::SequenceZoneDefinitionBuild()
 {

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionBuild.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
 
 class SequenceZoneDefinitionBuild final : public ZoneDefinitionParser::sequence_t
 {

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionEntry.cpp

@@ -1,6 +1,6 @@
 #include "SequenceZoneDefinitionEntry.h"
 
-#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
+#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
 
 SequenceZoneDefinitionEntry::SequenceZoneDefinitionEntry()
 {

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionEntry.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
 
 class SequenceZoneDefinitionEntry final : public ZoneDefinitionParser::sequence_t
 {

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionIgnore.cpp

@@ -1,6 +1,6 @@
 #include "SequenceZoneDefinitionIgnore.h"
 
-#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
+#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
 
 SequenceZoneDefinitionIgnore::SequenceZoneDefinitionIgnore()
 {

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionIgnore.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
 
 class SequenceZoneDefinitionIgnore final : public ZoneDefinitionParser::sequence_t
 {

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionInclude.cpp

@@ -0,0 +1,32 @@
+#include "SequenceZoneDefinitionInclude.h"
+
+#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
+
+#include <format>
+
+SequenceZoneDefinitionInclude::SequenceZoneDefinitionInclude()
+{
+    const ZoneDefinitionMatcherFactory create(this);
+
+    AddMatchers({
+        create.Keyword("include"),
+        create.Char(','),
+        create.Field().Capture(CAPTURE_INCLUDE_NAME),
+    });
+}
+
+void SequenceZoneDefinitionInclude::ProcessMatch(ZoneDefinitionParserState* state, SequenceResult<ZoneDefinitionParserValue>& result) const
+{
+    const auto& inclusionNameToken = result.NextCapture(CAPTURE_INCLUDE_NAME);
+    const auto inclusionName = inclusionNameToken.FieldValue();
+    const auto existingInclusion = state->m_inclusions.find(inclusionName);
+    if (existingInclusion != state->m_inclusions.end())
+        throw ParsingException(inclusionNameToken.GetPos(), "Zone definition has already been included");
+
+    const auto zoneDefinitionFileNameToInclude = std::format("{}.zone", inclusionName);
+
+    if (!state->m_underlying_stream.IncludeFile(zoneDefinitionFileNameToInclude))
+        throw ParsingException(inclusionNameToken.GetPos(), "Could not find zone definition with this filename");
+
+    state->m_inclusions.emplace(inclusionName);
+}

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionInclude.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
 
 class SequenceZoneDefinitionInclude final : public ZoneDefinitionParser::sequence_t
 {

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionMetaData.cpp

@@ -1,7 +1,7 @@
 #include "SequenceZoneDefinitionMetaData.h"
 
-#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
 #include "Utils/StringUtils.h"
+#include "Zone/Definition/Parsing/Matcher/ZoneDefinitionMatcherFactory.h"
 
 #include <cstdint>
 #include <format>

src/ZoneCommon/Zone/Definition/Parsing/Sequence/SequenceZoneDefinitionMetaData.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
 
 class SequenceZoneDefinitionMetaData final : public ZoneDefinitionParser::sequence_t
 {

src/ZoneCommon/Zone/Definition/Parsing/ZoneDefinitionParser.cpp

@@ -7,8 +7,9 @@
 #include "Sequence/SequenceZoneDefinitionInclude.h"
 #include "Sequence/SequenceZoneDefinitionMetaData.h"
 
-ZoneDefinitionParser::ZoneDefinitionParser(ZoneDefinitionLexer* lexer, std::string targetName, const std::optional<GameId> maybeGame)
+ZoneDefinitionParser::ZoneDefinitionParser(
-    : AbstractParser(lexer, std::make_unique<ZoneDefinitionParserState>(std::move(targetName)))
+    ZoneDefinitionLexer* lexer, std::string targetName, ISearchPath& searchPath, IParserLineStream& underlyingStream, const std::optional<GameId> maybeGame)
+    : AbstractParser(lexer, std::make_unique<ZoneDefinitionParserState>(std::move(targetName), searchPath, underlyingStream))
 {
     if (maybeGame)
         m_state->SetGame(*maybeGame);

src/ZoneCommon/Zone/Definition/Parsing/ZoneDefinitionParser.h

@@ -2,6 +2,7 @@
 
 #include "Game/IGame.h"
 #include "Parsing/Impl/AbstractParser.h"
+#include "SearchPath/ISearchPath.h"
 #include "Zone/Definition/ZoneDefinition.h"
 #include "ZoneDefinitionLexer.h"
 #include "ZoneDefinitionParserState.h"
@@ -11,10 +12,14 @@
 
 class ZoneDefinitionParser final : public AbstractParser<ZoneDefinitionParserValue, ZoneDefinitionParserState>
 {
+public:
+    ZoneDefinitionParser(ZoneDefinitionLexer* lexer,
+                         std::string targetName,
+                         ISearchPath& searchPath,
+                         IParserLineStream& underlyingStream,
+                         std::optional<GameId> maybeGame = std::nullopt);
+    std::unique_ptr<ZoneDefinition> GetParsedValue();
+
 protected:
     const std::vector<sequence_t*>& GetTestsForState() override;
-
-public:
-    ZoneDefinitionParser(ZoneDefinitionLexer* lexer, std::string targetName, std::optional<GameId> maybeGame = std::nullopt);
-    std::unique_ptr<ZoneDefinition> GetParsedValue();
 };

src/ZoneCommon/Zone/Definition/Parsing/ZoneDefinitionParserState.cpp

@@ -2,10 +2,14 @@
 
 #include <algorithm>
 
-ZoneDefinitionParserState::ZoneDefinitionParserState(std::string targetName)
+ZoneDefinitionParserState::ZoneDefinitionParserState(std::string targetName, ISearchPath& searchPath, IParserLineStream& underlyingStream)
-    : m_asset_name_resolver(nullptr),
+    : m_search_path(searchPath),
+      m_underlying_stream(underlyingStream),
+      m_asset_name_resolver(nullptr),
       m_definition(std::make_unique<ZoneDefinition>())
+
 {
+    m_inclusions.emplace(targetName);
     m_definition->m_name = std::move(targetName);
 }
 

src/ZoneCommon/Zone/Definition/Parsing/ZoneDefinitionParserState.h

@@ -1,15 +1,19 @@
 #pragma once
 
+#include "Parsing/IParserLineStream.h"
+#include "SearchPath/ISearchPath.h"
 #include "Zone/AssetNameResolver.h"
 #include "Zone/Definition/ZoneDefinition.h"
 
 #include <memory>
 #include <optional>
+#include <string>
+#include <unordered_set>
 
 class ZoneDefinitionParserState
 {
 public:
-    explicit ZoneDefinitionParserState(std::string targetName);
+    ZoneDefinitionParserState(std::string targetName, ISearchPath& searchPath, IParserLineStream& underlyingStream);
 
     void SetGame(GameId game);
 
@@ -18,6 +22,10 @@ public:
 
     void Finalize();
 
+    ISearchPath& m_search_path;
+    IParserLineStream& m_underlying_stream;
+    std::unordered_set<std::string> m_inclusions;
+
     const IAssetNameResolver* m_asset_name_resolver;
 
     std::optional<ZoneDefinitionObjContainer> m_current_ipak;

src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp

@@ -32,20 +32,3 @@ ZoneDefinition::ZoneDefinition()
     : m_game(GameId::COUNT)
 {
 }
-
-void ZoneDefinition::Include(const AssetList& assetListToInclude)
-{
-    for (const auto& entry : assetListToInclude.m_entries)
-        m_assets.emplace_back(entry.m_type, entry.m_name, false);
-}
-
-void ZoneDefinition::Include(const ZoneDefinition& definitionToInclude)
-{
-    m_properties.Include(definitionToInclude.m_properties);
-
-    for (const auto& ignore : definitionToInclude.m_ignores)
-        m_ignores.emplace_back(ignore);
-
-    for (const auto& asset : definitionToInclude.m_assets)
-        m_assets.emplace_back(asset);
-}

src/ZoneCommon/Zone/Definition/ZoneDefinition.h

@@ -60,14 +60,9 @@ class ZoneDefinition
 public:
     ZoneDefinition();
 
-    void Include(const AssetList& assetListToInclude);
-    void Include(const ZoneDefinition& definitionToInclude);
-
     std::string m_name;
     GameId m_game;
     ZoneDefinitionProperties m_properties;
-    std::vector<std::string> m_includes;
-    std::vector<std::string> m_asset_lists;
     std::vector<std::string> m_ignores;
     std::vector<std::string> m_targets_to_build;
     std::vector<std::string> m_gdts;

src/ZoneCommon/Zone/Definition/ZoneDefinitionStream.cpp

@@ -2,17 +2,18 @@
 
 #include "Parsing/Impl/CommentRemovingStreamProxy.h"
 #include "Parsing/Impl/DefinesStreamProxy.h"
+#include "Parsing/Impl/IncludingStreamProxy.h"
 #include "Parsing/Impl/ParserSingleInputStream.h"
-#include "Parsing/ZoneDefinition/ZoneDefinitionLexer.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionLexer.h"
-#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
+#include "Zone/Definition/Parsing/ZoneDefinitionParser.h"
 
 #include <chrono>
 #include <format>
 
-ZoneDefinitionInputStream::ZoneDefinitionInputStream(std::istream& stream, std::string targetName, std::string fileName, const bool verbose)
+ZoneDefinitionInputStream::ZoneDefinitionInputStream(std::istream& stream, std::string targetName, std::string fileName, ISearchPath& searchPath)
-    : m_target_name(std::move(targetName)),
+    : SearchPathMultiInputStream(searchPath),
+      m_target_name(std::move(targetName)),
       m_file_name(std::move(fileName)),
-      m_verbose(verbose),
       m_stream(nullptr),
       m_previously_set_game(std::nullopt)
 {
@@ -23,15 +24,15 @@ ZoneDefinitionInputStream::ZoneDefinitionInputStream(std::istream& stream, std::
 
 bool ZoneDefinitionInputStream::OpenBaseStream(std::istream& stream)
 {
-    m_open_streams.emplace_back(std::make_unique<ParserSingleInputStream>(stream, m_file_name));
+    m_open_streams.emplace_back(std::make_unique<ParserMultiInputStream>(stream, m_file_name, *this));
     return true;
 }
 
 void ZoneDefinitionInputStream::SetupStreamProxies()
 {
     m_open_streams.emplace_back(std::make_unique<CommentRemovingStreamProxy>(m_open_streams.back().get()));
-    auto definesProxy = std::make_unique<DefinesStreamProxy>(m_open_streams.back().get());
+    m_open_streams.emplace_back(std::make_unique<IncludingStreamProxy>(m_open_streams.back().get()));
-    m_open_streams.emplace_back(std::move(definesProxy));
+    m_open_streams.emplace_back(std::make_unique<DefinesStreamProxy>(m_open_streams.back().get()));
 
     m_stream = m_open_streams.back().get();
 }
@@ -43,11 +44,10 @@ void ZoneDefinitionInputStream::SetPreviouslySetGame(GameId game)
 
 std::unique_ptr<ZoneDefinition> ZoneDefinitionInputStream::ReadDefinition()
 {
-    if (m_verbose)
+    std::cout << std::format("Reading zone definition file: {}\n", m_file_name);
-        std::cout << std::format("Reading zone definition file: {}\n", m_file_name);
 
     const auto lexer = std::make_unique<ZoneDefinitionLexer>(m_stream);
-    const auto parser = std::make_unique<ZoneDefinitionParser>(lexer.get(), m_target_name, m_previously_set_game);
+    const auto parser = std::make_unique<ZoneDefinitionParser>(lexer.get(), m_target_name, m_search_path, *m_stream, m_previously_set_game);
 
     const auto start = std::chrono::steady_clock::now();
     std::unique_ptr<ZoneDefinition> definition;
@@ -55,8 +55,7 @@ std::unique_ptr<ZoneDefinition> ZoneDefinitionInputStream::ReadDefinition()
         definition = parser->GetParsedValue();
     const auto end = std::chrono::steady_clock::now();
 
-    if (m_verbose)
+    std::cout << std::format("Processing zone definition took {}ms\n", std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count());
-        std::cout << std::format("Processing zone definition took {}ms\n", std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count());
 
     return std::move(definition);
 }

src/ZoneCommon/Zone/Definition/ZoneDefinitionStream.h

@@ -2,16 +2,18 @@
 
 #include "Game/IGame.h"
 #include "Parsing/IParserLineStream.h"
+#include "SearchPath/ISearchPath.h"
+#include "SearchPath/SearchPathMultiInputStream.h"
 #include "ZoneDefinition.h"
 
 #include <iostream>
 #include <memory>
 #include <optional>
 
-class ZoneDefinitionInputStream
+class ZoneDefinitionInputStream : public SearchPathMultiInputStream
 {
 public:
-    ZoneDefinitionInputStream(std::istream& stream, std::string targetName, std::string fileName, bool verbose);
+    ZoneDefinitionInputStream(std::istream& stream, std::string targetName, std::string fileName, ISearchPath& searchPath);
 
     void SetPreviouslySetGame(GameId game);
     std::unique_ptr<ZoneDefinition> ReadDefinition();
@@ -22,7 +24,6 @@ private:
 
     std::string m_target_name;
     std::string m_file_name;
-    bool m_verbose;
     IParserLineStream* m_stream;
     std::vector<std::unique_ptr<IParserLineStream>> m_open_streams;
     std::optional<GameId> m_previously_set_game;

test/ObjCommonTestUtils.lua

@@ -0,0 +1,53 @@
+ObjCommonTestUtils = {}
+
+function ObjCommonTestUtils:include(includes)
+	if includes:handle(self:name()) then
+		includedirs {
+			path.join(TestFolder(), "ObjCommonTestUtils")
+		}
+	end
+end
+
+function ObjCommonTestUtils:link(links)
+	links:add(self:name())
+	links:linkto(ObjCommon)
+end
+
+function ObjCommonTestUtils:use()
+	
+end
+
+function ObjCommonTestUtils:name()
+    return "ObjCommonTestUtils"
+end
+
+function ObjCommonTestUtils:project()
+	local folder = TestFolder()
+	local includes = Includes:create()
+	local links = Links:create()
+
+	project(self:name())
+        targetdir(TargetDirectoryTest)
+		location "%{wks.location}/test/%{prj.name}"
+		kind "StaticLib"
+		language "C++"
+		
+		files {
+			path.join(folder, "ObjCommonTestUtils/**.h"), 
+			path.join(folder, "ObjCommonTestUtils/**.cpp")
+		}
+		
+        vpaths {
+			["*"] = {
+				path.join(folder, "ObjCommonTestUtils")
+			}
+		}
+		
+		self:include(includes)
+		ObjCommon:include(includes)
+		catch2:include(includes)
+
+		links:linkto(ObjCommon)
+		links:linkto(catch2)
+		links:linkall()
+end

test/ObjCommonTestUtils/SearchPath/MockSearchPath.h

@@ -1,16 +1,18 @@
 #pragma once
+
 #include "SearchPath/ISearchPath.h"
 
-#include <map>
+#include <unordered_map>
 
 class MockSearchPath final : public ISearchPath
 {
-    std::map<std::string, std::string> m_file_data_map;
-
 public:
     void AddFileData(std::string fileName, std::string fileData);
 
     SearchPathOpenFile Open(const std::string& fileName) override;
     const std::string& GetPath() override;
     void Find(const SearchPathSearchOptions& options, const std::function<void(const std::string&)>& callback) override;
+
+private:
+    std::unordered_map<std::string, std::string> m_file_data_map;
 };

test/ObjLoadingTests.lua

@@ -43,10 +43,12 @@ function ObjLoadingTests:project()
 		}
 		
 		self:include(includes)
+		ObjCommonTestUtils:include(includes)
 		ParserTestUtils:include(includes)
 		ObjLoading:include(includes)
 		catch2:include(includes)
 
+		links:linkto(ObjCommonTestUtils)
 		links:linkto(ParserTestUtils)
 		links:linkto(ObjLoading)
 		links:linkto(catch2)

test/ObjLoadingTests/Game/IW3/StringTable/AssetLoaderStringTableIW3Test.cpp

@@ -1,8 +1,8 @@
 #include "Game/IW3/StringTable/AssetLoaderStringTableIW3.h"
 
 #include "Game/IW3/GameIW3.h"
-#include "Mock/MockSearchPath.h"
 #include "Pool/ZoneAssetPools.h"
+#include "SearchPath/MockSearchPath.h"
 #include "Utils/MemoryManager.h"
 
 #include <catch2/catch_test_macros.hpp>

test/ObjLoadingTests/Game/IW4/AssetLoaders/LoaderStringTableIW4Test.cpp

@@ -2,7 +2,7 @@
 
 #include "Game/IW4/CommonIW4.h"
 #include "Game/IW4/GameIW4.h"
-#include "Mock/MockSearchPath.h"
+#include "SearchPath/MockSearchPath.h"
 #include "Utils/MemoryManager.h"
 
 #include <catch2/catch_test_macros.hpp>

test/ObjLoadingTests/Game/IW4/Menu/LoaderMenuListIW4Test.cpp

@@ -1,8 +1,8 @@
 #include "Game/IW4/Menu/LoaderMenuListIW4.h"
 
 #include "Game/IW4/GameIW4.h"
-#include "Mock/MockSearchPath.h"
 #include "Parsing/Menu/MenuFileReader.h"
+#include "SearchPath/MockSearchPath.h"
 #include "Utils/MemoryManager.h"
 
 #include <catch2/catch_test_macros.hpp>

test/ObjLoadingTests/Game/IW5/AssetLoaders/LoaderStringTableIW5Test.cpp

@@ -1,7 +1,7 @@
 #include "Game/IW5/StringTable/LoaderStringTableIW5.h"
 
 #include "Game/IW5/GameIW5.h"
-#include "Mock/MockSearchPath.h"
+#include "SearchPath/MockSearchPath.h"
 #include "Utils/MemoryManager.h"
 
 #include <catch2/catch_test_macros.hpp>

test/ObjLoadingTests/Game/T5/AssetLoaders/LoaderStringTableT5Test.cpp

@@ -1,7 +1,7 @@
 #include "Game/T5/StringTable/LoaderStringTableT5.h"
 
 #include "Game/T5/GameT5.h"
-#include "Mock/MockSearchPath.h"
+#include "SearchPath/MockSearchPath.h"
 #include "Utils/MemoryManager.h"
 
 #include <catch2/catch_test_macros.hpp>

test/ObjLoadingTests/Game/T6/AssetLoaders/LoaderStringTableT6Test.cpp

@@ -1,7 +1,7 @@
 #include "Game/T6/StringTable/LoaderStringTableT6.h"
 
 #include "Game/T6/GameT6.h"
-#include "Mock/MockSearchPath.h"
+#include "SearchPath/MockSearchPath.h"
 #include "Utils/MemoryManager.h"
 
 #include <catch2/catch_test_macros.hpp>

test/ZoneCommonTests.lua

@@ -45,9 +45,11 @@ function ZoneCommonTests:project()
 		}
 		
 		self:include(includes)
+		ObjCommonTestUtils:include(includes)
 		ZoneCommon:include(includes)
 		catch2:include(includes)
 
+		links:linkto(ObjCommonTestUtils)
 		links:linkto(ZoneCommon)
 		links:linkto(catch2)
 		links:linkall()

test/ZoneCommonTests/Zone/Definition/ZoneDefinitionStreamTests.cpp

@@ -1,4 +1,5 @@
 #include "Game/T6/T6.h"
+#include "SearchPath/MockSearchPath.h"
 #include "Zone/Definition/ZoneDefinitionStream.h"
 
 #include <catch2/catch_test_macros.hpp>
@@ -19,7 +20,8 @@ material,gradient_top
 menu,demo_ingame
 )sampledata");
 
-        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", true);
+        MockSearchPath mockSearchPath;
+        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", mockSearchPath);
 
         const auto result = inputStream.ReadDefinition();
         REQUIRE(result);
@@ -50,7 +52,8 @@ menu,demo_ingame
 material,demo_material
 )sampledata");
 
-        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", true);
+        MockSearchPath mockSearchPath;
+        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", mockSearchPath);
 
         const auto result = inputStream.ReadDefinition();
         REQUIRE(result);
@@ -68,19 +71,34 @@ material,demo_material
 >name,test_mod
 
 include,demo_gun
+material,asdf
 include,demo_scripts
 )sampledata");
 
-        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", true);
+        MockSearchPath mockSearchPath;
+
+        mockSearchPath.AddFileData("demo_gun.zone", R"sampledata(
+material,demo_gun_material
+)sampledata");
+
+        mockSearchPath.AddFileData("demo_scripts.zone", R"sampledata(
+rawfile,demo_gun_script.gsc
+)sampledata");
+
+        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", mockSearchPath);
 
         const auto result = inputStream.ReadDefinition();
         REQUIRE(result);
+        REQUIRE(result->m_assets.size() == 3);
 
-        REQUIRE(result->m_assets.empty());
+        REQUIRE(result->m_assets[0].m_asset_name == "demo_gun_material");
-        REQUIRE(result->m_includes.size() == 2);
+        REQUIRE(result->m_assets[0].m_asset_type == T6::ASSET_TYPE_MATERIAL);
 
-        REQUIRE(result->m_includes[0] == "demo_gun");
+        REQUIRE(result->m_assets[1].m_asset_name == "asdf");
-        REQUIRE(result->m_includes[1] == "demo_scripts");
+        REQUIRE(result->m_assets[1].m_asset_type == T6::ASSET_TYPE_MATERIAL);
+
+        REQUIRE(result->m_assets[2].m_asset_name == "demo_gun_script.gsc");
+        REQUIRE(result->m_assets[2].m_asset_type == T6::ASSET_TYPE_RAWFILE);
     }
 
     TEST_CASE("ZoneDefinitionInputStream: Ensure can include assetlists", "[zonedefinition]")
@@ -91,21 +109,47 @@ include,demo_scripts
 >name,test_mod
 
 assetlist,code_post_gfx_mp
+material,asdf
 assetlist,common_mp
 
 material,test_material
 )sampledata");
 
-        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", true);
+        MockSearchPath mockSearchPath;
+
+        mockSearchPath.AddFileData("assetlist/code_post_gfx_mp.csv", R"sampledata(
+material,post_fx_mat
+rawfile,code_post_gfx_mp
+)sampledata");
+
+        mockSearchPath.AddFileData("assetlist/common_mp.csv", R"sampledata(
+material,common_mat
+rawfile,common_mp
+)sampledata");
+
+        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", mockSearchPath);
 
         const auto result = inputStream.ReadDefinition();
         REQUIRE(result);
+        REQUIRE(result->m_assets.size() == 6);
 
-        REQUIRE(result->m_assets.size() == 1);
+        REQUIRE(result->m_assets[0].m_asset_name == "post_fx_mat");
-        REQUIRE(result->m_asset_lists.size() == 2);
+        REQUIRE(result->m_assets[0].m_asset_type == T6::ASSET_TYPE_MATERIAL);
 
-        REQUIRE(result->m_asset_lists[0] == "code_post_gfx_mp");
+        REQUIRE(result->m_assets[1].m_asset_name == "code_post_gfx_mp");
-        REQUIRE(result->m_asset_lists[1] == "common_mp");
+        REQUIRE(result->m_assets[1].m_asset_type == T6::ASSET_TYPE_RAWFILE);
+
+        REQUIRE(result->m_assets[2].m_asset_name == "asdf");
+        REQUIRE(result->m_assets[2].m_asset_type == T6::ASSET_TYPE_MATERIAL);
+
+        REQUIRE(result->m_assets[3].m_asset_name == "common_mat");
+        REQUIRE(result->m_assets[3].m_asset_type == T6::ASSET_TYPE_MATERIAL);
+
+        REQUIRE(result->m_assets[4].m_asset_name == "common_mp");
+        REQUIRE(result->m_assets[4].m_asset_type == T6::ASSET_TYPE_RAWFILE);
+
+        REQUIRE(result->m_assets[5].m_asset_name == "test_material");
+        REQUIRE(result->m_assets[5].m_asset_type == T6::ASSET_TYPE_MATERIAL);
     }
 
     TEST_CASE("ZoneDefinitionInputStream: Ensure can define other build targets", "[zonedefinition]")
@@ -122,7 +166,8 @@ material,test_material
 build,more_mods
 )sampledata");
 
-        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", true);
+        MockSearchPath mockSearchPath;
+        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", mockSearchPath);
 
         const auto result = inputStream.ReadDefinition();
         REQUIRE(result);
@@ -147,7 +192,8 @@ ignore,common_mp
 material,test_material
 )sampledata");
 
-        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", true);
+        MockSearchPath mockSearchPath;
+        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", mockSearchPath);
 
         const auto result = inputStream.ReadDefinition();
         REQUIRE(result);
@@ -171,7 +217,8 @@ material,test_material
 material,test_material
 )sampledata");
 
-        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", true);
+        MockSearchPath mockSearchPath;
+        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", mockSearchPath);
 
         const auto result = inputStream.ReadDefinition();
         REQUIRE(result);
@@ -197,7 +244,8 @@ material,test_material
 >level.ipak_read,code_post_gfx
 )sampledata");
 
-        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", true);
+        MockSearchPath mockSearchPath;
+        ZoneDefinitionInputStream inputStream(inputData, "test", "test.zone", mockSearchPath);
 
         const auto result = inputStream.ReadDefinition();
         REQUIRE(result);