ObjLoading/ObjWriting: Initial skeleton for loading and writing obj files

premake5.lua

@@ -105,6 +105,7 @@ include "src/ZoneCommon.lua"
 include "src/ZoneLoading.lua"
 include "src/ZoneWriting.lua"
 include "src/ZoneCommon.lua"
+include "src/ObjCommon.lua"
 include "src/ObjLoading.lua"
 include "src/ObjWriting.lua"
 
@@ -117,6 +118,7 @@ group "Components"
     ZoneCommon:project()
     ZoneLoading:project()
     ZoneWriting:project()
+    ObjCommon:project()
     ObjLoading:project()
     ObjWriting:project()
 group ""

src/ObjCommon.lua

@@ -0,0 +1,44 @@
+ObjCommon = {}
+
+function ObjCommon:include()
+	ZoneCommon:include()
+	includedirs {
+		path.join(ProjectFolder(), "ObjCommon")
+	}
+end
+
+function ObjCommon:link()
+    Utils:link()
+	ZoneCommon:link()
+	links {
+		"ObjCommon"
+	}
+end
+
+function ObjCommon:use()
+	
+end
+
+function ObjCommon:project()
+	local folder = ProjectFolder();
+
+	project "ObjCommon"
+        targetdir(TargetDirectoryLib)
+		location "%{wks.location}/src/%{prj.name}"
+		kind "StaticLib"
+		language "C++"
+		
+		files {
+			path.join(folder, "ObjCommon/**.h"), 
+			path.join(folder, "ObjCommon/**.cpp")
+		}
+		
+        vpaths {
+			["*"] = {
+				path.join(folder, "ObjCommon")
+			}
+		}
+		
+        self:include()
+		Utils:include()
+end

src/ObjCommon/ObjContainer/IObjContainer.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+
+class IObjContainer
+{
+public:
+    virtual ~IObjContainer() = default;
+
+    virtual const std::string& GetName() = 0;
+};

src/ObjCommon/ObjContainer/IPak/IPakTypes.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <cstdint>
+
+typedef uint32_t IPakHash;
+
+struct IPakHeader
+{
+	uint32_t magic;
+	uint32_t version;
+	uint32_t size;
+	uint32_t sectionCount;
+};
+
+struct IPakSection
+{
+	uint32_t type;
+	uint32_t offset;
+	uint32_t size;
+	uint32_t itemCount;
+};
+
+union IPakIndexEntryKey
+{
+	struct
+	{
+		IPakHash nameHash;
+		IPakHash dataHash;
+	};
+	uint64_t combinedKey;
+};
+
+struct IPakIndexEntry
+{
+	IPakIndexEntryKey key;
+	uint32_t offset;
+	uint32_t size;
+};
+
+struct IPakDataChunkHeader
+{
+	union
+	{
+		uint32_t countAndOffset;
+		struct
+		{
+			uint32_t offset : 24;
+			uint32_t count : 8;
+		};
+	};
+	union
+	{
+		uint32_t commands[31];
+		struct
+		{
+			uint32_t size : 24;
+			uint32_t compressed : 8;
+		}_commands[31];
+	};
+};

src/ObjCommon/ObjContainer/ObjContainerReferenceable.cpp

@@ -0,0 +1,16 @@
+#include "ObjContainerReferenceable.h"
+
+void ObjContainerReferenceable::AddReference(Zone* referencer)
+{
+    m_references.insert(referencer);
+}
+
+bool ObjContainerReferenceable::RemoveReference(Zone* zone)
+{
+    return m_references.erase(zone) > 0;
+}
+
+bool ObjContainerReferenceable::IsReferenced() const
+{
+    return !m_references.empty();
+}

src/ObjCommon/ObjContainer/ObjContainerReferenceable.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "IObjContainer.h"
+#include "Zone/Zone.h"
+
+#include <set>
+
+class ObjContainerReferenceable : public IObjContainer
+{
+    std::set<Zone*> m_references;
+
+public:
+    void AddReference(Zone* referencer);
+    bool RemoveReference(Zone* zone);
+    bool IsReferenced() const;
+};

src/ObjLoading.lua

@@ -1,6 +1,7 @@
 ObjLoading = {}
 
 function ObjLoading:include()
+	ObjCommon:include()
 	ZoneCommon:include()
 	includedirs {
 		path.join(ProjectFolder(), "ObjLoading")
@@ -9,7 +10,11 @@ end
 
 function ObjLoading:link()
     Utils:link()
+	ObjCommon:link()
 	ZoneCommon:link()
+	minilzo:link()
+	minizip:link()
+	zlib:link()
 	links {
 		"ObjLoading"
 	}
@@ -39,6 +44,10 @@ function ObjLoading:project()
 			}
 		}
 		
-        self:include()
+		self:include()
+		Crypto:include()
 		Utils:include()
+		minilzo:include()
+		minizip:include()
+		zlib:include()
 end

src/ObjLoading/Game/T6/ObjLoaderT6.cpp

@@ -0,0 +1,60 @@
+#include "ObjLoaderT6.h"
+#include "Game/T6/GameT6.h"
+#include "Game/T6/GameAssetPoolT6.h"
+
+const int ObjLoaderT6::IPAK_READ_HASH = ObjLoaderT6::Com_HashKey("ipak_read", 64);
+const int ObjLoaderT6::GLOBAL_HASH = ObjLoaderT6::Com_HashKey("GLOBAL", 64);
+
+int ObjLoaderT6::Com_HashKey(const char* str, const int maxLen)
+{
+    if (str == nullptr)
+        return 0;
+
+    int hash = 0;
+    for (int i = 0; i < maxLen; i++)
+    {
+        if (str[i] == '\0')
+            break;
+
+        hash += str[i] * (0x77 + i);
+    }
+
+    return hash ^ ((hash ^ (hash >> 10)) >> 10);
+}
+
+bool ObjLoaderT6::SupportsZone(Zone* zone)
+{
+    return zone->m_game == &g_GameT6;
+}
+
+void ObjLoaderT6::LoadIPakForZone(std::string ipakName, Zone* zone)
+{
+    printf("Loading ipak '%s' for zone '%s'\n", ipakName.c_str(), zone->m_name.c_str());
+    // TODO
+}
+
+void ObjLoaderT6::LoadReferencedContainersForZone(Zone* zone)
+{
+    auto* assetPoolT6 = dynamic_cast<GameAssetPoolT6*>(zone->GetPools());
+    const int zoneNameHash = Com_HashKey(zone->m_name.c_str(), 64);
+
+    if(assetPoolT6->m_key_value_pairs != nullptr)
+    {
+        for(auto* keyValuePairs : *assetPoolT6->m_key_value_pairs)
+        {
+            for(int variableIndex = 0; variableIndex < keyValuePairs->m_asset->numVariables; variableIndex++)
+            {
+                T6::KeyValuePair* variable = &keyValuePairs->m_asset->keyValuePairs[variableIndex];
+                
+                if(variable->namespaceHash == zoneNameHash && variable->keyHash == IPAK_READ_HASH)
+                {
+                    LoadIPakForZone(variable->value, zone);
+                }
+            }
+        }
+    }
+}
+
+void ObjLoaderT6::LoadObjDataForZone(Zone* zone)
+{
+}

src/ObjLoading/Game/T6/ObjLoaderT6.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "IObjLoader.h"
+
+class ObjLoaderT6 final : public IObjLoader
+{
+    static const int IPAK_READ_HASH;
+    static const int GLOBAL_HASH;
+    static int Com_HashKey(const char* str, int maxLen);
+
+    static void LoadIPakForZone(std::string ipakName, Zone* zone);
+
+public:
+    bool SupportsZone(Zone* zone) override;
+    void LoadReferencedContainersForZone(Zone* zone) override;
+    void LoadObjDataForZone(Zone* zone) override;
+};

src/ObjLoading/IObjLoader.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "Zone/Zone.h"
+
+class IObjLoader
+{
+public:
+    virtual ~IObjLoader() = default;
+
+    virtual bool SupportsZone(Zone* zone) = 0;
+    virtual void LoadReferencedContainersForZone(Zone* zone) = 0;
+    virtual void LoadObjDataForZone(Zone* zone) = 0;
+};

src/ObjLoading/ObjContainer/IPak/Exception/IPakLoadException.cpp

@@ -0,0 +1,16 @@
+#include "IPakLoadException.h"
+
+IPakLoadException::IPakLoadException(std::string message)
+{
+    m_message = std::move(message);
+}
+
+const std::string& IPakLoadException::DetailedMessage() const
+{
+    return m_message;
+}
+
+char const* IPakLoadException::what() const
+{
+    return "There was an error when trying to load an ipak file.";
+}

src/ObjLoading/ObjContainer/IPak/Exception/IPakLoadException.h

@@ -0,0 +1,14 @@
+#pragma once
+#include <exception>
+#include <string>
+
+class IPakLoadException final : public std::exception
+{
+    std::string m_message;
+
+public:
+    explicit IPakLoadException(std::string message);
+
+    const std::string& DetailedMessage() const;
+    char const* what() const override;
+};

src/ObjLoading/ObjContainer/IPak/IPak.cpp

@@ -0,0 +1,122 @@
+#include "IPak.h"
+#include "zlib.h"
+#include "Exception/IPakLoadException.h"
+
+#include <sstream>
+
+const uint32_t IPak::MAGIC = 'IPAK';
+const uint32_t IPak::VERSION = 0x50000;
+
+uint32_t IPak::R_HashString(const char* str, uint32_t hash)
+{
+    for (const char* pos = str; *pos; pos++)
+    {
+        hash = 33 * hash ^ (*pos | 0x20);
+    }
+
+    return hash;
+}
+
+IPak::IPak(FileAPI::IFile* file)
+{
+    m_file = file;
+    m_initialized = false;
+    m_index_section = nullptr;
+    m_data_section = nullptr;
+}
+
+IPak::~IPak()
+{
+    delete m_index_section;
+    m_index_section = nullptr;
+
+    delete m_data_section;
+    m_data_section = nullptr;
+}
+
+void IPak::ReadSection()
+{
+    IPakSection section{};
+
+    if (m_file->Read(&section, sizeof section, 1) != sizeof section)
+        throw IPakLoadException("Unexpected eof when trying to load section.");
+
+    switch (section.type)
+    {
+    case 1:
+        m_index_section = new IPakSection(section);
+        break;
+
+    case 2:
+        m_data_section = new IPakSection(section);
+        break;
+
+    default:
+        break;
+    }
+}
+
+void IPak::ReadHeader()
+{
+    IPakHeader header{};
+
+    if (m_file->Read(&header, sizeof header, 1) != sizeof header)
+        throw IPakLoadException("Unexpected eof when trying to load header.");
+
+    if (header.magic != MAGIC)
+    {
+        std::ostringstream oss;
+        oss << "Invalid magic '0x" << std::hex << header.magic << "'.";
+
+        throw IPakLoadException(oss.str());
+    }
+
+    if(header.version != VERSION)
+    {
+        std::ostringstream oss;
+        oss << "Unsupported version '" << header.version << "'.";
+
+        throw IPakLoadException(oss.str());
+    }
+
+    for(unsigned section = 0; section < header.sectionCount; section++)
+    {
+        ReadSection();
+    }
+
+    if(m_index_section == nullptr)
+    {
+        throw IPakLoadException("IPak does not contain an index section.");
+    }
+
+    if(m_data_section == nullptr)
+    {
+        throw IPakLoadException("IPak does not contain a data section.");
+    }
+}
+
+void IPak::Initialize()
+{
+    if (m_initialized)
+        return;
+
+    ReadHeader();
+
+    m_initialized = true;
+}
+
+FileAPI::IFile* IPak::GetEntryData(IPakHash nameHash, IPakHash dataHash)
+{
+    // TODO
+    return nullptr;
+}
+
+IPakHash IPak::HashString(const std::string& str)
+{
+    return R_HashString(str.c_str(), 0);
+}
+
+IPakHash IPak::HashData(const void* data, const size_t dataSize)
+{
+    return crc32(0, static_cast<const Bytef*>(data), dataSize);
+}

src/ObjLoading/ObjContainer/IPak/IPak.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "Utils/FileAPI.h"
+#include "ObjContainer/IPak/IPakTypes.h"
+#include "ObjContainer/ObjContainerReferenceable.h"
+
+#include <vector>
+
+class IPak final : public ObjContainerReferenceable
+{
+    static const uint32_t MAGIC;
+    static const uint32_t VERSION;
+
+    FileAPI::IFile* m_file;
+
+    bool m_initialized;
+
+    IPakSection* m_index_section;
+    IPakSection* m_data_section;
+
+    std::vector<IPakIndexEntry> m_index_entries;
+
+    static uint32_t R_HashString(const char* str, uint32_t hash);
+
+    void ReadSection();
+    void ReadHeader();
+
+public:
+    explicit IPak(FileAPI::IFile* file);
+    ~IPak();
+
+    void Initialize();
+    FileAPI::IFile* GetEntryData(IPakHash nameHash, IPakHash dataHash);
+
+    static IPakHash HashString(const std::string& str);
+    static IPakHash HashData(const void* data, size_t dataSize);
+};

src/ObjLoading/ObjContainer/ObjContainerRegistry.cpp

@@ -0,0 +1,52 @@
+#include "ObjContainerRegistry.h"
+#include <cassert>
+
+ObjContainerRegistry g_ObjContainerRegistry;
+
+IObjContainer* ObjContainerRegistry::GetContainerByName(const std::string& name)
+{
+    for (auto* container : m_containers)
+    {
+        if (container->GetName() == name)
+        {
+            return container;
+        }
+    }
+
+    return nullptr;
+}
+
+void ObjContainerRegistry::AddContainer(IObjContainer* container)
+{
+    assert(dynamic_cast<ObjContainerReferenceable*>(container) == nullptr);
+
+    m_containers.push_back(container);
+}
+
+void ObjContainerRegistry::AddContainerWithReference(ObjContainerReferenceable* container, Zone* referencer)
+{
+    container->AddReference(referencer);
+    m_containers.push_back(container);
+}
+
+void ObjContainerRegistry::RemoveContainerReferences(Zone* referencer)
+{
+    auto iContainer = m_containers.begin();
+
+    while (iContainer != m_containers.end())
+    {
+        auto* container = *iContainer;
+
+        if (auto* referenceableContainer = dynamic_cast<ObjContainerReferenceable*>(container))
+        {
+            if (referenceableContainer->RemoveReference(referencer) && !referenceableContainer->IsReferenced())
+            {
+                delete container;
+                iContainer = m_containers.erase(iContainer);
+                continue;
+            }
+        }
+
+        ++iContainer;
+    }
+}

src/ObjLoading/ObjContainer/ObjContainerRegistry.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "ObjContainer/IObjContainer.h"
+#include "ObjContainer/ObjContainerReferenceable.h"
+#include "Zone/Zone.h"
+
+#include <string>
+#include <vector>
+
+class ObjContainerRegistry
+{
+    std::vector<IObjContainer*> m_containers;
+
+public:
+    void AddContainer(IObjContainer* container);
+    void AddContainerWithReference(ObjContainerReferenceable* container, Zone* referencer);
+    void RemoveContainerReferences(Zone* referencer);
+
+    IObjContainer* GetContainerByName(const std::string& name);
+};
+
+extern ObjContainerRegistry g_ObjContainerRegistry;

src/ObjLoading/ObjLoading.cpp

@@ -0,0 +1,38 @@
+#include "ObjLoading.h"
+#include "IObjLoader.h"
+#include "Game/T6/ObjLoaderT6.h"
+#include "ObjContainer/ObjContainerRegistry.h"
+
+IObjLoader* objLoaders[]
+{
+    new ObjLoaderT6()
+};
+
+void ObjLoading::LoadReferencedContainersForZone(Zone* zone)
+{
+    for (auto* loader : objLoaders)
+    {
+        if (loader->SupportsZone(zone))
+        {
+            loader->LoadReferencedContainersForZone(zone);
+            return;
+        }
+    }
+}
+
+void ObjLoading::LoadObjDataForZone(Zone* zone)
+{
+    for (auto* loader : objLoaders)
+    {
+        if (loader->SupportsZone(zone))
+        {
+            loader->LoadObjDataForZone(zone);
+            return;
+        }
+    }
+}
+
+void ObjLoading::UnloadContainersOfZone(Zone* zone)
+{
+    g_ObjContainerRegistry.RemoveContainerReferences(zone);
+}

src/ObjLoading/ObjLoading.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "Zone/Zone.h"
+
+class ObjLoading
+{
+public:
+    static void LoadReferencedContainersForZone(Zone* zone);
+    static void LoadObjDataForZone(Zone* zone);
+    static void UnloadContainersOfZone(Zone* zone);
+};

src/ObjLoading/SearchPath/ISearchPath.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "Utils/FileAPI.h"
+#include <string>
+
+class ISearchPath
+{
+public:
+    virtual ~ISearchPath() = default;
+
+    virtual FileAPI::IFile* Open(const std::string& fileName) = 0;
+};

src/ObjLoading/SearchPath/SearchPaths.cpp

@@ -0,0 +1,78 @@
+#include "SearchPaths.h"
+
+SearchPaths::~SearchPaths()
+{
+    for(auto searchPath : m_search_paths)
+    {
+        delete searchPath;
+    }
+    m_search_paths.clear();
+}
+
+SearchPaths::SearchPaths(const SearchPaths& other)
+    : m_search_paths(other.m_search_paths)
+{
+    
+}
+
+SearchPaths::SearchPaths(SearchPaths&& other) noexcept
+    : m_search_paths(std::move(other.m_search_paths))
+{
+    
+}
+
+SearchPaths& SearchPaths::operator=(const SearchPaths& other)
+{
+    m_search_paths = other.m_search_paths;
+
+    return *this;
+}
+
+SearchPaths& SearchPaths::operator=(SearchPaths&& other) noexcept
+{
+    m_search_paths = std::move(other.m_search_paths);
+
+    return *this;
+}
+
+FileAPI::IFile* SearchPaths::Open(const std::string& fileName)
+{
+    for(auto searchPath : m_search_paths)
+    {
+        auto* file = searchPath->Open(fileName);
+
+        if(file != nullptr)
+        {
+            return file;
+        }
+    }
+
+    return nullptr;
+}
+
+void SearchPaths::AddSearchPath(ISearchPath* searchPath)
+{
+    m_search_paths.push_back(searchPath);
+}
+
+void SearchPaths::RemoveSearchPath(ISearchPath* searchPath)
+{
+    for(auto i = m_search_paths.begin(); i != m_search_paths.end(); ++i)
+    {
+        if(*i == searchPath)
+        {
+            m_search_paths.erase(i);
+            return;
+        }
+    }
+}
+
+SearchPaths::iterator SearchPaths::begin()
+{
+    return m_search_paths.begin();
+}
+
+SearchPaths::iterator SearchPaths::end()
+{
+    return m_search_paths.end();
+}

src/ObjLoading/SearchPath/SearchPaths.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "ISearchPath.h"
+#include <vector>
+
+class SearchPaths final : public ISearchPath
+{
+    std::vector<ISearchPath*> m_search_paths;
+
+public:
+    using iterator = std::vector<ISearchPath*>::iterator;
+
+    ~SearchPaths() override;
+    FileAPI::IFile* Open(const std::string& fileName) override;
+
+    SearchPaths(const SearchPaths& other);
+    SearchPaths(SearchPaths&& other) noexcept;
+    SearchPaths& operator=(const SearchPaths& other);
+    SearchPaths& operator=(SearchPaths&& other) noexcept;
+
+    void AddSearchPath(ISearchPath* searchPath);
+    void RemoveSearchPath(ISearchPath* searchPath);
+
+    iterator begin();
+    iterator end();
+};

src/ObjWriting.lua

@@ -1,6 +1,7 @@
 ObjWriting = {}
 
 function ObjWriting:include()
+	ObjCommon:include()
 	ZoneCommon:include()
 	includedirs {
 		path.join(ProjectFolder(), "ObjWriting")
@@ -9,7 +10,10 @@ end
 
 function ObjWriting:link()
     Utils:link()
+	ObjCommon:link()
 	ZoneCommon:link()
+	minilzo:link()
+	minizip:link()
 	links {
 		"ObjWriting"
 	}
@@ -41,4 +45,6 @@ function ObjWriting:project()
 		
         self:include()
 		Utils:include()
+		minilzo:include()
+		minizip:include()
 end

src/Unlinker.lua

@@ -31,9 +31,11 @@ function Unlinker:project()
 		self:include()
 		Utils:include()
         ZoneLoading:include()
+        ObjLoading:include()
         ObjWriting:include()
 
 		Utils:link()
         ZoneLoading:link()
+        ObjLoading:link()
         ObjWriting:link()
 end

src/Unlinker/main.cpp

@@ -6,8 +6,8 @@
 #include "Utils/PathUtils.h"
 #include "Utils/FileAPI.h"
 
-#include <cstdlib>
 #include <regex>
+#include "ObjLoading.h"
 
 const CommandLineOption* optionHelp = CommandLineOption::Builder::Create()
                                       .WithShortName("?")
@@ -148,6 +148,9 @@ int main(const int argc, const char** argv)
             return 1;
         }
 
+        ObjLoading::LoadReferencedContainersForZone(zone);
+        ObjLoading::LoadObjDataForZone(zone);
+
         if(!HandleZone(zone, &argumentParser))
         {
             return 1;