Write IPak base skeleton without data

src/Common/Game/T6/CommonT6.cpp

@@ -58,6 +58,16 @@ int Common::Com_HashString(const char* str, const int len)
     return result;
 }
 
+uint32_t Common::R_HashString(const char* str, uint32_t hash)
+{
+    for (const auto* pos = str; *pos; pos++)
+    {
+        hash = 33 * hash ^ (*pos | 0x20);
+    }
+
+    return hash;
+}
+
 PackedTexCoords Common::Vec2PackTexCoords(const vec2_t* in)
 {
     return PackedTexCoords{ Pack32::Vec2PackTexCoords(in->v) };

src/Common/Game/T6/CommonT6.h

@@ -10,6 +10,7 @@ namespace T6
         static int Com_HashKey(const char* str, int maxLen);
         static int Com_HashString(const char* str);
         static int Com_HashString(const char* str, int len);
+        static uint32_t R_HashString(const char* str, uint32_t hash);
 
         static PackedTexCoords Vec2PackTexCoords(const vec2_t* in);
         static PackedUnitVec Vec3PackUnitVec(const vec3_t* in);

src/Linker/Game/IW3/ZoneCreatorIW3.cpp

@@ -72,7 +72,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
         if (!assetEntry.m_is_reference)
             continue;
 
-        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name);
+        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
     }
 
     const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

src/Linker/Game/IW4/ZoneCreatorIW4.cpp

@@ -71,7 +71,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
         if (!assetEntry.m_is_reference)
             continue;
 
-        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name);
+        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
     }
 
     const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

src/Linker/Game/IW5/ZoneCreatorIW5.cpp

@@ -71,7 +71,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
         if (!assetEntry.m_is_reference)
             continue;
 
-        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name);
+        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
     }
 
     const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

src/Linker/Game/T5/ZoneCreatorT5.cpp

@@ -72,7 +72,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
         if (!assetEntry.m_is_reference)
             continue;
 
-        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name);
+        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
     }
 
     const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

src/Linker/Game/T6/ZoneCreatorT6.cpp

@@ -126,7 +126,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
         if (!assetEntry.m_is_reference)
             continue;
 
-        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name);
+        context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
     }
 
     const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

src/Linker/Linker.cpp

@@ -23,6 +23,7 @@
 #include "Game/IW5/ZoneCreatorIW5.h"
 #include "Game/T5/ZoneCreatorT5.h"
 #include "Game/T6/ZoneCreatorT6.h"
+#include "ObjContainer/IPak/IPakWriter.h"
 
 #include "Utils/ObjFileStream.h"
 #include "Utils/StringUtils.h"
@@ -141,7 +142,7 @@ class LinkerImpl final : public Linker
             {
                 for (const auto& entry : zoneDefinition->m_assets)
                 {
-                    assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name);
+                    assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name, entry.m_is_reference);
                 }
                 return true;
             }
@@ -414,9 +415,39 @@ class LinkerImpl final : public Linker
         return result;
     }
 
-    bool BuildIPak(const std::string& projectName, ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& sourceSearchPaths)
+    bool BuildIPak(const std::string& projectName, const ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& sourceSearchPaths) const
     {
-        return false;
+        const fs::path ipakFolderPath(m_args.GetOutputFolderPathForProject(projectName));
+        auto ipakFilePath(ipakFolderPath);
+        ipakFilePath.append(zoneDefinition.m_name + ".ipak");
+
+        fs::create_directories(ipakFolderPath);
+
+        std::ofstream stream(ipakFilePath, std::fstream::out | std::fstream::binary);
+        if (!stream.is_open())
+            return false;
+
+        const auto ipakWriter = IPakWriter::Create(stream, &assetSearchPaths);
+        for (const auto& assetEntry : zoneDefinition.m_assets)
+        {
+            if (assetEntry.m_is_reference)
+                continue;
+
+            if (assetEntry.m_asset_type == "image")
+                ipakWriter->AddImage(assetEntry.m_asset_name);
+        }
+
+        if (!ipakWriter->Write())
+        {
+            std::cout << "Writing ipak failed." << std::endl;
+            stream.close();
+            return false;
+        }
+
+        std::cout << "Created ipak \"" << ipakFilePath.string() << "\"\n";
+
+        stream.close();
+        return true;
     }
 
     bool BuildProject(const std::string& projectName)

src/ObjCommon/ObjContainer/IPak/IPakTypes.h

@@ -2,65 +2,78 @@
 
 #include <cstdint>
 
-typedef uint32_t IPakHash;
+#include "Utils/FileUtils.h"
 
 namespace ipak_consts
 {
-	static constexpr size_t IPAK_CHUNK_SIZE = 0x8000;
+    static constexpr uint32_t IPAK_MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I');
-	static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8;
+    static constexpr uint32_t IPAK_VERSION = 0x50000;
+
+    static constexpr uint32_t IPAK_INDEX_SECTION = 1;
+    static constexpr uint32_t IPAK_DATA_SECTION = 2;
+    static constexpr uint32_t IPAK_BRANDING_SECTION = FileUtils::MakeMagic32('M', 'E', 'T', 'A');
+
+    static constexpr size_t IPAK_CHUNK_SIZE = 0x8000;
+    static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8;
 }
 
+typedef uint32_t IPakHash;
+
 struct IPakHeader
 {
-	uint32_t magic;
+    uint32_t magic;
-	uint32_t version;
+    uint32_t version;
-	uint32_t size;
+    uint32_t size;
-	uint32_t sectionCount;
+    uint32_t sectionCount;
 };
 
 struct IPakSection
 {
-	uint32_t type;
+    uint32_t type;
-	uint32_t offset;
+    uint32_t offset;
-	uint32_t size;
+    uint32_t size;
-	uint32_t itemCount;
+    uint32_t itemCount;
 };
 
 union IPakIndexEntryKey
 {
-	struct
+    struct
-	{
+    {
-		IPakHash dataHash;
+        IPakHash dataHash;
-		IPakHash nameHash;
+        IPakHash nameHash;
-	};
+    };
-	uint64_t combinedKey;
+
+    uint64_t combinedKey;
 };
 
 struct IPakIndexEntry
 {
-	IPakIndexEntryKey key;
+    IPakIndexEntryKey key;
-	uint32_t offset;
+    uint32_t offset;
-	uint32_t size;
+    uint32_t size;
 };
 
 struct IPakDataBlockHeader
 {
-	union
+    union
-	{
+    {
-		uint32_t countAndOffset;
+        uint32_t countAndOffset;
-		struct
+
-		{
+        struct
-			uint32_t offset : 24;
+        {
-			uint32_t count : 8;
+            uint32_t offset : 24;
-		};
+            uint32_t count : 8;
-	};
+        };
-	union
+    };
-	{
+
-		uint32_t commands[31];
+    union
-		struct
+    {
-		{
+        uint32_t commands[31];
-			uint32_t size : 24;
+
-			uint32_t compressed : 8;
+        struct
-		}_commands[31];
+        {
-	};
+            uint32_t size : 24;
-};
+            uint32_t compressed : 8;
+        } _commands[31];
+    };
+};

src/ObjLoading/ObjContainer/IPak/IPak.cpp

@@ -18,9 +18,6 @@ ObjContainerRepository<IPak, Zone> IPak::Repository;
 
 class IPak::Impl : public ObjContainerReferenceable
 {
-    static const uint32_t MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I');
-    static const uint32_t VERSION = 0x50000;
-
     std::string m_path;
     std::unique_ptr<std::istream> m_stream;
 
@@ -82,11 +79,11 @@ class IPak::Impl : public ObjContainerReferenceable
 
         switch (section.type)
         {
-        case 1:
+        case ipak_consts::IPAK_INDEX_SECTION:
             m_index_section = std::make_unique<IPakSection>(section);
             break;
 
-        case 2:
+        case ipak_consts::IPAK_DATA_SECTION:
             m_data_section = std::make_unique<IPakSection>(section);
             break;
 
@@ -108,13 +105,13 @@ class IPak::Impl : public ObjContainerReferenceable
             return false;
         }
 
-        if (header.magic != MAGIC)
+        if (header.magic != ipak_consts::IPAK_MAGIC)
         {
             printf("Invalid ipak magic '0x%x'.\n", header.magic);
             return false;
         }
 
-        if (header.version != VERSION)
+        if (header.version != ipak_consts::IPAK_VERSION)
         {
             printf("Unsupported ipak version '%u'.\n", header.version);
             return false;

src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp

@@ -0,0 +1,241 @@
+#include "IPakWriter.h"
+
+#include <algorithm>
+#include <iostream>
+#include <sstream>
+#include <zlib.h>
+
+#include "Game/T6/CommonT6.h"
+#include "Game/T6/GameT6.h"
+#include "ObjContainer/IPak/IPakTypes.h"
+#include "Utils/Alignment.h"
+
+class IPakWriterImpl final : public IPakWriter
+{
+    static constexpr char BRANDING[] = "Created with OAT - OpenAssetTools";
+    static constexpr auto SECTION_COUNT = 3; // Index + Data + Branding
+
+    inline static const std::string PAD_DATA = std::string(256, '\xA7');
+
+public:
+    explicit IPakWriterImpl(std::ostream& stream, ISearchPath* assetSearchPath)
+        : m_stream(stream),
+          m_asset_search_path(assetSearchPath),
+          m_current_offset(0),
+          m_total_size(0),
+          m_data_section_offset(0),
+          m_data_section_size(0u),
+          m_index_section_offset(0),
+          m_branding_section_offset(0)
+    {
+    }
+
+    void AddImage(std::string imageName) override
+    {
+        m_images.emplace_back(std::move(imageName));
+    }
+
+    void GoTo(const int64_t offset)
+    {
+        m_stream.seekp(offset, std::ios::beg);
+        m_current_offset = offset;
+    }
+
+    void Write(const void* data, const size_t dataSize)
+    {
+        m_stream.write(static_cast<const char*>(data), dataSize);
+        m_current_offset += dataSize;
+    }
+
+    void Pad(const size_t paddingSize)
+    {
+        auto paddingSizeLeft = paddingSize;
+        while (paddingSizeLeft > 0)
+        {
+            const auto writeSize = std::min(paddingSizeLeft, PAD_DATA.size());
+            Write(PAD_DATA.data(), writeSize);
+
+            paddingSizeLeft -= writeSize;
+        }
+    }
+
+    void AlignToChunkWrite()
+    {
+        Pad(static_cast<size_t>(utils::Align(m_current_offset, 0x8000i64) - m_current_offset));
+    }
+
+    void WriteHeaderData()
+    {
+        GoTo(0);
+
+        const IPakHeader header{
+            ipak_consts::IPAK_MAGIC,
+            ipak_consts::IPAK_VERSION,
+            static_cast<uint32_t>(m_total_size),
+            SECTION_COUNT
+        };
+
+        const IPakSection dataSection{
+            ipak_consts::IPAK_DATA_SECTION,
+            static_cast<uint32_t>(m_data_section_offset),
+            static_cast<uint32_t>(m_data_section_size),
+            static_cast<uint32_t>(m_index_entries.size())
+        };
+
+        const IPakSection indexSection{
+            ipak_consts::IPAK_INDEX_SECTION,
+            static_cast<uint32_t>(m_index_section_offset),
+            static_cast<uint32_t>(sizeof(IPakIndexEntry) * m_index_entries.size()),
+            static_cast<uint32_t>(m_index_entries.size())
+        };
+
+        const IPakSection brandingSection{
+            ipak_consts::IPAK_BRANDING_SECTION,
+            static_cast<uint32_t>(m_branding_section_offset),
+            std::extent_v<decltype(BRANDING)>,
+            1
+        };
+
+        Write(&header, sizeof(header));
+        Write(&dataSection, sizeof(dataSection));
+        Write(&indexSection, sizeof(indexSection));
+        Write(&brandingSection, sizeof(brandingSection));
+    }
+
+    static std::string ImageFileName(const std::string& imageName)
+    {
+        std::ostringstream ss;
+        ss << "images/" << imageName << ".iwi";
+
+        return ss.str();
+    }
+
+    std::unique_ptr<char[]> ReadImageDataFromSearchPath(const std::string& imageName, size_t& imageSize) const
+    {
+        const auto fileName = ImageFileName(imageName);
+
+        const auto openFile = m_asset_search_path->Open(fileName);
+        if (!openFile.IsOpen())
+        {
+            std::cerr << "Could not open image for writing to IPak \"" << fileName << "\"\n";
+            return nullptr;
+        }
+
+        imageSize = static_cast<size_t>(openFile.m_length);
+        auto imageData = std::make_unique<char[]>(imageSize);
+        openFile.m_stream->read(imageData.get(), imageSize);
+
+        return imageData;
+    }
+
+    bool WriteImageData(const std::string& imageName)
+    {
+        size_t imageSize;
+        const auto imageData = ReadImageDataFromSearchPath(imageName, imageSize);
+        if (!imageData)
+            return false;
+
+        const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0);
+        const auto dataHash = static_cast<unsigned>(crc32(0u, reinterpret_cast<const Bytef*>(imageData.get()), imageSize));
+
+        IPakIndexEntry indexEntry{};
+        indexEntry.key.nameHash = nameHash;
+        indexEntry.key.dataHash = dataHash & 0x1FFFFFFF;
+        indexEntry.offset = static_cast<uint32_t>(m_current_offset - m_data_section_offset);
+
+        size_t writtenImageSize = 0u;
+
+        // TODO: Write image data
+
+        indexEntry.size = writtenImageSize;
+        m_index_entries.emplace_back(indexEntry);
+
+        m_data_section_size += writtenImageSize;
+        return true;
+    }
+
+    bool WriteDataSection()
+    {
+        AlignToChunkWrite();
+        m_data_section_offset = m_current_offset;
+        m_data_section_size = 0u;
+
+        m_index_entries.reserve(m_images.size());
+
+        return std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName)
+        {
+            return WriteImageData(imageName);
+        });
+    }
+
+    static bool CompareIndices(const IPakIndexEntry& entry1, const IPakIndexEntry& entry2)
+    {
+        return entry1.key.combinedKey < entry2.key.combinedKey;
+    }
+
+    void SortIndexSectionEntries()
+    {
+        std::sort(m_index_entries.begin(), m_index_entries.end(), CompareIndices);
+    }
+
+    void WriteIndexSection()
+    {
+        AlignToChunkWrite();
+        m_index_section_offset = m_current_offset;
+
+        SortIndexSectionEntries();
+
+        for (const auto& indexEntry : m_index_entries)
+            Write(&indexEntry, sizeof(indexEntry));
+    }
+
+    void WriteBrandingSection()
+    {
+        AlignToChunkWrite();
+        m_branding_section_offset = m_current_offset;
+
+        Write(BRANDING, std::extent_v<decltype(BRANDING)>);
+    }
+
+    void WriteFileEnding()
+    {
+        AlignToChunkWrite();
+        m_total_size = m_current_offset;
+    }
+
+    bool Write() override
+    {
+        // We will write the header and sections later since they need complementary data
+        GoTo(sizeof(IPakHeader) + sizeof(IPakSection) * SECTION_COUNT);
+        AlignToChunkWrite();
+
+        if (!WriteDataSection())
+            return false;
+
+        WriteIndexSection();
+        WriteBrandingSection();
+        WriteFileEnding();
+
+        WriteHeaderData();
+
+        return true;
+    }
+
+private:
+    std::ostream& m_stream;
+    ISearchPath* m_asset_search_path;
+    std::vector<std::string> m_images;
+
+    int64_t m_current_offset;
+    std::vector<IPakIndexEntry> m_index_entries;
+    int64_t m_total_size;
+    int64_t m_data_section_offset;
+    size_t m_data_section_size;
+    int64_t m_index_section_offset;
+    int64_t m_branding_section_offset;
+};
+
+std::unique_ptr<IPakWriter> IPakWriter::Create(std::ostream& stream, ISearchPath* assetSearchPath)
+{
+    return std::make_unique<IPakWriterImpl>(stream, assetSearchPath);
+}

src/ObjWriting/ObjContainer/IPak/IPakWriter.h

@@ -0,0 +1,22 @@
+#pragma once
+#include <memory>
+#include <ostream>
+
+#include "SearchPath/ISearchPath.h"
+
+class IPakWriter
+{
+public:
+    IPakWriter() = default;
+    virtual ~IPakWriter() = default;
+
+    IPakWriter(const IPakWriter& other) = default;
+    IPakWriter(IPakWriter&& other) noexcept = default;
+    IPakWriter& operator=(const IPakWriter& other) = default;
+    IPakWriter& operator=(IPakWriter&& other) noexcept = default;
+
+    virtual void AddImage(std::string imageName) = 0;
+    virtual bool Write() = 0;
+
+    static std::unique_ptr<IPakWriter> Create(std::ostream& stream, ISearchPath* assetSearchPath);
+};

src/ZoneCommon/Zone/AssetList/AssetList.cpp

@@ -1,10 +1,13 @@
 #include "AssetList.h"
 
 AssetListEntry::AssetListEntry()
-= default;
+    : m_is_reference(false)
-
+{
-AssetListEntry::AssetListEntry(std::string type, std::string name)
+}
-    : m_type(std::move(type)),
+
-      m_name(std::move(name))
+AssetListEntry::AssetListEntry(std::string type, std::string name, const bool isReference)
+    : m_type(std::move(type)),
+      m_name(std::move(name)),
+      m_is_reference(isReference)
 {
 }

src/ZoneCommon/Zone/AssetList/AssetList.h

@@ -7,9 +7,10 @@ class AssetListEntry
 public:
     std::string m_type;
     std::string m_name;
+    bool m_is_reference;
 
     AssetListEntry();
-    AssetListEntry(std::string type, std::string name);
+    AssetListEntry(std::string type, std::string name, bool isReference);
 };
 
 class AssetList

src/ZoneCommon/Zone/AssetList/AssetListStream.cpp

@@ -9,7 +9,7 @@ bool AssetListInputStream::NextEntry(AssetListEntry& entry) const
 {
     std::vector<std::string> row;
 
-    while(true)
+    while (true)
     {
         if (!m_stream.NextRow(row))
             return false;
@@ -18,8 +18,17 @@ bool AssetListInputStream::NextEntry(AssetListEntry& entry) const
             continue;
 
         entry.m_type = row[0];
-        if (row.size() >= 2)
+        if (row.size() >= 3 && row[1].empty())
+        {
+            entry.m_name = row[2];
+            entry.m_is_reference = true;
+        }
+        else
+        {
             entry.m_name = row[1];
+            entry.m_is_reference = false;
+        }
+
         return true;
     }
 }