Add data writing for IPaks

src/ObjCommon/ObjContainer/IPak/IPakTypes.h

@@ -16,7 +16,12 @@ namespace ipak_consts
     static constexpr size_t IPAK_CHUNK_SIZE = 0x8000;
     static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8;
 
+    static constexpr uint32_t IPAK_COMMAND_DEFAULT_SIZE = 0x7F00;
+    static constexpr uint32_t IPAK_COMMAND_UNCOMPRESSED = 0;
+    static constexpr uint32_t IPAK_COMMAND_COMPRESSED = 1;
     static constexpr uint32_t IPAK_COMMAND_SKIP = 0xCF;
+
+    static_assert(IPAK_COMMAND_DEFAULT_SIZE <= IPAK_CHUNK_SIZE);
 }
 
 typedef uint32_t IPakHash;

src/ObjLoading/ObjContainer/IPak/IPak.cpp

@@ -70,8 +70,8 @@ class IPak::Impl : public ObjContainerReferenceable
     {
         IPakSection section{};
 
-        m_stream->read(reinterpret_cast<char*>(&section), sizeof section);
+        m_stream->read(reinterpret_cast<char*>(&section), sizeof(section));
-        if (m_stream->gcount() != sizeof section)
+        if (m_stream->gcount() != sizeof(section))
         {
             printf("Unexpected eof when trying to load section.\n");
             return false;
@@ -98,7 +98,7 @@ class IPak::Impl : public ObjContainerReferenceable
     {
         IPakHeader header{};
 
-        m_stream->read(reinterpret_cast<char*>(&header), sizeof header);
+        m_stream->read(reinterpret_cast<char*>(&header), sizeof(header));
         if (m_stream->gcount() != sizeof header)
         {
             printf("Unexpected eof when trying to load header.\n");

src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp

@@ -2,6 +2,7 @@
 
 #include <algorithm>
 #include <iostream>
+#include <minilzo.h>
 #include <sstream>
 #include <zlib.h>
 
@@ -26,8 +27,14 @@ public:
           m_data_section_offset(0),
           m_data_section_size(0u),
           m_index_section_offset(0),
-          m_branding_section_offset(0)
+          m_branding_section_offset(0),
+          m_file_offset(0u),
+          m_chunk_buffer_window_start(0),
+          m_current_block{},
+          m_current_block_header_offset(0)
     {
+        m_decompressed_buffer = std::make_unique<char[]>(ipak_consts::IPAK_CHUNK_SIZE);
+        m_lzo_work_buffer = std::make_unique<char[]>(LZO1X_1_MEM_COMPRESS);
     }
 
     void AddImage(std::string imageName) override
@@ -59,9 +66,14 @@ public:
         }
     }
 
-    void AlignToChunkWrite()
+    void AlignToChunk()
     {
-        Pad(static_cast<size_t>(utils::Align(m_current_offset, 0x8000i64) - m_current_offset));
+        Pad(static_cast<size_t>(utils::Align(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE)) - m_current_offset));
+    }
+
+    void AlignToBlockHeader()
+    {
+        Pad(static_cast<size_t>(utils::Align(m_current_offset, static_cast<int64_t>(sizeof(IPakDataBlockHeader))) - m_current_offset));
     }
 
     void WriteHeaderData()
@@ -128,6 +140,106 @@ public:
         return imageData;
     }
 
+    void FlushBlock()
+    {
+        if (m_current_block_header_offset > 0)
+        {
+            const auto previousOffset = m_current_offset;
+
+            GoTo(m_current_block_header_offset);
+            Write(&m_current_block, sizeof(m_current_block));
+            GoTo(previousOffset);
+        }
+    }
+
+    void FlushChunk()
+    {
+        FlushBlock();
+        AlignToBlockHeader();
+
+        const auto nextChunkOffset = utils::Align(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE));
+        const auto sizeToSkip = static_cast<size_t>(nextChunkOffset - m_current_offset);
+
+        if (sizeToSkip >= sizeof(IPakDataBlockHeader))
+        {
+            IPakDataBlockHeader skipBlockHeader{};
+            skipBlockHeader.countAndOffset.count = 1;
+            skipBlockHeader.commands[0].compressed = ipak_consts::IPAK_COMMAND_SKIP;
+            skipBlockHeader.commands[0].size = sizeToSkip - sizeof(IPakDataBlockHeader);
+            Write(&skipBlockHeader, sizeof(skipBlockHeader));
+        }
+
+        AlignToChunk();
+        m_chunk_buffer_window_start = m_current_offset;
+    }
+
+    void StartNewBlock()
+    {
+        AlignToBlockHeader();
+        m_current_block_header_offset = m_current_offset;
+        m_current_block = {};
+        m_current_block.countAndOffset.offset = static_cast<uint32_t>(m_file_offset);
+    }
+
+    void WriteChunkData(const void* data, const size_t dataSize)
+    {
+        auto dataOffset = 0u;
+        while (dataOffset < dataSize)
+        {
+            if (m_current_block.countAndOffset.count >= std::extent_v<decltype(IPakDataBlockHeader::commands)>)
+            {
+                FlushBlock();
+                StartNewBlock();
+            }
+
+            const auto remainingSize = dataSize - dataOffset;
+            const auto remainingChunkBufferWindowSize = std::max((ipak_consts::IPAK_CHUNK_COUNT_PER_READ * ipak_consts::IPAK_CHUNK_SIZE)
+                                                                 - static_cast<size_t>(m_current_offset - m_chunk_buffer_window_start), 0u);
+            const auto commandSize = std::min(std::min(remainingSize, ipak_consts::IPAK_COMMAND_DEFAULT_SIZE), remainingChunkBufferWindowSize);
+
+            auto writeUncompressed = true;
+            if (USE_COMPRESSION)
+            {
+                auto outLen = static_cast<lzo_uint>(ipak_consts::IPAK_CHUNK_SIZE);
+                const auto result = lzo1x_1_compress(&static_cast<const unsigned char*>(data)[dataOffset], commandSize, reinterpret_cast<unsigned char*>(m_decompressed_buffer.get()), &outLen,
+                                                     m_lzo_work_buffer.get());
+
+                if (result == LZO_E_OK && outLen < commandSize)
+                {
+                    writeUncompressed = false;
+                    Write(m_decompressed_buffer.get(), outLen);
+
+                    const auto currentCommand = m_current_block.countAndOffset.count;
+                    m_current_block.commands[currentCommand].size = static_cast<uint32_t>(outLen);
+                    m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_COMPRESSED;
+                    m_current_block.countAndOffset.count = currentCommand + 1u;
+                }
+            }
+
+            if (writeUncompressed)
+            {
+                Write(data, dataSize);
+
+                const auto currentCommand = m_current_block.countAndOffset.count;
+                m_current_block.commands[currentCommand].size = commandSize;
+                m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_UNCOMPRESSED;
+                m_current_block.countAndOffset.count = currentCommand + 1u;
+            }
+
+            dataOffset += commandSize;
+        }
+
+        m_file_offset += dataSize;
+    }
+
+    void StartNewFile()
+    {
+        FlushBlock();
+        StartNewBlock();
+        m_file_offset = 0u;
+        m_chunk_buffer_window_start = utils::AlignToPrevious(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE));
+    }
+
     bool WriteImageData(const std::string& imageName)
     {
         size_t imageSize;
@@ -138,34 +250,39 @@ public:
         const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0);
         const auto dataHash = static_cast<unsigned>(crc32(0u, reinterpret_cast<const Bytef*>(imageData.get()), imageSize));
 
-        IPakIndexEntry indexEntry{};
+        IPakIndexEntry indexEntry;
         indexEntry.key.nameHash = nameHash;
         indexEntry.key.dataHash = dataHash & 0x1FFFFFFF;
         indexEntry.offset = static_cast<uint32_t>(m_current_offset - m_data_section_offset);
 
-        size_t writtenImageSize = 0u;
+        StartNewFile();
-
+        const auto startOffset = m_current_offset;
-        // TODO: Write image data
+        WriteChunkData(imageData.get(), imageSize);
+        const auto writtenImageSize = static_cast<size_t>(m_current_offset - startOffset);
 
         indexEntry.size = writtenImageSize;
         m_index_entries.emplace_back(indexEntry);
 
-        m_data_section_size += writtenImageSize;
         return true;
     }
 
     bool WriteDataSection()
     {
-        AlignToChunkWrite();
+        AlignToChunk();
         m_data_section_offset = m_current_offset;
         m_data_section_size = 0u;
 
         m_index_entries.reserve(m_images.size());
 
-        return std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName)
+        const auto result = std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName)
         {
             return WriteImageData(imageName);
         });
+
+        FlushBlock();
+        m_data_section_size = static_cast<size_t>(m_current_offset - m_data_section_offset);
+
+        return result;
     }
 
     static bool CompareIndices(const IPakIndexEntry& entry1, const IPakIndexEntry& entry2)
@@ -180,7 +297,7 @@ public:
 
     void WriteIndexSection()
     {
-        AlignToChunkWrite();
+        AlignToChunk();
         m_index_section_offset = m_current_offset;
 
         SortIndexSectionEntries();
@@ -191,7 +308,7 @@ public:
 
     void WriteBrandingSection()
     {
-        AlignToChunkWrite();
+        AlignToChunk();
         m_branding_section_offset = m_current_offset;
 
         Write(BRANDING, std::extent_v<decltype(BRANDING)>);
@@ -199,7 +316,7 @@ public:
 
     void WriteFileEnding()
     {
-        AlignToChunkWrite();
+        AlignToChunk();
         m_total_size = m_current_offset;
     }
 
@@ -207,7 +324,7 @@ public:
     {
         // We will write the header and sections later since they need complementary data
         GoTo(sizeof(IPakHeader) + sizeof(IPakSection) * SECTION_COUNT);
-        AlignToChunkWrite();
+        AlignToChunk();
 
         if (!WriteDataSection())
             return false;
@@ -233,6 +350,13 @@ private:
     size_t m_data_section_size;
     int64_t m_index_section_offset;
     int64_t m_branding_section_offset;
+
+    std::unique_ptr<char[]> m_decompressed_buffer;
+    std::unique_ptr<char[]> m_lzo_work_buffer;
+    size_t m_file_offset;
+    int64_t m_chunk_buffer_window_start;
+    IPakDataBlockHeader m_current_block;
+    int64_t m_current_block_header_offset;
 };
 
 std::unique_ptr<IPakWriter> IPakWriter::Create(std::ostream& stream, ISearchPath* assetSearchPath)

src/ObjWriting/ObjContainer/IPak/IPakWriter.h

@@ -7,6 +7,8 @@
 class IPakWriter
 {
 public:
+    static constexpr auto USE_COMPRESSION = true;
+
     IPakWriter() = default;
     virtual ~IPakWriter() = default;
 

src/Utils/Utils/Alignment.h

@@ -3,10 +3,18 @@
 namespace utils
 {
     template <typename T>
-    constexpr T Align(T value, T toNext)
+    constexpr T Align(const T value, const T toNext)
     {
         if (toNext > 0)
             return (value + toNext - 1) / toNext * toNext;
         return value;
     }
+
+    template <typename T>
+    constexpr T AlignToPrevious(const T value, const T toPrevious)
+    {
+        if (toPrevious > 0)
+            return value / toPrevious * toPrevious;
+        return value;
+    }
 }