Merge pull request #797 from Laupetin/feature/ipak-dumping-without-ff

feat: ipak dumping without ff

premake5.lua

@@ -136,6 +136,7 @@ include "src/RawTemplater.lua"
 include "src/UnlinkerCli.lua"
 include "src/Unlinking.lua"
 include "src/Utils.lua"
+include "src/XMemCompress.lua"
 include "src/ZoneCode.lua"
 include "src/ZoneCodeGeneratorLib.lua"
 include "src/ZoneCodeGenerator.lua"
@@ -156,6 +157,7 @@ group "Components"
     Cryptography:project()
     Parser:project()
     Utils:project()
+    XMemCompress:project()
     ZoneCode:project()
     ZoneCodeGeneratorLib:project()
     ZoneCommon:project()

src/ImageConverter.lua

@@ -39,12 +39,14 @@ function ImageConverter:project()
 		self:include(includes)
 		Utils:include(includes)
 		Common:include(includes)
+        ObjLoading:include(includes)
         ObjImage:include(includes)
 
 		Raw:use()
 
 		links:linkto(Utils)
 		links:linkto(Common)
+		links:linkto(ObjLoading)
 		links:linkto(ObjImage)
 		links:linkall()
 end

src/ImageConverter/ImageConverter.cpp

@@ -8,12 +8,13 @@
 #include "Image/IwiWriter27.h"
 #include "Image/IwiWriter6.h"
 #include "Image/IwiWriter8.h"
-#include "Image/Texture.h"
 #include "ImageConverterArgs.h"
+#include "ObjContainer/IPak/IPak.h"
 #include "Utils/Logging/Log.h"
 #include "Utils/StringUtils.h"
 
 #include <assert.h>
+#include <cstring>
 #include <filesystem>
 #include <format>
 #include <fstream>
@@ -27,36 +28,26 @@ namespace
 {
     constexpr auto EXTENSION_IWI = ".iwi";
     constexpr auto EXTENSION_DDS = ".dds";
+    constexpr auto EXTENSION_IPAK = ".ipak";
+    constexpr char IWI_MAGIC[3]{'I', 'W', 'i'};
 
-    class ImageConverterImpl final : public ImageConverter
+    const char* DetermineExtensionForBytes(const void* bytes, const size_t byteCount)
+    {
+        if (byteCount >= sizeof(IWI_MAGIC) && !memcmp(bytes, IWI_MAGIC, sizeof(IWI_MAGIC)))
+            return ".iwi";
+
+        return "";
+    }
+
+    class ImageConverter
     {
     public:
-        ImageConverterImpl()
+        explicit ImageConverter(const ImageConverterArgs& args)
-            : m_game_to_convert_to(std::nullopt)
+            : m_game_to_convert_to(args.m_game_to_convert_to)
         {
         }
 
-        bool Start(const int argc, const char** argv) override
+        void HandleFile(const std::string& file)
-        {
-            con::init();
-
-            auto shouldContinue = true;
-            if (!m_args.ParseArgs(argc, argv, shouldContinue))
-                return false;
-
-            if (!shouldContinue)
-                return true;
-
-            m_game_to_convert_to = m_args.m_game_to_convert_to;
-
-            for (const auto& file : m_args.m_files_to_convert)
-                Convert(file);
-
-            return true;
-        }
-
-    private:
-        void Convert(const std::string& file)
         {
             const fs::path filePath(file);
             auto extension = filePath.extension().string();
@@ -66,10 +57,13 @@ namespace
                 ConvertIwi(filePath);
             else if (extension == EXTENSION_DDS)
                 ConvertDds(filePath);
+            else if (extension == EXTENSION_IPAK)
+                ExtractIpak(filePath);
             else
                 con::error("Unsupported extension {}", extension);
         }
 
+    private:
         bool ConvertIwi(const fs::path& iwiPath)
         {
             std::ifstream file(iwiPath, std::ios::in | std::ios::binary);
@@ -195,6 +189,66 @@ namespace
             return true;
         }
 
+        bool ExtractIpak(const fs::path& ipakPath)
+        {
+            auto file = std::make_unique<std::ifstream>(ipakPath, std::ios::in | std::ios::binary);
+            if (!file->is_open())
+            {
+                con::error("Failed to open ipak {}", ipakPath.string());
+                return false;
+            }
+
+            auto ipak = IIPak::Create(ipakPath.string(), std::move(file));
+            if (!ipak->Initialize())
+            {
+                con::error("Failed to read ipak {}", ipakPath.string());
+                return false;
+            }
+
+            const auto outDir = fs::absolute(ipakPath).parent_path() / ipakPath.filename().replace_extension();
+            fs::create_directories(outDir);
+
+            for (const auto& indexEntry : ipak->GetIndexEntries())
+            {
+                const auto baseFileName = std::format("{:0>6x}_{:0>6x}", indexEntry.key.dataHash, indexEntry.key.nameHash);
+
+                auto entryStream = ipak->GetEntryStream(indexEntry.key.nameHash, indexEntry.key.dataHash);
+                if (!entryStream)
+                {
+                    con::error("Failed to open entry stream for {}", baseFileName);
+                    return false;
+                }
+
+                char buffer[0x2000];
+                entryStream->read(buffer, sizeof(buffer));
+                auto readCount = entryStream->gcount();
+
+                const auto extension = DetermineExtensionForBytes(buffer, static_cast<size_t>(readCount));
+                const auto fileNameWithExtension = std::format("{}{}", baseFileName, extension);
+                std::ofstream outFile(outDir / fileNameWithExtension, std::ios::out | std::ios::binary);
+                if (!outFile.is_open())
+                {
+                    con::error("Failed to open ipak file {}", fileNameWithExtension);
+                    return false;
+                }
+
+                while (readCount > 0)
+                {
+                    outFile.write(buffer, readCount);
+
+                    entryStream->read(buffer, sizeof(buffer));
+                    readCount = entryStream->gcount();
+                }
+
+                entryStream->close();
+                outFile.close();
+
+                con::info("Dumped {}", fileNameWithExtension);
+            }
+
+            return true;
+        }
+
         ImageConverterArgs m_args;
         std::optional<GameId> m_game_to_convert_to;
         DdsWriter m_dds_writer;
@@ -202,7 +256,21 @@ namespace
     };
 } // namespace
 
-std::unique_ptr<ImageConverter> ImageConverter::Create()
+bool RunImageConverter(const int argc, const char** argv)
 {
-    return std::make_unique<ImageConverterImpl>();
+    con::init();
+
+    auto shouldContinue = true;
+    ImageConverterArgs args;
+    if (!args.ParseArgs(argc, argv, shouldContinue))
+        return false;
+
+    if (!shouldContinue)
+        return true;
+
+    ImageConverter imageConverter(args);
+    for (const auto& file : args.m_files_to_convert)
+        imageConverter.HandleFile(file);
+
+    return true;
 }

src/ImageConverter/ImageConverter.h

@@ -1,24 +1,3 @@
 #pragma once
-#include <memory>
 
-class ImageConverter
+bool RunImageConverter(int argc, const char** argv);
-{
-public:
-    ImageConverter() = default;
-    virtual ~ImageConverter() = default;
-
-    ImageConverter(const ImageConverter& other) = delete;
-    ImageConverter(ImageConverter&& other) noexcept = delete;
-    ImageConverter& operator=(const ImageConverter& other) = delete;
-    ImageConverter& operator=(ImageConverter&& other) noexcept = delete;
-
-    /**
-     * \brief Starts the ImageConverter application logic.
-     * \param argc The amount of command line arguments specified.
-     * \param argv The command line arguments.
-     * \return \c true if the application was successful or \c false if an error occurred.
-     */
-    virtual bool Start(int argc, const char** argv) = 0;
-
-    static std::unique_ptr<ImageConverter> Create();
-};

src/ImageConverter/main.cpp

@@ -2,7 +2,5 @@
 
 int main(const int argc, const char** argv)
 {
-    const auto imageConverter = ImageConverter::Create();
+    return RunImageConverter(argc, argv) ? 0 : 1;
-
-    return imageConverter->Start(argc, argv) ? 0 : 1;
 }

src/ObjCommon/ObjContainer/IPak/IPakTypes.h

@@ -6,7 +6,8 @@
 
 namespace ipak_consts
 {
-    static constexpr uint32_t IPAK_MAGIC = utils::MakeMagic32('K', 'A', 'P', 'I');
+    static constexpr uint32_t IPAK_MAGIC_LITTLE_ENDIAN = utils::MakeMagic32('K', 'A', 'P', 'I');
+    static constexpr uint32_t IPAK_MAGIC_BIG_ENDIAN = utils::MakeMagic32('I', 'P', 'A', 'K');
     static constexpr uint32_t IPAK_VERSION = 0x50000;
 
     static constexpr uint32_t IPAK_INDEX_SECTION = 1;
@@ -60,18 +61,28 @@ struct IPakIndexEntry
     uint32_t size;
 };
 
-struct IPakDataBlockCountAndOffset
+union IPakDataBlockCountAndOffset
 {
-    uint32_t offset : 24;
+    struct
-    uint32_t count : 8;
+    {
+        uint32_t offset : 24;
+        uint32_t count : 8;
+    };
+
+    uint32_t raw;
 };
 
 static_assert(sizeof(IPakDataBlockCountAndOffset) == 4);
 
-struct IPakDataBlockCommand
+union IPakDataBlockCommand
 {
-    uint32_t size : 24;
+    struct
-    uint32_t compressed : 8;
+    {
+        uint32_t size : 24;
+        uint32_t compressed : 8;
+    };
+
+    uint32_t raw;
 };
 
 static_assert(sizeof(IPakDataBlockCommand) == 4);

src/ObjCompiling/Image/IPak/IPakCreator.cpp

@@ -100,7 +100,7 @@ namespace
         {
             GoTo(0);
 
-            const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC,
+            const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN,
                                     .version = ipak_consts::IPAK_VERSION,
                                     .size = static_cast<uint32_t>(m_total_size),
                                     .sectionCount = SECTION_COUNT};

src/ObjLoading.lua

@@ -16,6 +16,7 @@ function ObjLoading:link(links)
 	links:linkto(Utils)
 	links:linkto(ObjCommon)
 	links:linkto(ObjImage)
+	links:linkto(XMemCompress)
 	links:linkto(ZoneCommon)
 	links:linkto(minilzo)
 	links:linkto(minizip)
@@ -58,6 +59,7 @@ function ObjLoading:project()
 		self:include(includes)
 		Cryptography:include(includes)
 		Utils:include(includes)
+		XMemCompress:include(includes)
 		minilzo:include(includes)
 		minizip:include(includes)
 		zlib:include(includes)

src/ObjLoading/ObjContainer/IPak/IPak.cpp

@@ -2,8 +2,10 @@
 
 #include "IPakStreamManager.h"
 #include "ObjContainer/IPak/IPakTypes.h"
+#include "Utils/Endianness.h"
 #include "Utils/Logging/Log.h"
 
+#include <concepts>
 #include <filesystem>
 #include <format>
 #include <iostream>
@@ -37,9 +39,10 @@ namespace
             : m_path(std::move(path)),
               m_stream(std::move(stream)),
               m_initialized(false),
+              m_little_endian(true),
               m_index_section(nullptr),
               m_data_section(nullptr),
-              m_stream_manager(*m_stream)
+              m_stream_manager(nullptr)
         {
         }
 
@@ -55,7 +58,7 @@ namespace
             return true;
         }
 
-        [[nodiscard]] std::unique_ptr<iobjstream> GetEntryStream(const Hash nameHash, const Hash dataHash) const override
+        [[nodiscard]] std::unique_ptr<iobjstream> GetEntryStream(const IPakHash nameHash, const IPakHash dataHash) const override
         {
             const IPakIndexEntryKey wantedKey{
                 {.dataHash = dataHash, .nameHash = nameHash}
@@ -65,7 +68,7 @@ namespace
             {
                 if (entry.key.combinedKey == wantedKey.combinedKey)
                 {
-                    return m_stream_manager.OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
+                    return m_stream_manager->OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
                 }
                 else if (entry.key.combinedKey > wantedKey.combinedKey)
                 {
@@ -77,6 +80,11 @@ namespace
             return nullptr;
         }
 
+        [[nodiscard]] const std::vector<IPakIndexEntry>& GetIndexEntries() const override
+        {
+            return m_index_entries;
+        }
+
         std::string GetName() override
         {
             return fs::path(m_path).filename().replace_extension("").string();
@@ -97,7 +105,11 @@ namespace
                     return false;
                 }
 
-                m_index_entries.push_back(indexEntry);
+                SwapBytesIfNecessary(indexEntry.key.combinedKey);
+                SwapBytesIfNecessary(indexEntry.offset);
+                SwapBytesIfNecessary(indexEntry.size);
+
+                m_index_entries.emplace_back(indexEntry);
             }
 
             std::ranges::sort(m_index_entries,
@@ -120,6 +132,11 @@ namespace
                 return false;
             }
 
+            SwapBytesIfNecessary(section.type);
+            SwapBytesIfNecessary(section.offset);
+            SwapBytesIfNecessary(section.size);
+            SwapBytesIfNecessary(section.itemCount);
+
             switch (section.type)
             {
             case ipak_consts::IPAK_INDEX_SECTION:
@@ -148,18 +165,31 @@ namespace
                 return false;
             }
 
-            if (header.magic != ipak_consts::IPAK_MAGIC)
+            if (header.magic == ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN)
+            {
+                m_little_endian = true;
+                m_stream_manager = IPakStreamManager::Create(*m_stream, true);
+            }
+            else if (header.magic == ipak_consts::IPAK_MAGIC_BIG_ENDIAN)
+            {
+                m_little_endian = false;
+                m_stream_manager = IPakStreamManager::Create(*m_stream, false);
+            }
+            else
             {
                 con::error("Invalid ipak magic '{:#x}'.", header.magic);
                 return false;
             }
 
+            SwapBytesIfNecessary(header.version);
             if (header.version != ipak_consts::IPAK_VERSION)
             {
                 con::error("Unsupported ipak version '{}'.", header.version);
                 return false;
             }
 
+            SwapBytesIfNecessary(header.size);
+            SwapBytesIfNecessary(header.sectionCount);
             for (unsigned section = 0; section < header.sectionCount; section++)
             {
                 if (!ReadSection())
@@ -184,17 +214,26 @@ namespace
             return true;
         }
 
+        template<std::integral T> void SwapBytesIfNecessary(T& value)
+        {
+            if (m_little_endian)
+                value = endianness::FromLittleEndian(value);
+            else
+                value = endianness::FromBigEndian(value);
+        }
+
         std::string m_path;
         std::unique_ptr<std::istream> m_stream;
 
         bool m_initialized;
+        bool m_little_endian;
 
         std::unique_ptr<IPakSection> m_index_section;
         std::unique_ptr<IPakSection> m_data_section;
 
         std::vector<IPakIndexEntry> m_index_entries;
 
-        IPakStreamManager m_stream_manager;
+        std::unique_ptr<IPakStreamManager> m_stream_manager;
     };
 } // namespace
 
@@ -203,12 +242,12 @@ std::unique_ptr<IIPak> IIPak::Create(std::string path, std::unique_ptr<std::istr
     return std::make_unique<IPak>(std::move(path), std::move(stream));
 }
 
-IIPak::Hash IIPak::HashString(const std::string& str)
+IPakHash IIPak::HashString(const std::string& str)
 {
     return R_HashString(str.c_str(), 0);
 }
 
-IIPak::Hash IIPak::HashData(const void* data, const size_t dataSize)
+IPakHash IIPak::HashData(const void* data, const size_t dataSize)
 {
     return crc32(0, static_cast<const Bytef*>(data), static_cast<unsigned>(dataSize));
 }

src/ObjLoading/ObjContainer/IPak/IPak.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "ObjContainer/IPak/IPakTypes.h"
 #include "ObjContainer/ObjContainerReferenceable.h"
 #include "ObjContainer/ObjContainerRepository.h"
 #include "Utils/ObjStream.h"
@@ -14,19 +15,20 @@ class IIPak : public ObjContainerReferenceable
 {
 public:
     static ObjContainerRepository<IIPak, Zone> Repository;
-    typedef std::uint32_t Hash;
 
     IIPak() = default;
-    virtual ~IIPak() = default;
+    ~IIPak() override = default;
     IIPak(const IIPak& other) = default;
     IIPak(IIPak&& other) noexcept = default;
     IIPak& operator=(const IIPak& other) = default;
     IIPak& operator=(IIPak&& other) noexcept = default;
 
     virtual bool Initialize() = 0;
-    [[nodiscard]] virtual std::unique_ptr<iobjstream> GetEntryStream(Hash nameHash, Hash dataHash) const = 0;
+    [[nodiscard]] virtual std::unique_ptr<iobjstream> GetEntryStream(IPakHash nameHash, IPakHash dataHash) const = 0;
+
+    [[nodiscard]] virtual const std::vector<IPakIndexEntry>& GetIndexEntries() const = 0;
 
     static std::unique_ptr<IIPak> Create(std::string path, std::unique_ptr<std::istream> stream);
-    static Hash HashString(const std::string& str);
+    static IPakHash HashString(const std::string& str);
-    static Hash HashData(const void* data, size_t dataSize);
+    static IPakHash HashData(const void* data, size_t dataSize);
 };

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp

@@ -7,12 +7,18 @@
 #include <cassert>
 #include <cstring>
 #include <minilzo.h>
+#include <utility>
 
 using namespace ipak_consts;
 
-IPakEntryReadStream::IPakEntryReadStream(
+IPakEntryReadStream::IPakEntryReadStream(std::istream& stream,
-    std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, const int64_t startOffset, const size_t entrySize)
+                                         const bool isLittleEndian,
+                                         IPakStreamManagerActions* streamManagerActions,
+                                         uint8_t* chunkBuffer,
+                                         const int64_t startOffset,
+                                         const size_t entrySize)
     : m_chunk_buffer(chunkBuffer),
+      m_little_endian(isLittleEndian),
       m_stream(stream),
       m_stream_manager_actions(streamManagerActions),
       m_file_offset(0),
@@ -134,7 +140,7 @@ bool IPakEntryReadStream::ValidateBlockHeader(const IPakDataBlockHeader* blockHe
     }
 
     // We expect the current file to be continued where we left off
-    if (static_cast<int64_t>(blockHeader->countAndOffset.offset) != m_file_head)
+    if (std::cmp_not_equal(blockHeader->countAndOffset.offset, m_file_head))
     {
         // A matching offset is only relevant if a command contains data
         for (unsigned currentCommand = 0; currentCommand < blockHeader->countAndOffset.count; currentCommand++)
@@ -181,23 +187,33 @@ bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(const IPakDataBl
 
 bool IPakEntryReadStream::NextBlock()
 {
+    m_pos = AlignForward<int64_t>(m_pos, sizeof(IPakDataBlockHeader));
+
     if (m_pos >= m_end_pos)
         return false;
 
-    m_pos = AlignForward<int64_t>(m_pos, sizeof(IPakDataBlockHeader));
-
     const auto chunkStartPos = AlignBackwards<int64_t>(m_pos, IPAK_CHUNK_SIZE);
     const auto blockOffsetInChunk = static_cast<size_t>(m_pos - chunkStartPos);
 
     auto estimatedChunksToRead = AlignForward(m_entry_size - static_cast<size_t>(m_pos - m_base_pos), IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE;
 
-    if (estimatedChunksToRead > IPAK_CHUNK_COUNT_PER_READ)
+    estimatedChunksToRead = std::min(estimatedChunksToRead, IPAK_CHUNK_COUNT_PER_READ);
-        estimatedChunksToRead = IPAK_CHUNK_COUNT_PER_READ;
 
     if (!SetChunkBufferWindow(chunkStartPos, estimatedChunksToRead))
         return false;
 
     m_current_block = reinterpret_cast<IPakDataBlockHeader*>(&m_chunk_buffer[blockOffsetInChunk]);
+    SwapBytesIfNecessary(m_current_block->countAndOffset.raw);
+    for (auto& command : m_current_block->commands)
+    {
+        if (!m_little_endian)
+            command.raw &= 0xFFFFFFDF; // ? idk, the game seems to do this? halp
+
+        SwapBytesIfNecessary(command.raw);
+
+        auto size = command.size;
+        command.size = size;
+    }
 
     if (!ValidateBlockHeader(m_current_block))
         return false;
@@ -231,6 +247,23 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com
             m_current_command_offset = 0;
             m_file_head += static_cast<int64_t>(outputSize);
         }
+        else if (compressed == 2)
+        {
+            m_xmemdecompress_context.Reset();
+            const auto maybeDecompressSize = m_xmemdecompress_context.Process(
+                &m_chunk_buffer[m_pos - m_buffer_start_pos], static_cast<int>(commandSize), m_decompress_buffer, sizeof(m_decompress_buffer));
+
+            if (!maybeDecompressSize.has_value())
+            {
+                con::error("Decompressing block with XMemDecompress failed!");
+                return false;
+            }
+
+            m_current_command_buffer = m_decompress_buffer;
+            m_current_command_length = *maybeDecompressSize;
+            m_current_command_offset = 0;
+            m_file_head += static_cast<int64_t>(*maybeDecompressSize);
+        }
         else
         {
             // Do not process data but instead skip specified commandSize
@@ -324,7 +357,7 @@ std::streambuf::int_type IPakEntryReadStream::uflow()
     return EOF;
 }
 
-std::streamsize IPakEntryReadStream::xsgetn(char* ptr, const std::streamsize count)
+std::streamsize IPakEntryReadStream::xsgetn(char* ptr, std::streamsize count)
 {
     auto* destBuffer = reinterpret_cast<uint8_t*>(ptr);
     std::streamsize countRead = 0;

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h

@@ -2,37 +2,32 @@
 
 #include "IPakStreamManager.h"
 #include "ObjContainer/IPak/IPakTypes.h"
+#include "Utils/Endianness.h"
 #include "Utils/ObjStream.h"
+#include "XMemDecompress.h"
 
+#include <concepts>
 #include <istream>
 
 class IPakEntryReadStream final : public objbuf
 {
-    static constexpr size_t IPAK_DECOMPRESS_BUFFER_SIZE = 0x8000;
+public:
+    IPakEntryReadStream(
+        std::istream& stream, bool isLittleEndian, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
+    ~IPakEntryReadStream() override;
 
-    uint8_t* m_chunk_buffer;
+    [[nodiscard]] bool is_open() const override;
+    bool close() override;
 
-    std::istream& m_stream;
+protected:
-    IPakStreamManagerActions* m_stream_manager_actions;
+    std::streamsize showmanyc() override;
-
+    int_type underflow() override;
-    int64_t m_file_offset;
+    int_type uflow() override;
-    int64_t m_file_head;
+    std::streamsize xsgetn(char* ptr, std::streamsize count) override;
-
+    pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode mode) override;
-    size_t m_entry_size;
+    pos_type seekpos(pos_type pos, std::ios_base::openmode mode) override;
-
-    uint8_t m_decompress_buffer[IPAK_DECOMPRESS_BUFFER_SIZE];
-    IPakDataBlockHeader* m_current_block;
-    unsigned m_next_command;
-    uint8_t* m_current_command_buffer;
-    size_t m_current_command_length;
-    size_t m_current_command_offset;
-
-    int64_t m_pos;
-    int64_t m_base_pos;
-    int64_t m_end_pos;
-    int64_t m_buffer_start_pos;
-    int64_t m_buffer_end_pos;
 
+private:
     template<typename T> static T AlignForward(const T num, const T alignTo)
     {
         return (num + alignTo - 1) / alignTo * alignTo;
@@ -43,6 +38,14 @@ class IPakEntryReadStream final : public objbuf
         return num / alignTo * alignTo;
     }
 
+    template<std::integral T> void SwapBytesIfNecessary(T& value)
+    {
+        if (m_little_endian)
+            value = endianness::FromLittleEndian(value);
+        else
+            value = endianness::FromBigEndian(value);
+    }
+
     /**
      * \brief Reads the specified chunks from disk.
      * \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data.
@@ -95,18 +98,31 @@ class IPakEntryReadStream final : public objbuf
      */
     bool AdvanceStream();
 
-public:
+    static constexpr size_t IPAK_DECOMPRESS_BUFFER_SIZE = 0x8000;
-    IPakEntryReadStream(std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
-    ~IPakEntryReadStream() override;
 
-    [[nodiscard]] bool is_open() const override;
+    uint8_t* m_chunk_buffer;
-    bool close() override;
 
-protected:
+    bool m_little_endian;
-    std::streamsize showmanyc() override;
+    XMemDecompressContext m_xmemdecompress_context;
-    int_type underflow() override;
+
-    int_type uflow() override;
+    std::istream& m_stream;
-    std::streamsize xsgetn(char* ptr, std::streamsize count) override;
+    IPakStreamManagerActions* m_stream_manager_actions;
-    pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode mode) override;
+
-    pos_type seekpos(pos_type pos, std::ios_base::openmode mode) override;
+    int64_t m_file_offset;
+    int64_t m_file_head;
+
+    size_t m_entry_size;
+
+    uint8_t m_decompress_buffer[IPAK_DECOMPRESS_BUFFER_SIZE];
+    IPakDataBlockHeader* m_current_block;
+    unsigned m_next_command;
+    uint8_t* m_current_command_buffer;
+    size_t m_current_command_length;
+    size_t m_current_command_offset;
+
+    int64_t m_pos;
+    int64_t m_base_pos;
+    int64_t m_end_pos;
+    int64_t m_buffer_start_pos;
+    int64_t m_buffer_end_pos;
 };

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp

@@ -8,10 +8,8 @@
 
 using namespace ipak_consts;
 
-class IPakStreamManager::Impl final : public IPakStreamManagerActions
+namespace
 {
-    static constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
-
     class ChunkBuffer
     {
     public:
@@ -32,125 +30,116 @@ class IPakStreamManager::Impl final : public IPakStreamManagerActions
         }
     };
 
-    std::istream& m_stream;
+    constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
 
-    std::mutex m_read_mutex;
+    class IPakStreamManagerImpl final : public IPakStreamManager, public IPakStreamManagerActions
-    std::mutex m_stream_mutex;
-
-    std::vector<ManagedStream> m_open_streams;
-    std::vector<ChunkBuffer*> m_chunk_buffers;
-
-public:
-    explicit Impl(std::istream& stream)
-        : m_stream(stream)
     {
-        m_chunk_buffers.push_back(new ChunkBuffer());
+    public:
-    }
+        IPakStreamManagerImpl(std::istream& stream, const bool isLittleEndian)
-
+            : m_stream(stream),
-    Impl(const Impl& other) = delete;
+              m_little_endian(isLittleEndian)
-    Impl(Impl&& other) noexcept = delete;
-
-    virtual ~Impl()
-    {
-        m_stream_mutex.lock();
-
-        for (const auto& openStream : m_open_streams)
         {
-            openStream.m_stream->close();
+            m_chunk_buffers.push_back(new ChunkBuffer());
         }
-        m_open_streams.clear();
 
-        m_stream_mutex.unlock();
+        ~IPakStreamManagerImpl() override
-    }
-
-    Impl& operator=(const Impl& other) = delete;
-    Impl& operator=(Impl&& other) noexcept = delete;
-
-    std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length)
-    {
-        m_stream_mutex.lock();
-
-        ChunkBuffer* reservedChunkBuffer;
-        const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers,
-                                                          [](ChunkBuffer* chunkBuffer)
-                                                          {
-                                                              return chunkBuffer->m_using_stream == nullptr;
-                                                          });
-
-        if (freeChunkBuffer == m_chunk_buffers.end())
         {
-            reservedChunkBuffer = new ChunkBuffer();
+            m_stream_mutex.lock();
-            m_chunk_buffers.push_back(reservedChunkBuffer);
-        }
-        else
-            reservedChunkBuffer = *freeChunkBuffer;
 
-        auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, this, reservedChunkBuffer->m_buffer, startPosition, length);
+            for (const auto& openStream : m_open_streams)
-
-        reservedChunkBuffer->m_using_stream = ipakEntryStream.get();
-
-        m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer);
-
-        m_stream_mutex.unlock();
-
-        return std::make_unique<iobjstream>(std::move(ipakEntryStream));
-    }
-
-    void StartReading() override
-    {
-        m_read_mutex.lock();
-    }
-
-    void StopReading() override
-    {
-        m_read_mutex.unlock();
-    }
-
-    void CloseStream(objbuf* stream) override
-    {
-        m_stream_mutex.lock();
-
-        const auto openStreamEntry = std::ranges::find_if(m_open_streams,
-                                                          [stream](const ManagedStream& managedStream)
-                                                          {
-                                                              return managedStream.m_stream == stream;
-                                                          });
-
-        if (openStreamEntry != m_open_streams.end())
-        {
-            auto* chunkBuffer = openStreamEntry->m_chunk_buffer;
-            m_open_streams.erase(openStreamEntry);
-            chunkBuffer->m_using_stream = nullptr;
-
-            // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers
-            if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT)
             {
-                const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer);
+                openStream.m_stream->close();
+            }
+            m_open_streams.clear();
 
-                if (chunkBufferEntry != m_chunk_buffers.end())
+            m_stream_mutex.unlock();
+        }
+
+        std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length) override
+        {
+            m_stream_mutex.lock();
+
+            ChunkBuffer* reservedChunkBuffer;
+            const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers,
+                                                              [](ChunkBuffer* chunkBuffer)
+                                                              {
+                                                                  return chunkBuffer->m_using_stream == nullptr;
+                                                              });
+
+            if (freeChunkBuffer == m_chunk_buffers.end())
+            {
+                reservedChunkBuffer = new ChunkBuffer();
+                m_chunk_buffers.push_back(reservedChunkBuffer);
+            }
+            else
+                reservedChunkBuffer = *freeChunkBuffer;
+
+            auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, m_little_endian, this, reservedChunkBuffer->m_buffer, startPosition, length);
+
+            reservedChunkBuffer->m_using_stream = ipakEntryStream.get();
+
+            m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer);
+
+            m_stream_mutex.unlock();
+
+            return std::make_unique<iobjstream>(std::move(ipakEntryStream));
+        }
+
+        void StartReading() override
+        {
+            m_read_mutex.lock();
+        }
+
+        void StopReading() override
+        {
+            m_read_mutex.unlock();
+        }
+
+        void CloseStream(objbuf* stream) override
+        {
+            m_stream_mutex.lock();
+
+            const auto openStreamEntry = std::ranges::find_if(m_open_streams,
+                                                              [stream](const ManagedStream& managedStream)
+                                                              {
+                                                                  return managedStream.m_stream == stream;
+                                                              });
+
+            if (openStreamEntry != m_open_streams.end())
+            {
+                auto* chunkBuffer = openStreamEntry->m_chunk_buffer;
+                m_open_streams.erase(openStreamEntry);
+                chunkBuffer->m_using_stream = nullptr;
+
+                // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers
+                if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT)
                 {
-                    m_chunk_buffers.erase(chunkBufferEntry);
+                    const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer);
-                    delete chunkBuffer;
+
+                    if (chunkBufferEntry != m_chunk_buffers.end())
+                    {
+                        m_chunk_buffers.erase(chunkBufferEntry);
+                        delete chunkBuffer;
+                    }
                 }
             }
+
+            m_stream_mutex.unlock();
         }
 
-        m_stream_mutex.unlock();
+    private:
-    }
+        std::istream& m_stream;
-};
+        bool m_little_endian;
 
-IPakStreamManager::IPakStreamManager(std::istream& stream)
+        std::mutex m_read_mutex;
-    : m_impl(new Impl(stream))
+        std::mutex m_stream_mutex;
-{
-}
 
-IPakStreamManager::~IPakStreamManager()
+        std::vector<ManagedStream> m_open_streams;
-{
+        std::vector<ChunkBuffer*> m_chunk_buffers;
-    delete m_impl;
+    };
-    m_impl = nullptr;
+} // namespace
-}
 
-std::unique_ptr<iobjstream> IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) const
+std::unique_ptr<IPakStreamManager> IPakStreamManager::Create(std::istream& stream, const bool isLittleEndian)
 {
-    return m_impl->OpenStream(startPosition, length);
+    return std::make_unique<IPakStreamManagerImpl>(stream, isLittleEndian);
 }

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h

@@ -4,6 +4,7 @@
 
 #include <cstdint>
 #include <istream>
+#include <memory>
 #include <mutex>
 
 class IPakStreamManagerActions
@@ -17,17 +18,11 @@ public:
 
 class IPakStreamManager
 {
-    class Impl;
-    Impl* m_impl;
-
 public:
-    explicit IPakStreamManager(std::istream& stream);
+    IPakStreamManager() = default;
-    IPakStreamManager(const IPakStreamManager& other) = delete;
+    virtual ~IPakStreamManager() = default;
-    IPakStreamManager(IPakStreamManager&& other) noexcept = delete;
-    ~IPakStreamManager();
 
-    IPakStreamManager& operator=(const IPakStreamManager& other) = delete;
+    static std::unique_ptr<IPakStreamManager> Create(std::istream& stream, bool isLittleEndian);
-    IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete;
 
-    [[nodiscard]] std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) const;
+    [[nodiscard]] virtual std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) = 0;
 };

src/XMemCompress.lua

@@ -0,0 +1,49 @@
+XMemCompress = {}
+
+function XMemCompress:include(includes)
+	if includes:handle(self:name()) then
+		includedirs {
+			path.join(ProjectFolder(), "XMemCompress")
+		}
+	end
+end
+
+function XMemCompress:link(links)
+	links:add(self:name())
+	links:linkto(Utils)
+	links:linkto(lzx)
+end
+
+function XMemCompress:use()
+	
+end
+
+function XMemCompress:name()
+	return "XMemCompress"
+end
+
+function XMemCompress:project()
+	local folder = ProjectFolder()
+	local includes = Includes:create()
+
+	project(self:name())
+        targetdir(TargetDirectoryLib)
+		location "%{wks.location}/src/%{prj.name}"
+		kind "StaticLib"
+		language "C++"
+		
+		files {
+			path.join(folder, "XMemCompress/**.h"), 
+			path.join(folder, "XMemCompress/**.cpp")
+		}
+		
+        vpaths {
+			["*"] = {
+				path.join(folder, "XMemCompress")
+			}
+		}
+		
+		self:include(includes)
+		Utils:include(includes)
+		lzx:include(includes)
+end

src/XMemCompress/XMemDecompress.cpp

@@ -0,0 +1,141 @@
+#include "XMemDecompress.h"
+
+#include "Utils/Logging/Log.h"
+
+#include <format>
+#include <lzx.h>
+
+namespace
+{
+    uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize)
+    {
+        const auto value = input[offset];
+        offset++;
+        remainingSize--;
+        return value;
+    }
+
+    uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte)
+    {
+        return static_cast<uint16_t>(static_cast<uint16_t>(static_cast<uint16_t>(highByte) << 8u) | static_cast<uint16_t>(lowByte));
+    }
+
+    void LogErrorHeaderSpace(size_t remainingInputSize)
+    {
+        con::error("XMemDecompress: Not enough data for header: {}", remainingInputSize);
+    }
+} // namespace
+
+XMemDecompressContext::XMemDecompressContext()
+    : m_lzx_state(lzx_init(17))
+{
+}
+
+XMemDecompressContext::~XMemDecompressContext()
+{
+    if (m_lzx_state)
+        lzx_teardown(static_cast<lzx_state*>(m_lzx_state));
+}
+
+XMemDecompressContext::XMemDecompressContext(XMemDecompressContext&& other) noexcept
+    : m_lzx_state(other.m_lzx_state)
+{
+    other.m_lzx_state = nullptr;
+}
+
+XMemDecompressContext& XMemDecompressContext::operator=(XMemDecompressContext&& other) noexcept
+{
+    m_lzx_state = other.m_lzx_state;
+    other.m_lzx_state = nullptr;
+    return *this;
+}
+
+void XMemDecompressContext::Reset() const
+{
+    lzx_reset(static_cast<lzx_state*>(m_lzx_state));
+}
+
+std::optional<size_t> XMemDecompressContext::Process(const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) const
+{
+    size_t curInputOffset = 0uz;
+    size_t curInputSize = inputLength;
+
+    size_t curOutputOffset = 0uz;
+    size_t curOutputSize = outputBufferSize;
+
+    uint8_t lowByte;
+    uint16_t dstSize, srcSize;
+
+    while (curInputSize > 0)
+    {
+        uint8_t highByte = NextByte(input, curInputOffset, curInputSize);
+
+        uint8_t suffixSize;
+        if (highByte == 0xFF) // magic number: output is smaller than 0x8000
+        {
+            if (curInputSize < 4)
+            {
+                LogErrorHeaderSpace(curInputSize);
+                return std::nullopt;
+            }
+
+            highByte = NextByte(input, curInputOffset, curInputSize);
+            lowByte = NextByte(input, curInputOffset, curInputSize);
+            dstSize = CombineHighLow(highByte, lowByte);
+
+            highByte = NextByte(input, curInputOffset, curInputSize);
+            lowByte = NextByte(input, curInputOffset, curInputSize);
+            srcSize = CombineHighLow(highByte, lowByte);
+
+            // The game seems to skip a 5 byte suffix after these blocks, not sure why.
+            suffixSize = 5u;
+        }
+        else
+        {
+            if (curInputSize < 1)
+            {
+                LogErrorHeaderSpace(curInputSize);
+                return std::nullopt;
+            }
+
+            dstSize = 0x8000u;
+            lowByte = NextByte(input, curInputOffset, curInputSize);
+            srcSize = CombineHighLow(highByte, lowByte);
+
+            suffixSize = 0u;
+        }
+
+        if (srcSize == 0 || dstSize == 0)
+        {
+            // Other implementations do not handle this as a failure, game code suggests otherwise though
+            con::error("XMemDecompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize);
+            return std::nullopt;
+        }
+
+        if (static_cast<size_t>(srcSize) + suffixSize > curInputSize)
+        {
+            con::error("XMemDecompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize);
+            return std::nullopt;
+        }
+
+        if (dstSize > curOutputSize)
+        {
+            con::error("XMemDecompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize);
+            return std::nullopt;
+        }
+
+        auto ret = lzx_decompress(static_cast<lzx_state*>(m_lzx_state), &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize);
+        curInputOffset += srcSize + suffixSize;
+        curInputSize -= (srcSize + suffixSize);
+        curOutputOffset += dstSize;
+        curOutputSize -= srcSize;
+
+        if (ret != DECR_OK)
+        {
+            con::error("XMemDecompress: lzx decompression failed: {}", ret);
+            return std::nullopt;
+        }
+    }
+
+    return curOutputOffset;
+}

src/XMemCompress/XMemDecompress.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <cstdint>
+#include <optional>
+
+class XMemDecompressContext
+{
+public:
+    XMemDecompressContext();
+    ~XMemDecompressContext();
+    XMemDecompressContext(const XMemDecompressContext& other) = delete;
+    XMemDecompressContext(XMemDecompressContext&& other) noexcept;
+    XMemDecompressContext& operator=(const XMemDecompressContext& other) = delete;
+    XMemDecompressContext& operator=(XMemDecompressContext&& other) noexcept;
+
+    void Reset() const;
+    std::optional<size_t> Process(const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) const;
+
+private:
+    void* m_lzx_state;
+};

src/ZoneCommon.lua

@@ -6,6 +6,7 @@ function ZoneCommon:include(includes)
 			path.join(ProjectFolder(), "ZoneCommon")
 		}
 		Utils:include(includes)
+		XMemCompress:include(includes)
 		Common:include(includes)
 		ObjCommon:include(includes)
 		Parser:include(includes)
@@ -22,7 +23,7 @@ function ZoneCommon:link(links)
 	links:linkto(ObjCommon)
 	links:linkto(Parser)
 	links:linkto(Utils)
-	links:linkto(lzx)
+	links:linkto(XMemCompress)
 	ZoneCode:use()
 end
 
@@ -58,7 +59,6 @@ function ZoneCommon:project()
 		}
 		
         self:include(includes)
-		lzx:include(includes)
 		ZoneCode:include(includes)
 
 		ZoneCode:use()

src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp

@@ -2,133 +2,25 @@
 
 #include "Utils/Logging/Log.h"
 
-#include <cstring>
-#include <format>
-#include <iostream>
-#include <lzx.h>
-
-namespace
-{
-    uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize)
-    {
-        const auto value = input[offset];
-        offset++;
-        remainingSize--;
-        return value;
-    }
-
-    uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte)
-    {
-        return static_cast<uint16_t>(static_cast<uint16_t>(static_cast<uint16_t>(highByte) << 8u) | static_cast<uint16_t>(lowByte));
-    }
-
-    void LogErrorHeaderSpace(size_t remainingInputSize)
-    {
-        con::error("XMemCompress: Not enough data for header: {}", remainingInputSize);
-    }
-} // namespace
-
 XChunkProcessorLzxDecompress::XChunkProcessorLzxDecompress(const unsigned streamCount)
-    : m_lzx_states(streamCount)
+    : m_xmemdecompress_contexts(streamCount)
 {
-    // T6 uses 17 for window bits
-    for (auto& lzxState : m_lzx_states)
-        lzxState = lzx_init(17);
-}
-
-XChunkProcessorLzxDecompress::~XChunkProcessorLzxDecompress()
-{
-    for (auto* lzxState : m_lzx_states)
-        lzx_teardown(static_cast<lzx_state*>(lzxState));
 }
 
 size_t XChunkProcessorLzxDecompress::Process(
     const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
 {
-    auto* state = static_cast<lzx_state*>(m_lzx_states[streamNumber]);
+    const auto& xMemDecompress = m_xmemdecompress_contexts[streamNumber];
 
     // lzx state is reset before each chunk
-    lzx_reset(state);
+    xMemDecompress.Reset();
 
-    size_t curInputOffset = 0uz;
+    const auto maybeDecompressedSize = xMemDecompress.Process(input, inputLength, output, outputBufferSize);
-    size_t curInputSize = inputLength;
+    if (!maybeDecompressedSize.has_value())
-
-    size_t curOutputOffset = 0uz;
-    size_t curOutputSize = outputBufferSize;
-
-    uint8_t lowByte;
-    uint16_t dstSize, srcSize;
-
-    while (curInputSize > 0)
     {
-        uint8_t highByte = NextByte(input, curInputOffset, curInputSize);
+        con::error("Failed to decompress xchunk with XMemDecompress");
-
+        return 0;
-        uint8_t suffixSize;
-        if (highByte == 0xFF) // magic number: output is smaller than 0x8000
-        {
-            if (curInputSize < 4)
-            {
-                LogErrorHeaderSpace(curInputSize);
-                return curOutputOffset;
-            }
-
-            highByte = NextByte(input, curInputOffset, curInputSize);
-            lowByte = NextByte(input, curInputOffset, curInputSize);
-            dstSize = CombineHighLow(highByte, lowByte);
-
-            highByte = NextByte(input, curInputOffset, curInputSize);
-            lowByte = NextByte(input, curInputOffset, curInputSize);
-            srcSize = CombineHighLow(highByte, lowByte);
-
-            // The game seems to skip a 5 byte suffix after these blocks, not sure why.
-            suffixSize = 5u;
-        }
-        else
-        {
-            if (curInputSize < 1)
-            {
-                LogErrorHeaderSpace(curInputSize);
-                return curOutputOffset;
-            }
-
-            dstSize = 0x8000u;
-            lowByte = NextByte(input, curInputOffset, curInputSize);
-            srcSize = CombineHighLow(highByte, lowByte);
-
-            suffixSize = 0u;
-        }
-
-        if (srcSize == 0 || dstSize == 0)
-        {
-            // Other implementations do not handle this as a failure, game code suggests otherwise though
-            con::error("XMemCompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize);
-            return curOutputOffset;
-        }
-
-        if (static_cast<size_t>(srcSize) + suffixSize > curInputSize)
-        {
-            con::error("XMemCompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize);
-            return curOutputOffset;
-        }
-
-        if (dstSize > curOutputSize)
-        {
-            con::error("XMemCompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize);
-            return curOutputOffset;
-        }
-
-        auto ret = lzx_decompress(state, &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize);
-        curInputOffset += srcSize + suffixSize;
-        curInputSize -= (srcSize + suffixSize);
-        curOutputOffset += dstSize;
-        curOutputSize -= srcSize;
-
-        if (ret != DECR_OK)
-        {
-            con::error("XMemCompress: lzx decompression failed: {}", ret);
-            return curOutputOffset;
-        }
     }
 
-    return curOutputOffset;
+    return *maybeDecompressedSize;
 }

src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "IXChunkProcessor.h"
+#include "XMemDecompress.h"
 
 #include <vector>
 
@@ -8,9 +9,8 @@ class XChunkProcessorLzxDecompress final : public IXChunkProcessor
 {
 public:
     explicit XChunkProcessorLzxDecompress(unsigned streamCount);
-    ~XChunkProcessorLzxDecompress();
     size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
 
 private:
-    std::vector<void*> m_lzx_states;
+    std::vector<XMemDecompressContext> m_xmemdecompress_contexts;
 };