chore: enable reading of big endian ipaks

src/ObjCommon/ObjContainer/IPak/IPakTypes.h

@@ -6,7 +6,8 @@
 
 namespace ipak_consts
 {
-    static constexpr uint32_t IPAK_MAGIC = utils::MakeMagic32('K', 'A', 'P', 'I');
+    static constexpr uint32_t IPAK_MAGIC_LITTLE_ENDIAN = utils::MakeMagic32('K', 'A', 'P', 'I');
+    static constexpr uint32_t IPAK_MAGIC_BIG_ENDIAN = utils::MakeMagic32('I', 'P', 'A', 'K');
     static constexpr uint32_t IPAK_VERSION = 0x50000;
 
     static constexpr uint32_t IPAK_INDEX_SECTION = 1;
@@ -60,10 +61,15 @@ struct IPakIndexEntry
     uint32_t size;
 };
 
-struct IPakDataBlockCountAndOffset
+union IPakDataBlockCountAndOffset
 {
-    uint32_t offset : 24;
+    struct
-    uint32_t count : 8;
+    {
+        uint32_t offset : 24;
+        uint32_t count : 8;
+    };
+
+    uint32_t raw;
 };
 
 static_assert(sizeof(IPakDataBlockCountAndOffset) == 4);

src/ObjCompiling/Image/IPak/IPakCreator.cpp

@@ -100,7 +100,7 @@ namespace
         {
             GoTo(0);
 
-            const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC,
+            const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN,
                                     .version = ipak_consts::IPAK_VERSION,
                                     .size = static_cast<uint32_t>(m_total_size),
                                     .sectionCount = SECTION_COUNT};

src/ObjLoading/ObjContainer/IPak/IPak.cpp

@@ -2,8 +2,10 @@
 
 #include "IPakStreamManager.h"
 #include "ObjContainer/IPak/IPakTypes.h"
+#include "Utils/Endianness.h"
 #include "Utils/Logging/Log.h"
 
+#include <concepts>
 #include <filesystem>
 #include <format>
 #include <iostream>
@@ -37,9 +39,10 @@ namespace
             : m_path(std::move(path)),
               m_stream(std::move(stream)),
               m_initialized(false),
+              m_little_endian(true),
               m_index_section(nullptr),
               m_data_section(nullptr),
-              m_stream_manager(*m_stream)
+              m_stream_manager(nullptr)
         {
         }
 
@@ -65,7 +68,7 @@ namespace
             {
                 if (entry.key.combinedKey == wantedKey.combinedKey)
                 {
-                    return m_stream_manager.OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
+                    return m_stream_manager->OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
                 }
                 else if (entry.key.combinedKey > wantedKey.combinedKey)
                 {
@@ -102,7 +105,11 @@ namespace
                     return false;
                 }
 
-                m_index_entries.push_back(indexEntry);
+                SwapBytesIfNecessary(indexEntry.key.combinedKey);
+                SwapBytesIfNecessary(indexEntry.offset);
+                SwapBytesIfNecessary(indexEntry.size);
+
+                m_index_entries.emplace_back(indexEntry);
             }
 
             std::ranges::sort(m_index_entries,
@@ -125,6 +132,11 @@ namespace
                 return false;
             }
 
+            SwapBytesIfNecessary(section.type);
+            SwapBytesIfNecessary(section.offset);
+            SwapBytesIfNecessary(section.size);
+            SwapBytesIfNecessary(section.itemCount);
+
             switch (section.type)
             {
             case ipak_consts::IPAK_INDEX_SECTION:
@@ -153,18 +165,31 @@ namespace
                 return false;
             }
 
-            if (header.magic != ipak_consts::IPAK_MAGIC)
+            if (header.magic == ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN)
+            {
+                m_little_endian = true;
+                m_stream_manager = IPakStreamManager::Create(*m_stream, true);
+            }
+            else if (header.magic == ipak_consts::IPAK_MAGIC_BIG_ENDIAN)
+            {
+                m_little_endian = false;
+                m_stream_manager = IPakStreamManager::Create(*m_stream, false);
+            }
+            else
             {
                 con::error("Invalid ipak magic '{:#x}'.", header.magic);
                 return false;
             }
 
+            SwapBytesIfNecessary(header.version);
             if (header.version != ipak_consts::IPAK_VERSION)
             {
                 con::error("Unsupported ipak version '{}'.", header.version);
                 return false;
             }
 
+            SwapBytesIfNecessary(header.size);
+            SwapBytesIfNecessary(header.sectionCount);
             for (unsigned section = 0; section < header.sectionCount; section++)
             {
                 if (!ReadSection())
@@ -189,17 +214,26 @@ namespace
             return true;
         }
 
+        template<std::integral T> void SwapBytesIfNecessary(T& value)
+        {
+            if (m_little_endian)
+                value = endianness::FromLittleEndian(value);
+            else
+                value = endianness::FromBigEndian(value);
+        }
+
         std::string m_path;
         std::unique_ptr<std::istream> m_stream;
 
         bool m_initialized;
+        bool m_little_endian;
 
         std::unique_ptr<IPakSection> m_index_section;
         std::unique_ptr<IPakSection> m_data_section;
 
         std::vector<IPakIndexEntry> m_index_entries;
 
-        IPakStreamManager m_stream_manager;
+        std::unique_ptr<IPakStreamManager> m_stream_manager;
     };
 } // namespace
 

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp

@@ -11,9 +11,14 @@
 
 using namespace ipak_consts;
 
-IPakEntryReadStream::IPakEntryReadStream(
+IPakEntryReadStream::IPakEntryReadStream(std::istream& stream,
-    std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, const int64_t startOffset, const size_t entrySize)
+                                         const bool isLittleEndian,
+                                         IPakStreamManagerActions* streamManagerActions,
+                                         uint8_t* chunkBuffer,
+                                         const int64_t startOffset,
+                                         const size_t entrySize)
     : m_chunk_buffer(chunkBuffer),
+      m_little_endian(isLittleEndian),
       m_stream(stream),
       m_stream_manager_actions(streamManagerActions),
       m_file_offset(0),
@@ -198,6 +203,13 @@ bool IPakEntryReadStream::NextBlock()
         return false;
 
     m_current_block = reinterpret_cast<IPakDataBlockHeader*>(&m_chunk_buffer[blockOffsetInChunk]);
+    SwapBytesIfNecessary(m_current_block->countAndOffset.raw);
+    for (auto& command : m_current_block->commands)
+    {
+        auto size = command.size;
+        SwapBytesIfNecessary(size);
+        command.size = size;
+    }
 
     if (!ValidateBlockHeader(m_current_block))
         return false;
@@ -231,6 +243,11 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com
             m_current_command_offset = 0;
             m_file_head += static_cast<int64_t>(outputSize);
         }
+        else if (compressed == 2)
+        {
+            // This seems to use XMemDecompress
+            assert(false);
+        }
         else
         {
             // Do not process data but instead skip specified commandSize

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h

@@ -2,14 +2,17 @@
 
 #include "IPakStreamManager.h"
 #include "ObjContainer/IPak/IPakTypes.h"
+#include "Utils/Endianness.h"
 #include "Utils/ObjStream.h"
 
+#include <concepts>
 #include <istream>
 
 class IPakEntryReadStream final : public objbuf
 {
 public:
-    IPakEntryReadStream(std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
+    IPakEntryReadStream(
+        std::istream& stream, bool isLittleEndian, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
     ~IPakEntryReadStream() override;
 
     [[nodiscard]] bool is_open() const override;
@@ -34,6 +37,14 @@ private:
         return num / alignTo * alignTo;
     }
 
+    template<std::integral T> void SwapBytesIfNecessary(T& value)
+    {
+        if (m_little_endian)
+            value = endianness::FromLittleEndian(value);
+        else
+            value = endianness::FromBigEndian(value);
+    }
+
     /**
      * \brief Reads the specified chunks from disk.
      * \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data.
@@ -90,6 +101,8 @@ private:
 
     uint8_t* m_chunk_buffer;
 
+    bool m_little_endian;
+
     std::istream& m_stream;
     IPakStreamManagerActions* m_stream_manager_actions;
 

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp

@@ -8,10 +8,8 @@
 
 using namespace ipak_consts;
 
-class IPakStreamManager::Impl final : public IPakStreamManagerActions
+namespace
 {
-    static constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
-
     class ChunkBuffer
     {
     public:
@@ -32,125 +30,116 @@ class IPakStreamManager::Impl final : public IPakStreamManagerActions
         }
     };
 
-    std::istream& m_stream;
+    constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
 
-    std::mutex m_read_mutex;
+    class IPakStreamManagerImpl final : public IPakStreamManager, public IPakStreamManagerActions
-    std::mutex m_stream_mutex;
-
-    std::vector<ManagedStream> m_open_streams;
-    std::vector<ChunkBuffer*> m_chunk_buffers;
-
-public:
-    explicit Impl(std::istream& stream)
-        : m_stream(stream)
     {
-        m_chunk_buffers.push_back(new ChunkBuffer());
+    public:
-    }
+        IPakStreamManagerImpl(std::istream& stream, const bool isLittleEndian)
-
+            : m_stream(stream),
-    Impl(const Impl& other) = delete;
+              m_little_endian(isLittleEndian)
-    Impl(Impl&& other) noexcept = delete;
-
-    virtual ~Impl()
-    {
-        m_stream_mutex.lock();
-
-        for (const auto& openStream : m_open_streams)
         {
-            openStream.m_stream->close();
+            m_chunk_buffers.push_back(new ChunkBuffer());
         }
-        m_open_streams.clear();
 
-        m_stream_mutex.unlock();
+        ~IPakStreamManagerImpl() override
-    }
-
-    Impl& operator=(const Impl& other) = delete;
-    Impl& operator=(Impl&& other) noexcept = delete;
-
-    std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length)
-    {
-        m_stream_mutex.lock();
-
-        ChunkBuffer* reservedChunkBuffer;
-        const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers,
-                                                          [](ChunkBuffer* chunkBuffer)
-                                                          {
-                                                              return chunkBuffer->m_using_stream == nullptr;
-                                                          });
-
-        if (freeChunkBuffer == m_chunk_buffers.end())
         {
-            reservedChunkBuffer = new ChunkBuffer();
+            m_stream_mutex.lock();
-            m_chunk_buffers.push_back(reservedChunkBuffer);
-        }
-        else
-            reservedChunkBuffer = *freeChunkBuffer;
 
-        auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, this, reservedChunkBuffer->m_buffer, startPosition, length);
+            for (const auto& openStream : m_open_streams)
-
-        reservedChunkBuffer->m_using_stream = ipakEntryStream.get();
-
-        m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer);
-
-        m_stream_mutex.unlock();
-
-        return std::make_unique<iobjstream>(std::move(ipakEntryStream));
-    }
-
-    void StartReading() override
-    {
-        m_read_mutex.lock();
-    }
-
-    void StopReading() override
-    {
-        m_read_mutex.unlock();
-    }
-
-    void CloseStream(objbuf* stream) override
-    {
-        m_stream_mutex.lock();
-
-        const auto openStreamEntry = std::ranges::find_if(m_open_streams,
-                                                          [stream](const ManagedStream& managedStream)
-                                                          {
-                                                              return managedStream.m_stream == stream;
-                                                          });
-
-        if (openStreamEntry != m_open_streams.end())
-        {
-            auto* chunkBuffer = openStreamEntry->m_chunk_buffer;
-            m_open_streams.erase(openStreamEntry);
-            chunkBuffer->m_using_stream = nullptr;
-
-            // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers
-            if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT)
             {
-                const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer);
+                openStream.m_stream->close();
+            }
+            m_open_streams.clear();
 
-                if (chunkBufferEntry != m_chunk_buffers.end())
+            m_stream_mutex.unlock();
+        }
+
+        std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length) override
+        {
+            m_stream_mutex.lock();
+
+            ChunkBuffer* reservedChunkBuffer;
+            const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers,
+                                                              [](ChunkBuffer* chunkBuffer)
+                                                              {
+                                                                  return chunkBuffer->m_using_stream == nullptr;
+                                                              });
+
+            if (freeChunkBuffer == m_chunk_buffers.end())
+            {
+                reservedChunkBuffer = new ChunkBuffer();
+                m_chunk_buffers.push_back(reservedChunkBuffer);
+            }
+            else
+                reservedChunkBuffer = *freeChunkBuffer;
+
+            auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, m_little_endian, this, reservedChunkBuffer->m_buffer, startPosition, length);
+
+            reservedChunkBuffer->m_using_stream = ipakEntryStream.get();
+
+            m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer);
+
+            m_stream_mutex.unlock();
+
+            return std::make_unique<iobjstream>(std::move(ipakEntryStream));
+        }
+
+        void StartReading() override
+        {
+            m_read_mutex.lock();
+        }
+
+        void StopReading() override
+        {
+            m_read_mutex.unlock();
+        }
+
+        void CloseStream(objbuf* stream) override
+        {
+            m_stream_mutex.lock();
+
+            const auto openStreamEntry = std::ranges::find_if(m_open_streams,
+                                                              [stream](const ManagedStream& managedStream)
+                                                              {
+                                                                  return managedStream.m_stream == stream;
+                                                              });
+
+            if (openStreamEntry != m_open_streams.end())
+            {
+                auto* chunkBuffer = openStreamEntry->m_chunk_buffer;
+                m_open_streams.erase(openStreamEntry);
+                chunkBuffer->m_using_stream = nullptr;
+
+                // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers
+                if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT)
                 {
-                    m_chunk_buffers.erase(chunkBufferEntry);
+                    const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer);
-                    delete chunkBuffer;
+
+                    if (chunkBufferEntry != m_chunk_buffers.end())
+                    {
+                        m_chunk_buffers.erase(chunkBufferEntry);
+                        delete chunkBuffer;
+                    }
                 }
             }
+
+            m_stream_mutex.unlock();
         }
 
-        m_stream_mutex.unlock();
+    private:
-    }
+        std::istream& m_stream;
-};
+        bool m_little_endian;
 
-IPakStreamManager::IPakStreamManager(std::istream& stream)
+        std::mutex m_read_mutex;
-    : m_impl(new Impl(stream))
+        std::mutex m_stream_mutex;
-{
-}
 
-IPakStreamManager::~IPakStreamManager()
+        std::vector<ManagedStream> m_open_streams;
-{
+        std::vector<ChunkBuffer*> m_chunk_buffers;
-    delete m_impl;
+    };
-    m_impl = nullptr;
+} // namespace
-}
 
-std::unique_ptr<iobjstream> IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) const
+std::unique_ptr<IPakStreamManager> IPakStreamManager::Create(std::istream& stream, const bool isLittleEndian)
 {
-    return m_impl->OpenStream(startPosition, length);
+    return std::make_unique<IPakStreamManagerImpl>(stream, isLittleEndian);
 }

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h

@@ -4,6 +4,7 @@
 
 #include <cstdint>
 #include <istream>
+#include <memory>
 #include <mutex>
 
 class IPakStreamManagerActions
@@ -17,17 +18,11 @@ public:
 
 class IPakStreamManager
 {
-    class Impl;
-    Impl* m_impl;
-
 public:
-    explicit IPakStreamManager(std::istream& stream);
+    IPakStreamManager() = default;
-    IPakStreamManager(const IPakStreamManager& other) = delete;
+    virtual ~IPakStreamManager() = default;
-    IPakStreamManager(IPakStreamManager&& other) noexcept = delete;
-    ~IPakStreamManager();
 
-    IPakStreamManager& operator=(const IPakStreamManager& other) = delete;
+    static std::unique_ptr<IPakStreamManager> Create(std::istream& stream, bool isLittleEndian);
-    IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete;
 
-    [[nodiscard]] std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) const;
+    [[nodiscard]] virtual std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) = 0;
 };