chore: enable reading of big endian ipaks

src/ObjCommon/ObjContainer/IPak/IPakTypes.h

@@ -6,7 +6,8 @@
 
 namespace ipak_consts
 {
-    static constexpr uint32_t IPAK_MAGIC = utils::MakeMagic32('K', 'A', 'P', 'I');
+    static constexpr uint32_t IPAK_MAGIC_LITTLE_ENDIAN = utils::MakeMagic32('K', 'A', 'P', 'I');
+    static constexpr uint32_t IPAK_MAGIC_BIG_ENDIAN = utils::MakeMagic32('I', 'P', 'A', 'K');
     static constexpr uint32_t IPAK_VERSION = 0x50000;
 
     static constexpr uint32_t IPAK_INDEX_SECTION = 1;
@@ -60,12 +61,17 @@ struct IPakIndexEntry
     uint32_t size;
 };
 
-struct IPakDataBlockCountAndOffset
+union IPakDataBlockCountAndOffset
+{
+    struct
     {
         uint32_t offset : 24;
         uint32_t count : 8;
     };
 
+    uint32_t raw;
+};
+
 static_assert(sizeof(IPakDataBlockCountAndOffset) == 4);
 
 struct IPakDataBlockCommand

src/ObjCompiling/Image/IPak/IPakCreator.cpp

@@ -100,7 +100,7 @@ namespace
         {
             GoTo(0);
 
-            const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC,
+            const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN,
                                     .version = ipak_consts::IPAK_VERSION,
                                     .size = static_cast<uint32_t>(m_total_size),
                                     .sectionCount = SECTION_COUNT};

src/ObjLoading/ObjContainer/IPak/IPak.cpp

@@ -2,8 +2,10 @@
 
 #include "IPakStreamManager.h"
 #include "ObjContainer/IPak/IPakTypes.h"
+#include "Utils/Endianness.h"
 #include "Utils/Logging/Log.h"
 
+#include <concepts>
 #include <filesystem>
 #include <format>
 #include <iostream>
@@ -37,9 +39,10 @@ namespace
             : m_path(std::move(path)),
               m_stream(std::move(stream)),
               m_initialized(false),
+              m_little_endian(true),
               m_index_section(nullptr),
               m_data_section(nullptr),
-              m_stream_manager(*m_stream)
+              m_stream_manager(nullptr)
         {
         }
 
@@ -65,7 +68,7 @@ namespace
             {
                 if (entry.key.combinedKey == wantedKey.combinedKey)
                 {
-                    return m_stream_manager.OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
+                    return m_stream_manager->OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
                 }
                 else if (entry.key.combinedKey > wantedKey.combinedKey)
                 {
@@ -102,7 +105,11 @@ namespace
                     return false;
                 }
 
-                m_index_entries.push_back(indexEntry);
+                SwapBytesIfNecessary(indexEntry.key.combinedKey);
+                SwapBytesIfNecessary(indexEntry.offset);
+                SwapBytesIfNecessary(indexEntry.size);
+
+                m_index_entries.emplace_back(indexEntry);
             }
 
             std::ranges::sort(m_index_entries,
@@ -125,6 +132,11 @@ namespace
                 return false;
             }
 
+            SwapBytesIfNecessary(section.type);
+            SwapBytesIfNecessary(section.offset);
+            SwapBytesIfNecessary(section.size);
+            SwapBytesIfNecessary(section.itemCount);
+
             switch (section.type)
             {
             case ipak_consts::IPAK_INDEX_SECTION:
@@ -153,18 +165,31 @@ namespace
                 return false;
             }
 
-            if (header.magic != ipak_consts::IPAK_MAGIC)
+            if (header.magic == ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN)
+            {
+                m_little_endian = true;
+                m_stream_manager = IPakStreamManager::Create(*m_stream, true);
+            }
+            else if (header.magic == ipak_consts::IPAK_MAGIC_BIG_ENDIAN)
+            {
+                m_little_endian = false;
+                m_stream_manager = IPakStreamManager::Create(*m_stream, false);
+            }
+            else
             {
                 con::error("Invalid ipak magic '{:#x}'.", header.magic);
                 return false;
             }
 
+            SwapBytesIfNecessary(header.version);
             if (header.version != ipak_consts::IPAK_VERSION)
             {
                 con::error("Unsupported ipak version '{}'.", header.version);
                 return false;
             }
 
+            SwapBytesIfNecessary(header.size);
+            SwapBytesIfNecessary(header.sectionCount);
             for (unsigned section = 0; section < header.sectionCount; section++)
             {
                 if (!ReadSection())
@@ -189,17 +214,26 @@ namespace
             return true;
         }
 
+        template<std::integral T> void SwapBytesIfNecessary(T& value)
+        {
+            if (m_little_endian)
+                value = endianness::FromLittleEndian(value);
+            else
+                value = endianness::FromBigEndian(value);
+        }
+
         std::string m_path;
         std::unique_ptr<std::istream> m_stream;
 
         bool m_initialized;
+        bool m_little_endian;
 
         std::unique_ptr<IPakSection> m_index_section;
         std::unique_ptr<IPakSection> m_data_section;
 
         std::vector<IPakIndexEntry> m_index_entries;
 
-        IPakStreamManager m_stream_manager;
+        std::unique_ptr<IPakStreamManager> m_stream_manager;
     };
 } // namespace
 

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp

@@ -11,9 +11,14 @@
 
 using namespace ipak_consts;
 
-IPakEntryReadStream::IPakEntryReadStream(
+IPakEntryReadStream::IPakEntryReadStream(std::istream& stream,
-    std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, const int64_t startOffset, const size_t entrySize)
+                                         const bool isLittleEndian,
+                                         IPakStreamManagerActions* streamManagerActions,
+                                         uint8_t* chunkBuffer,
+                                         const int64_t startOffset,
+                                         const size_t entrySize)
     : m_chunk_buffer(chunkBuffer),
+      m_little_endian(isLittleEndian),
       m_stream(stream),
       m_stream_manager_actions(streamManagerActions),
       m_file_offset(0),
@@ -198,6 +203,13 @@ bool IPakEntryReadStream::NextBlock()
         return false;
 
     m_current_block = reinterpret_cast<IPakDataBlockHeader*>(&m_chunk_buffer[blockOffsetInChunk]);
+    SwapBytesIfNecessary(m_current_block->countAndOffset.raw);
+    for (auto& command : m_current_block->commands)
+    {
+        auto size = command.size;
+        SwapBytesIfNecessary(size);
+        command.size = size;
+    }
 
     if (!ValidateBlockHeader(m_current_block))
         return false;
@@ -231,6 +243,11 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com
             m_current_command_offset = 0;
             m_file_head += static_cast<int64_t>(outputSize);
         }
+        else if (compressed == 2)
+        {
+            // This seems to use XMemDecompress
+            assert(false);
+        }
         else
         {
             // Do not process data but instead skip specified commandSize

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h

@@ -2,14 +2,17 @@
 
 #include "IPakStreamManager.h"
 #include "ObjContainer/IPak/IPakTypes.h"
+#include "Utils/Endianness.h"
 #include "Utils/ObjStream.h"
 
+#include <concepts>
 #include <istream>
 
 class IPakEntryReadStream final : public objbuf
 {
 public:
-    IPakEntryReadStream(std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
+    IPakEntryReadStream(
+        std::istream& stream, bool isLittleEndian, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
     ~IPakEntryReadStream() override;
 
     [[nodiscard]] bool is_open() const override;
@@ -34,6 +37,14 @@ private:
         return num / alignTo * alignTo;
     }
 
+    template<std::integral T> void SwapBytesIfNecessary(T& value)
+    {
+        if (m_little_endian)
+            value = endianness::FromLittleEndian(value);
+        else
+            value = endianness::FromBigEndian(value);
+    }
+
     /**
      * \brief Reads the specified chunks from disk.
      * \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data.
@@ -90,6 +101,8 @@ private:
 
     uint8_t* m_chunk_buffer;
 
+    bool m_little_endian;
+
     std::istream& m_stream;
     IPakStreamManagerActions* m_stream_manager_actions;
 

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp

@@ -8,10 +8,8 @@
 
 using namespace ipak_consts;
 
-class IPakStreamManager::Impl final : public IPakStreamManagerActions
+namespace
 {
-    static constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
-
     class ChunkBuffer
     {
     public:
@@ -32,25 +30,19 @@ class IPakStreamManager::Impl final : public IPakStreamManagerActions
         }
     };
 
-    std::istream& m_stream;
+    constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
-
-    std::mutex m_read_mutex;
-    std::mutex m_stream_mutex;
-
-    std::vector<ManagedStream> m_open_streams;
-    std::vector<ChunkBuffer*> m_chunk_buffers;
 
+    class IPakStreamManagerImpl final : public IPakStreamManager, public IPakStreamManagerActions
+    {
     public:
-    explicit Impl(std::istream& stream)
+        IPakStreamManagerImpl(std::istream& stream, const bool isLittleEndian)
-        : m_stream(stream)
+            : m_stream(stream),
+              m_little_endian(isLittleEndian)
         {
             m_chunk_buffers.push_back(new ChunkBuffer());
         }
 
-    Impl(const Impl& other) = delete;
+        ~IPakStreamManagerImpl() override
-    Impl(Impl&& other) noexcept = delete;
-
-    virtual ~Impl()
         {
             m_stream_mutex.lock();
 
@@ -63,10 +55,7 @@ public:
             m_stream_mutex.unlock();
         }
 
-    Impl& operator=(const Impl& other) = delete;
+        std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length) override
-    Impl& operator=(Impl&& other) noexcept = delete;
-
-    std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length)
         {
             m_stream_mutex.lock();
 
@@ -85,7 +74,7 @@ public:
             else
                 reservedChunkBuffer = *freeChunkBuffer;
 
-        auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, this, reservedChunkBuffer->m_buffer, startPosition, length);
+            auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, m_little_endian, this, reservedChunkBuffer->m_buffer, startPosition, length);
 
             reservedChunkBuffer->m_using_stream = ipakEntryStream.get();
 
@@ -137,20 +126,20 @@ public:
 
             m_stream_mutex.unlock();
         }
+
+    private:
+        std::istream& m_stream;
+        bool m_little_endian;
+
+        std::mutex m_read_mutex;
+        std::mutex m_stream_mutex;
+
+        std::vector<ManagedStream> m_open_streams;
+        std::vector<ChunkBuffer*> m_chunk_buffers;
     };
+} // namespace
 
-IPakStreamManager::IPakStreamManager(std::istream& stream)
+std::unique_ptr<IPakStreamManager> IPakStreamManager::Create(std::istream& stream, const bool isLittleEndian)
-    : m_impl(new Impl(stream))
 {
-}
+    return std::make_unique<IPakStreamManagerImpl>(stream, isLittleEndian);
-
-IPakStreamManager::~IPakStreamManager()
-{
-    delete m_impl;
-    m_impl = nullptr;
-}
-
-std::unique_ptr<iobjstream> IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) const
-{
-    return m_impl->OpenStream(startPosition, length);
 }

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h

@@ -4,6 +4,7 @@
 
 #include <cstdint>
 #include <istream>
+#include <memory>
 #include <mutex>
 
 class IPakStreamManagerActions
@@ -17,17 +18,11 @@ public:
 
 class IPakStreamManager
 {
-    class Impl;
-    Impl* m_impl;
-
 public:
-    explicit IPakStreamManager(std::istream& stream);
+    IPakStreamManager() = default;
-    IPakStreamManager(const IPakStreamManager& other) = delete;
+    virtual ~IPakStreamManager() = default;
-    IPakStreamManager(IPakStreamManager&& other) noexcept = delete;
-    ~IPakStreamManager();
 
-    IPakStreamManager& operator=(const IPakStreamManager& other) = delete;
+    static std::unique_ptr<IPakStreamManager> Create(std::istream& stream, bool isLittleEndian);
-    IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete;
 
-    [[nodiscard]] std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) const;
+    [[nodiscard]] virtual std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) = 0;
 };