diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index 441e438c..6f08cc9e 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -6,7 +6,8 @@ namespace ipak_consts { - static constexpr uint32_t IPAK_MAGIC = utils::MakeMagic32('K', 'A', 'P', 'I'); + static constexpr uint32_t IPAK_MAGIC_LITTLE_ENDIAN = utils::MakeMagic32('K', 'A', 'P', 'I'); + static constexpr uint32_t IPAK_MAGIC_BIG_ENDIAN = utils::MakeMagic32('I', 'P', 'A', 'K'); static constexpr uint32_t IPAK_VERSION = 0x50000; static constexpr uint32_t IPAK_INDEX_SECTION = 1; @@ -60,10 +61,15 @@ struct IPakIndexEntry uint32_t size; }; -struct IPakDataBlockCountAndOffset +union IPakDataBlockCountAndOffset { - uint32_t offset : 24; - uint32_t count : 8; + struct + { + uint32_t offset : 24; + uint32_t count : 8; + }; + + uint32_t raw; }; static_assert(sizeof(IPakDataBlockCountAndOffset) == 4); diff --git a/src/ObjCompiling/Image/IPak/IPakCreator.cpp b/src/ObjCompiling/Image/IPak/IPakCreator.cpp index 370074d2..0d8bf457 100644 --- a/src/ObjCompiling/Image/IPak/IPakCreator.cpp +++ b/src/ObjCompiling/Image/IPak/IPakCreator.cpp @@ -100,7 +100,7 @@ namespace { GoTo(0); - const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC, + const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN, .version = ipak_consts::IPAK_VERSION, .size = static_cast(m_total_size), .sectionCount = SECTION_COUNT}; diff --git a/src/ObjLoading/ObjContainer/IPak/IPak.cpp b/src/ObjLoading/ObjContainer/IPak/IPak.cpp index 0585fd96..d5eb90f7 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPak.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPak.cpp @@ -2,8 +2,10 @@ #include "IPakStreamManager.h" #include "ObjContainer/IPak/IPakTypes.h" +#include "Utils/Endianness.h" #include "Utils/Logging/Log.h" +#include #include #include #include @@ -37,9 +39,10 @@ namespace : m_path(std::move(path)), m_stream(std::move(stream)), m_initialized(false), + m_little_endian(true), m_index_section(nullptr), m_data_section(nullptr), - m_stream_manager(*m_stream) + m_stream_manager(nullptr) { } @@ -65,7 +68,7 @@ namespace { if (entry.key.combinedKey == wantedKey.combinedKey) { - return m_stream_manager.OpenStream(static_cast(m_data_section->offset) + entry.offset, entry.size); + return m_stream_manager->OpenStream(static_cast(m_data_section->offset) + entry.offset, entry.size); } else if (entry.key.combinedKey > wantedKey.combinedKey) { @@ -102,7 +105,11 @@ namespace return false; } - m_index_entries.push_back(indexEntry); + SwapBytesIfNecessary(indexEntry.key.combinedKey); + SwapBytesIfNecessary(indexEntry.offset); + SwapBytesIfNecessary(indexEntry.size); + + m_index_entries.emplace_back(indexEntry); } std::ranges::sort(m_index_entries, @@ -125,6 +132,11 @@ namespace return false; } + SwapBytesIfNecessary(section.type); + SwapBytesIfNecessary(section.offset); + SwapBytesIfNecessary(section.size); + SwapBytesIfNecessary(section.itemCount); + switch (section.type) { case ipak_consts::IPAK_INDEX_SECTION: @@ -153,18 +165,31 @@ namespace return false; } - if (header.magic != ipak_consts::IPAK_MAGIC) + if (header.magic == ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN) + { + m_little_endian = true; + m_stream_manager = IPakStreamManager::Create(*m_stream, true); + } + else if (header.magic == ipak_consts::IPAK_MAGIC_BIG_ENDIAN) + { + m_little_endian = false; + m_stream_manager = IPakStreamManager::Create(*m_stream, false); + } + else { con::error("Invalid ipak magic '{:#x}'.", header.magic); return false; } + SwapBytesIfNecessary(header.version); if (header.version != ipak_consts::IPAK_VERSION) { con::error("Unsupported ipak version '{}'.", header.version); return false; } + SwapBytesIfNecessary(header.size); + SwapBytesIfNecessary(header.sectionCount); for (unsigned section = 0; section < header.sectionCount; section++) { if (!ReadSection()) @@ -189,17 +214,26 @@ namespace return true; } + template void SwapBytesIfNecessary(T& value) + { + if (m_little_endian) + value = endianness::FromLittleEndian(value); + else + value = endianness::FromBigEndian(value); + } + std::string m_path; std::unique_ptr m_stream; bool m_initialized; + bool m_little_endian; std::unique_ptr m_index_section; std::unique_ptr m_data_section; std::vector m_index_entries; - IPakStreamManager m_stream_manager; + std::unique_ptr m_stream_manager; }; } // namespace diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index 58a4e6b3..72a5ec21 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -11,9 +11,14 @@ using namespace ipak_consts; -IPakEntryReadStream::IPakEntryReadStream( - std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, const int64_t startOffset, const size_t entrySize) +IPakEntryReadStream::IPakEntryReadStream(std::istream& stream, + const bool isLittleEndian, + IPakStreamManagerActions* streamManagerActions, + uint8_t* chunkBuffer, + const int64_t startOffset, + const size_t entrySize) : m_chunk_buffer(chunkBuffer), + m_little_endian(isLittleEndian), m_stream(stream), m_stream_manager_actions(streamManagerActions), m_file_offset(0), @@ -198,6 +203,13 @@ bool IPakEntryReadStream::NextBlock() return false; m_current_block = reinterpret_cast(&m_chunk_buffer[blockOffsetInChunk]); + SwapBytesIfNecessary(m_current_block->countAndOffset.raw); + for (auto& command : m_current_block->commands) + { + auto size = command.size; + SwapBytesIfNecessary(size); + command.size = size; + } if (!ValidateBlockHeader(m_current_block)) return false; @@ -231,6 +243,11 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com m_current_command_offset = 0; m_file_head += static_cast(outputSize); } + else if (compressed == 2) + { + // This seems to use XMemDecompress + assert(false); + } else { // Do not process data but instead skip specified commandSize diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h index e7861b9b..5e574be8 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h @@ -2,14 +2,17 @@ #include "IPakStreamManager.h" #include "ObjContainer/IPak/IPakTypes.h" +#include "Utils/Endianness.h" #include "Utils/ObjStream.h" +#include #include class IPakEntryReadStream final : public objbuf { public: - IPakEntryReadStream(std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize); + IPakEntryReadStream( + std::istream& stream, bool isLittleEndian, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize); ~IPakEntryReadStream() override; [[nodiscard]] bool is_open() const override; @@ -34,6 +37,14 @@ private: return num / alignTo * alignTo; } + template void SwapBytesIfNecessary(T& value) + { + if (m_little_endian) + value = endianness::FromLittleEndian(value); + else + value = endianness::FromBigEndian(value); + } + /** * \brief Reads the specified chunks from disk. * \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data. @@ -90,6 +101,8 @@ private: uint8_t* m_chunk_buffer; + bool m_little_endian; + std::istream& m_stream; IPakStreamManagerActions* m_stream_manager_actions; diff --git a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp index 3243ebf6..d1fc6b4c 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp @@ -8,10 +8,8 @@ using namespace ipak_consts; -class IPakStreamManager::Impl final : public IPakStreamManagerActions +namespace { - static constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3; - class ChunkBuffer { public: @@ -32,125 +30,116 @@ class IPakStreamManager::Impl final : public IPakStreamManagerActions } }; - std::istream& m_stream; + constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3; - std::mutex m_read_mutex; - std::mutex m_stream_mutex; - - std::vector m_open_streams; - std::vector m_chunk_buffers; - -public: - explicit Impl(std::istream& stream) - : m_stream(stream) + class IPakStreamManagerImpl final : public IPakStreamManager, public IPakStreamManagerActions { - m_chunk_buffers.push_back(new ChunkBuffer()); - } - - Impl(const Impl& other) = delete; - Impl(Impl&& other) noexcept = delete; - - virtual ~Impl() - { - m_stream_mutex.lock(); - - for (const auto& openStream : m_open_streams) + public: + IPakStreamManagerImpl(std::istream& stream, const bool isLittleEndian) + : m_stream(stream), + m_little_endian(isLittleEndian) { - openStream.m_stream->close(); + m_chunk_buffers.push_back(new ChunkBuffer()); } - m_open_streams.clear(); - m_stream_mutex.unlock(); - } - - Impl& operator=(const Impl& other) = delete; - Impl& operator=(Impl&& other) noexcept = delete; - - std::unique_ptr OpenStream(const int64_t startPosition, const size_t length) - { - m_stream_mutex.lock(); - - ChunkBuffer* reservedChunkBuffer; - const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers, - [](ChunkBuffer* chunkBuffer) - { - return chunkBuffer->m_using_stream == nullptr; - }); - - if (freeChunkBuffer == m_chunk_buffers.end()) + ~IPakStreamManagerImpl() override { - reservedChunkBuffer = new ChunkBuffer(); - m_chunk_buffers.push_back(reservedChunkBuffer); - } - else - reservedChunkBuffer = *freeChunkBuffer; + m_stream_mutex.lock(); - auto ipakEntryStream = std::make_unique(m_stream, this, reservedChunkBuffer->m_buffer, startPosition, length); - - reservedChunkBuffer->m_using_stream = ipakEntryStream.get(); - - m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer); - - m_stream_mutex.unlock(); - - return std::make_unique(std::move(ipakEntryStream)); - } - - void StartReading() override - { - m_read_mutex.lock(); - } - - void StopReading() override - { - m_read_mutex.unlock(); - } - - void CloseStream(objbuf* stream) override - { - m_stream_mutex.lock(); - - const auto openStreamEntry = std::ranges::find_if(m_open_streams, - [stream](const ManagedStream& managedStream) - { - return managedStream.m_stream == stream; - }); - - if (openStreamEntry != m_open_streams.end()) - { - auto* chunkBuffer = openStreamEntry->m_chunk_buffer; - m_open_streams.erase(openStreamEntry); - chunkBuffer->m_using_stream = nullptr; - - // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers - if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT) + for (const auto& openStream : m_open_streams) { - const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer); + openStream.m_stream->close(); + } + m_open_streams.clear(); - if (chunkBufferEntry != m_chunk_buffers.end()) + m_stream_mutex.unlock(); + } + + std::unique_ptr OpenStream(const int64_t startPosition, const size_t length) override + { + m_stream_mutex.lock(); + + ChunkBuffer* reservedChunkBuffer; + const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers, + [](ChunkBuffer* chunkBuffer) + { + return chunkBuffer->m_using_stream == nullptr; + }); + + if (freeChunkBuffer == m_chunk_buffers.end()) + { + reservedChunkBuffer = new ChunkBuffer(); + m_chunk_buffers.push_back(reservedChunkBuffer); + } + else + reservedChunkBuffer = *freeChunkBuffer; + + auto ipakEntryStream = std::make_unique(m_stream, m_little_endian, this, reservedChunkBuffer->m_buffer, startPosition, length); + + reservedChunkBuffer->m_using_stream = ipakEntryStream.get(); + + m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer); + + m_stream_mutex.unlock(); + + return std::make_unique(std::move(ipakEntryStream)); + } + + void StartReading() override + { + m_read_mutex.lock(); + } + + void StopReading() override + { + m_read_mutex.unlock(); + } + + void CloseStream(objbuf* stream) override + { + m_stream_mutex.lock(); + + const auto openStreamEntry = std::ranges::find_if(m_open_streams, + [stream](const ManagedStream& managedStream) + { + return managedStream.m_stream == stream; + }); + + if (openStreamEntry != m_open_streams.end()) + { + auto* chunkBuffer = openStreamEntry->m_chunk_buffer; + m_open_streams.erase(openStreamEntry); + chunkBuffer->m_using_stream = nullptr; + + // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers + if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT) { - m_chunk_buffers.erase(chunkBufferEntry); - delete chunkBuffer; + const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer); + + if (chunkBufferEntry != m_chunk_buffers.end()) + { + m_chunk_buffers.erase(chunkBufferEntry); + delete chunkBuffer; + } } } + + m_stream_mutex.unlock(); } - m_stream_mutex.unlock(); - } -}; + private: + std::istream& m_stream; + bool m_little_endian; -IPakStreamManager::IPakStreamManager(std::istream& stream) - : m_impl(new Impl(stream)) -{ -} + std::mutex m_read_mutex; + std::mutex m_stream_mutex; -IPakStreamManager::~IPakStreamManager() -{ - delete m_impl; - m_impl = nullptr; -} + std::vector m_open_streams; + std::vector m_chunk_buffers; + }; +} // namespace -std::unique_ptr IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) const +std::unique_ptr IPakStreamManager::Create(std::istream& stream, const bool isLittleEndian) { - return m_impl->OpenStream(startPosition, length); + return std::make_unique(stream, isLittleEndian); } diff --git a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h index b443e80e..6877b2d8 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h @@ -4,6 +4,7 @@ #include #include +#include #include class IPakStreamManagerActions @@ -17,17 +18,11 @@ public: class IPakStreamManager { - class Impl; - Impl* m_impl; - public: - explicit IPakStreamManager(std::istream& stream); - IPakStreamManager(const IPakStreamManager& other) = delete; - IPakStreamManager(IPakStreamManager&& other) noexcept = delete; - ~IPakStreamManager(); + IPakStreamManager() = default; + virtual ~IPakStreamManager() = default; - IPakStreamManager& operator=(const IPakStreamManager& other) = delete; - IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete; + static std::unique_ptr Create(std::istream& stream, bool isLittleEndian); - [[nodiscard]] std::unique_ptr OpenStream(int64_t startPosition, size_t length) const; + [[nodiscard]] virtual std::unique_ptr OpenStream(int64_t startPosition, size_t length) = 0; };