From 36b4ec778143d1b1df92b38a5192c34ede812815 Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Mon, 11 May 2026 21:58:25 +0200 Subject: [PATCH 1/6] feat: add support for extracting ipaks to ImageConverter --- src/ImageConverter.lua | 2 + src/ImageConverter/ImageConverter.cpp | 106 ++++++++++++++++------ src/ImageConverter/ImageConverter.h | 23 +---- src/ImageConverter/main.cpp | 4 +- src/ObjLoading/ObjContainer/IPak/IPak.cpp | 11 ++- src/ObjLoading/ObjContainer/IPak/IPak.h | 12 ++- 6 files changed, 98 insertions(+), 60 deletions(-) diff --git a/src/ImageConverter.lua b/src/ImageConverter.lua index a34fa239..5d1f540e 100644 --- a/src/ImageConverter.lua +++ b/src/ImageConverter.lua @@ -39,12 +39,14 @@ function ImageConverter:project() self:include(includes) Utils:include(includes) Common:include(includes) + ObjLoading:include(includes) ObjImage:include(includes) Raw:use() links:linkto(Utils) links:linkto(Common) + links:linkto(ObjLoading) links:linkto(ObjImage) links:linkall() end diff --git a/src/ImageConverter/ImageConverter.cpp b/src/ImageConverter/ImageConverter.cpp index 89cde7f4..5b5694cb 100644 --- a/src/ImageConverter/ImageConverter.cpp +++ b/src/ImageConverter/ImageConverter.cpp @@ -8,8 +8,8 @@ #include "Image/IwiWriter27.h" #include "Image/IwiWriter6.h" #include "Image/IwiWriter8.h" -#include "Image/Texture.h" #include "ImageConverterArgs.h" +#include "ObjContainer/IPak/IPak.h" #include "Utils/Logging/Log.h" #include "Utils/StringUtils.h" @@ -27,36 +27,17 @@ namespace { constexpr auto EXTENSION_IWI = ".iwi"; constexpr auto EXTENSION_DDS = ".dds"; + constexpr auto EXTENSION_IPAK = ".ipak"; - class ImageConverterImpl final : public ImageConverter + class ImageConverter { public: - ImageConverterImpl() - : m_game_to_convert_to(std::nullopt) + explicit ImageConverter(const ImageConverterArgs& args) + : m_game_to_convert_to(args.m_game_to_convert_to) { } - bool Start(const int argc, const char** argv) override - { - con::init(); - - auto shouldContinue = true; - if (!m_args.ParseArgs(argc, argv, shouldContinue)) - return false; - - if (!shouldContinue) - return true; - - m_game_to_convert_to = m_args.m_game_to_convert_to; - - for (const auto& file : m_args.m_files_to_convert) - Convert(file); - - return true; - } - - private: - void Convert(const std::string& file) + void HandleFile(const std::string& file) { const fs::path filePath(file); auto extension = filePath.extension().string(); @@ -66,10 +47,13 @@ namespace ConvertIwi(filePath); else if (extension == EXTENSION_DDS) ConvertDds(filePath); + else if (extension == EXTENSION_IPAK) + ExtractIpak(filePath); else con::error("Unsupported extension {}", extension); } + private: bool ConvertIwi(const fs::path& iwiPath) { std::ifstream file(iwiPath, std::ios::in | std::ios::binary); @@ -195,6 +179,60 @@ namespace return true; } + bool ExtractIpak(const fs::path& ipakPath) + { + auto file = std::make_unique(ipakPath, std::ios::in | std::ios::binary); + if (!file->is_open()) + { + con::error("Failed to open ipak {}", ipakPath.string()); + return false; + } + + auto ipak = IIPak::Create(ipakPath.string(), std::move(file)); + if (!ipak->Initialize()) + { + con::error("Failed to read ipak {}", ipakPath.string()); + return false; + } + + const auto outDir = fs::absolute(ipakPath).parent_path() / ipakPath.filename().replace_extension(); + fs::create_directories(outDir); + + for (const auto& indexEntry : ipak->GetIndexEntries()) + { + const auto fileName = std::format("{:6x}_{:6x}.iwi", indexEntry.key.dataHash, indexEntry.key.nameHash); + std::ofstream outFile(outDir / fileName, std::ios::out | std::ios::binary); + if (!outFile.is_open()) + { + con::error("Failed to open ipak file {}", fileName); + return false; + } + + auto entryStream = ipak->GetEntryStream(indexEntry.key.nameHash, indexEntry.key.dataHash); + if (!entryStream) + { + con::error("Failed to open entry stream for {}", fileName); + return false; + } + + char buffer[0x2000]; + entryStream->read(buffer, 0x2000); + auto readCount = entryStream->gcount(); + while (readCount > 0) + { + outFile.write(buffer, readCount); + + entryStream->read(buffer, 0x2000); + readCount = entryStream->gcount(); + } + + entryStream->close(); + outFile.close(); + } + + return true; + } + ImageConverterArgs m_args; std::optional m_game_to_convert_to; DdsWriter m_dds_writer; @@ -202,7 +240,21 @@ namespace }; } // namespace -std::unique_ptr ImageConverter::Create() +bool RunImageConverter(const int argc, const char** argv) { - return std::make_unique(); + con::init(); + + auto shouldContinue = true; + ImageConverterArgs args; + if (!args.ParseArgs(argc, argv, shouldContinue)) + return false; + + if (!shouldContinue) + return true; + + ImageConverter imageConverter(args); + for (const auto& file : args.m_files_to_convert) + imageConverter.HandleFile(file); + + return true; } diff --git a/src/ImageConverter/ImageConverter.h b/src/ImageConverter/ImageConverter.h index d0e37a87..2e6cfc71 100644 --- a/src/ImageConverter/ImageConverter.h +++ b/src/ImageConverter/ImageConverter.h @@ -1,24 +1,3 @@ #pragma once -#include -class ImageConverter -{ -public: - ImageConverter() = default; - virtual ~ImageConverter() = default; - - ImageConverter(const ImageConverter& other) = delete; - ImageConverter(ImageConverter&& other) noexcept = delete; - ImageConverter& operator=(const ImageConverter& other) = delete; - ImageConverter& operator=(ImageConverter&& other) noexcept = delete; - - /** - * \brief Starts the ImageConverter application logic. - * \param argc The amount of command line arguments specified. - * \param argv The command line arguments. - * \return \c true if the application was successful or \c false if an error occurred. - */ - virtual bool Start(int argc, const char** argv) = 0; - - static std::unique_ptr Create(); -}; +bool RunImageConverter(int argc, const char** argv); diff --git a/src/ImageConverter/main.cpp b/src/ImageConverter/main.cpp index 23145c36..fea7281d 100644 --- a/src/ImageConverter/main.cpp +++ b/src/ImageConverter/main.cpp @@ -2,7 +2,5 @@ int main(const int argc, const char** argv) { - const auto imageConverter = ImageConverter::Create(); - - return imageConverter->Start(argc, argv) ? 0 : 1; + return RunImageConverter(argc, argv) ? 0 : 1; } diff --git a/src/ObjLoading/ObjContainer/IPak/IPak.cpp b/src/ObjLoading/ObjContainer/IPak/IPak.cpp index f61fb95a..0585fd96 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPak.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPak.cpp @@ -55,7 +55,7 @@ namespace return true; } - [[nodiscard]] std::unique_ptr GetEntryStream(const Hash nameHash, const Hash dataHash) const override + [[nodiscard]] std::unique_ptr GetEntryStream(const IPakHash nameHash, const IPakHash dataHash) const override { const IPakIndexEntryKey wantedKey{ {.dataHash = dataHash, .nameHash = nameHash} @@ -77,6 +77,11 @@ namespace return nullptr; } + [[nodiscard]] const std::vector& GetIndexEntries() const override + { + return m_index_entries; + } + std::string GetName() override { return fs::path(m_path).filename().replace_extension("").string(); @@ -203,12 +208,12 @@ std::unique_ptr IIPak::Create(std::string path, std::unique_ptr(std::move(path), std::move(stream)); } -IIPak::Hash IIPak::HashString(const std::string& str) +IPakHash IIPak::HashString(const std::string& str) { return R_HashString(str.c_str(), 0); } -IIPak::Hash IIPak::HashData(const void* data, const size_t dataSize) +IPakHash IIPak::HashData(const void* data, const size_t dataSize) { return crc32(0, static_cast(data), static_cast(dataSize)); } diff --git a/src/ObjLoading/ObjContainer/IPak/IPak.h b/src/ObjLoading/ObjContainer/IPak/IPak.h index 308513a5..b735807a 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPak.h +++ b/src/ObjLoading/ObjContainer/IPak/IPak.h @@ -1,5 +1,6 @@ #pragma once +#include "ObjContainer/IPak/IPakTypes.h" #include "ObjContainer/ObjContainerReferenceable.h" #include "ObjContainer/ObjContainerRepository.h" #include "Utils/ObjStream.h" @@ -14,19 +15,20 @@ class IIPak : public ObjContainerReferenceable { public: static ObjContainerRepository Repository; - typedef std::uint32_t Hash; IIPak() = default; - virtual ~IIPak() = default; + ~IIPak() override = default; IIPak(const IIPak& other) = default; IIPak(IIPak&& other) noexcept = default; IIPak& operator=(const IIPak& other) = default; IIPak& operator=(IIPak&& other) noexcept = default; virtual bool Initialize() = 0; - [[nodiscard]] virtual std::unique_ptr GetEntryStream(Hash nameHash, Hash dataHash) const = 0; + [[nodiscard]] virtual std::unique_ptr GetEntryStream(IPakHash nameHash, IPakHash dataHash) const = 0; + + [[nodiscard]] virtual const std::vector& GetIndexEntries() const = 0; static std::unique_ptr Create(std::string path, std::unique_ptr stream); - static Hash HashString(const std::string& str); - static Hash HashData(const void* data, size_t dataSize); + static IPakHash HashString(const std::string& str); + static IPakHash HashData(const void* data, size_t dataSize); }; From b2c271956a02994b4b63804c274dccfba1cc71d1 Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Mon, 11 May 2026 22:16:56 +0200 Subject: [PATCH 2/6] chore: reorder methods of ipak entry read stream --- .../ObjContainer/IPak/IPakEntryReadStream.cpp | 8 +-- .../ObjContainer/IPak/IPakEntryReadStream.h | 69 ++++++++++--------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index 3174a504..918528ea 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -7,6 +7,7 @@ #include #include #include +#include using namespace ipak_consts; @@ -134,7 +135,7 @@ bool IPakEntryReadStream::ValidateBlockHeader(const IPakDataBlockHeader* blockHe } // We expect the current file to be continued where we left off - if (static_cast(blockHeader->countAndOffset.offset) != m_file_head) + if (std::cmp_not_equal(blockHeader->countAndOffset.offset, m_file_head)) { // A matching offset is only relevant if a command contains data for (unsigned currentCommand = 0; currentCommand < blockHeader->countAndOffset.count; currentCommand++) @@ -191,8 +192,7 @@ bool IPakEntryReadStream::NextBlock() auto estimatedChunksToRead = AlignForward(m_entry_size - static_cast(m_pos - m_base_pos), IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE; - if (estimatedChunksToRead > IPAK_CHUNK_COUNT_PER_READ) - estimatedChunksToRead = IPAK_CHUNK_COUNT_PER_READ; + estimatedChunksToRead = std::min(estimatedChunksToRead, IPAK_CHUNK_COUNT_PER_READ); if (!SetChunkBufferWindow(chunkStartPos, estimatedChunksToRead)) return false; @@ -324,7 +324,7 @@ std::streambuf::int_type IPakEntryReadStream::uflow() return EOF; } -std::streamsize IPakEntryReadStream::xsgetn(char* ptr, const std::streamsize count) +std::streamsize IPakEntryReadStream::xsgetn(char* ptr, std::streamsize count) { auto* destBuffer = reinterpret_cast(ptr); std::streamsize countRead = 0; diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h index da90d261..e7861b9b 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h @@ -8,31 +8,22 @@ class IPakEntryReadStream final : public objbuf { - static constexpr size_t IPAK_DECOMPRESS_BUFFER_SIZE = 0x8000; +public: + IPakEntryReadStream(std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize); + ~IPakEntryReadStream() override; - uint8_t* m_chunk_buffer; + [[nodiscard]] bool is_open() const override; + bool close() override; - std::istream& m_stream; - IPakStreamManagerActions* m_stream_manager_actions; - - int64_t m_file_offset; - int64_t m_file_head; - - size_t m_entry_size; - - uint8_t m_decompress_buffer[IPAK_DECOMPRESS_BUFFER_SIZE]; - IPakDataBlockHeader* m_current_block; - unsigned m_next_command; - uint8_t* m_current_command_buffer; - size_t m_current_command_length; - size_t m_current_command_offset; - - int64_t m_pos; - int64_t m_base_pos; - int64_t m_end_pos; - int64_t m_buffer_start_pos; - int64_t m_buffer_end_pos; +protected: + std::streamsize showmanyc() override; + int_type underflow() override; + int_type uflow() override; + std::streamsize xsgetn(char* ptr, std::streamsize count) override; + pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode mode) override; + pos_type seekpos(pos_type pos, std::ios_base::openmode mode) override; +private: template static T AlignForward(const T num, const T alignTo) { return (num + alignTo - 1) / alignTo * alignTo; @@ -95,18 +86,28 @@ class IPakEntryReadStream final : public objbuf */ bool AdvanceStream(); -public: - IPakEntryReadStream(std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize); - ~IPakEntryReadStream() override; + static constexpr size_t IPAK_DECOMPRESS_BUFFER_SIZE = 0x8000; - [[nodiscard]] bool is_open() const override; - bool close() override; + uint8_t* m_chunk_buffer; -protected: - std::streamsize showmanyc() override; - int_type underflow() override; - int_type uflow() override; - std::streamsize xsgetn(char* ptr, std::streamsize count) override; - pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode mode) override; - pos_type seekpos(pos_type pos, std::ios_base::openmode mode) override; + std::istream& m_stream; + IPakStreamManagerActions* m_stream_manager_actions; + + int64_t m_file_offset; + int64_t m_file_head; + + size_t m_entry_size; + + uint8_t m_decompress_buffer[IPAK_DECOMPRESS_BUFFER_SIZE]; + IPakDataBlockHeader* m_current_block; + unsigned m_next_command; + uint8_t* m_current_command_buffer; + size_t m_current_command_length; + size_t m_current_command_offset; + + int64_t m_pos; + int64_t m_base_pos; + int64_t m_end_pos; + int64_t m_buffer_start_pos; + int64_t m_buffer_end_pos; }; From 7ae7cf85ff763f66c05a30c976bddfa62b9a1395 Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Mon, 11 May 2026 22:20:57 +0200 Subject: [PATCH 3/6] fix: make sure ipak reading stops in time --- src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index 918528ea..58a4e6b3 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -182,11 +182,11 @@ bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(const IPakDataBl bool IPakEntryReadStream::NextBlock() { + m_pos = AlignForward(m_pos, sizeof(IPakDataBlockHeader)); + if (m_pos >= m_end_pos) return false; - m_pos = AlignForward(m_pos, sizeof(IPakDataBlockHeader)); - const auto chunkStartPos = AlignBackwards(m_pos, IPAK_CHUNK_SIZE); const auto blockOffsetInChunk = static_cast(m_pos - chunkStartPos); From e1bb8ae4d289e3942f0cd33760fb5edad0797063 Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Mon, 11 May 2026 23:54:25 +0200 Subject: [PATCH 4/6] chore: enable reading of big endian ipaks --- src/ObjCommon/ObjContainer/IPak/IPakTypes.h | 14 +- src/ObjCompiling/Image/IPak/IPakCreator.cpp | 2 +- src/ObjLoading/ObjContainer/IPak/IPak.cpp | 44 +++- .../ObjContainer/IPak/IPakEntryReadStream.cpp | 21 +- .../ObjContainer/IPak/IPakEntryReadStream.h | 15 +- .../ObjContainer/IPak/IPakStreamManager.cpp | 201 +++++++++--------- .../ObjContainer/IPak/IPakStreamManager.h | 15 +- 7 files changed, 183 insertions(+), 129 deletions(-) diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index 441e438c..6f08cc9e 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -6,7 +6,8 @@ namespace ipak_consts { - static constexpr uint32_t IPAK_MAGIC = utils::MakeMagic32('K', 'A', 'P', 'I'); + static constexpr uint32_t IPAK_MAGIC_LITTLE_ENDIAN = utils::MakeMagic32('K', 'A', 'P', 'I'); + static constexpr uint32_t IPAK_MAGIC_BIG_ENDIAN = utils::MakeMagic32('I', 'P', 'A', 'K'); static constexpr uint32_t IPAK_VERSION = 0x50000; static constexpr uint32_t IPAK_INDEX_SECTION = 1; @@ -60,10 +61,15 @@ struct IPakIndexEntry uint32_t size; }; -struct IPakDataBlockCountAndOffset +union IPakDataBlockCountAndOffset { - uint32_t offset : 24; - uint32_t count : 8; + struct + { + uint32_t offset : 24; + uint32_t count : 8; + }; + + uint32_t raw; }; static_assert(sizeof(IPakDataBlockCountAndOffset) == 4); diff --git a/src/ObjCompiling/Image/IPak/IPakCreator.cpp b/src/ObjCompiling/Image/IPak/IPakCreator.cpp index 370074d2..0d8bf457 100644 --- a/src/ObjCompiling/Image/IPak/IPakCreator.cpp +++ b/src/ObjCompiling/Image/IPak/IPakCreator.cpp @@ -100,7 +100,7 @@ namespace { GoTo(0); - const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC, + const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN, .version = ipak_consts::IPAK_VERSION, .size = static_cast(m_total_size), .sectionCount = SECTION_COUNT}; diff --git a/src/ObjLoading/ObjContainer/IPak/IPak.cpp b/src/ObjLoading/ObjContainer/IPak/IPak.cpp index 0585fd96..d5eb90f7 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPak.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPak.cpp @@ -2,8 +2,10 @@ #include "IPakStreamManager.h" #include "ObjContainer/IPak/IPakTypes.h" +#include "Utils/Endianness.h" #include "Utils/Logging/Log.h" +#include #include #include #include @@ -37,9 +39,10 @@ namespace : m_path(std::move(path)), m_stream(std::move(stream)), m_initialized(false), + m_little_endian(true), m_index_section(nullptr), m_data_section(nullptr), - m_stream_manager(*m_stream) + m_stream_manager(nullptr) { } @@ -65,7 +68,7 @@ namespace { if (entry.key.combinedKey == wantedKey.combinedKey) { - return m_stream_manager.OpenStream(static_cast(m_data_section->offset) + entry.offset, entry.size); + return m_stream_manager->OpenStream(static_cast(m_data_section->offset) + entry.offset, entry.size); } else if (entry.key.combinedKey > wantedKey.combinedKey) { @@ -102,7 +105,11 @@ namespace return false; } - m_index_entries.push_back(indexEntry); + SwapBytesIfNecessary(indexEntry.key.combinedKey); + SwapBytesIfNecessary(indexEntry.offset); + SwapBytesIfNecessary(indexEntry.size); + + m_index_entries.emplace_back(indexEntry); } std::ranges::sort(m_index_entries, @@ -125,6 +132,11 @@ namespace return false; } + SwapBytesIfNecessary(section.type); + SwapBytesIfNecessary(section.offset); + SwapBytesIfNecessary(section.size); + SwapBytesIfNecessary(section.itemCount); + switch (section.type) { case ipak_consts::IPAK_INDEX_SECTION: @@ -153,18 +165,31 @@ namespace return false; } - if (header.magic != ipak_consts::IPAK_MAGIC) + if (header.magic == ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN) + { + m_little_endian = true; + m_stream_manager = IPakStreamManager::Create(*m_stream, true); + } + else if (header.magic == ipak_consts::IPAK_MAGIC_BIG_ENDIAN) + { + m_little_endian = false; + m_stream_manager = IPakStreamManager::Create(*m_stream, false); + } + else { con::error("Invalid ipak magic '{:#x}'.", header.magic); return false; } + SwapBytesIfNecessary(header.version); if (header.version != ipak_consts::IPAK_VERSION) { con::error("Unsupported ipak version '{}'.", header.version); return false; } + SwapBytesIfNecessary(header.size); + SwapBytesIfNecessary(header.sectionCount); for (unsigned section = 0; section < header.sectionCount; section++) { if (!ReadSection()) @@ -189,17 +214,26 @@ namespace return true; } + template void SwapBytesIfNecessary(T& value) + { + if (m_little_endian) + value = endianness::FromLittleEndian(value); + else + value = endianness::FromBigEndian(value); + } + std::string m_path; std::unique_ptr m_stream; bool m_initialized; + bool m_little_endian; std::unique_ptr m_index_section; std::unique_ptr m_data_section; std::vector m_index_entries; - IPakStreamManager m_stream_manager; + std::unique_ptr m_stream_manager; }; } // namespace diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index 58a4e6b3..72a5ec21 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -11,9 +11,14 @@ using namespace ipak_consts; -IPakEntryReadStream::IPakEntryReadStream( - std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, const int64_t startOffset, const size_t entrySize) +IPakEntryReadStream::IPakEntryReadStream(std::istream& stream, + const bool isLittleEndian, + IPakStreamManagerActions* streamManagerActions, + uint8_t* chunkBuffer, + const int64_t startOffset, + const size_t entrySize) : m_chunk_buffer(chunkBuffer), + m_little_endian(isLittleEndian), m_stream(stream), m_stream_manager_actions(streamManagerActions), m_file_offset(0), @@ -198,6 +203,13 @@ bool IPakEntryReadStream::NextBlock() return false; m_current_block = reinterpret_cast(&m_chunk_buffer[blockOffsetInChunk]); + SwapBytesIfNecessary(m_current_block->countAndOffset.raw); + for (auto& command : m_current_block->commands) + { + auto size = command.size; + SwapBytesIfNecessary(size); + command.size = size; + } if (!ValidateBlockHeader(m_current_block)) return false; @@ -231,6 +243,11 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com m_current_command_offset = 0; m_file_head += static_cast(outputSize); } + else if (compressed == 2) + { + // This seems to use XMemDecompress + assert(false); + } else { // Do not process data but instead skip specified commandSize diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h index e7861b9b..5e574be8 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h @@ -2,14 +2,17 @@ #include "IPakStreamManager.h" #include "ObjContainer/IPak/IPakTypes.h" +#include "Utils/Endianness.h" #include "Utils/ObjStream.h" +#include #include class IPakEntryReadStream final : public objbuf { public: - IPakEntryReadStream(std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize); + IPakEntryReadStream( + std::istream& stream, bool isLittleEndian, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize); ~IPakEntryReadStream() override; [[nodiscard]] bool is_open() const override; @@ -34,6 +37,14 @@ private: return num / alignTo * alignTo; } + template void SwapBytesIfNecessary(T& value) + { + if (m_little_endian) + value = endianness::FromLittleEndian(value); + else + value = endianness::FromBigEndian(value); + } + /** * \brief Reads the specified chunks from disk. * \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data. @@ -90,6 +101,8 @@ private: uint8_t* m_chunk_buffer; + bool m_little_endian; + std::istream& m_stream; IPakStreamManagerActions* m_stream_manager_actions; diff --git a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp index 3243ebf6..d1fc6b4c 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp @@ -8,10 +8,8 @@ using namespace ipak_consts; -class IPakStreamManager::Impl final : public IPakStreamManagerActions +namespace { - static constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3; - class ChunkBuffer { public: @@ -32,125 +30,116 @@ class IPakStreamManager::Impl final : public IPakStreamManagerActions } }; - std::istream& m_stream; + constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3; - std::mutex m_read_mutex; - std::mutex m_stream_mutex; - - std::vector m_open_streams; - std::vector m_chunk_buffers; - -public: - explicit Impl(std::istream& stream) - : m_stream(stream) + class IPakStreamManagerImpl final : public IPakStreamManager, public IPakStreamManagerActions { - m_chunk_buffers.push_back(new ChunkBuffer()); - } - - Impl(const Impl& other) = delete; - Impl(Impl&& other) noexcept = delete; - - virtual ~Impl() - { - m_stream_mutex.lock(); - - for (const auto& openStream : m_open_streams) + public: + IPakStreamManagerImpl(std::istream& stream, const bool isLittleEndian) + : m_stream(stream), + m_little_endian(isLittleEndian) { - openStream.m_stream->close(); + m_chunk_buffers.push_back(new ChunkBuffer()); } - m_open_streams.clear(); - m_stream_mutex.unlock(); - } - - Impl& operator=(const Impl& other) = delete; - Impl& operator=(Impl&& other) noexcept = delete; - - std::unique_ptr OpenStream(const int64_t startPosition, const size_t length) - { - m_stream_mutex.lock(); - - ChunkBuffer* reservedChunkBuffer; - const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers, - [](ChunkBuffer* chunkBuffer) - { - return chunkBuffer->m_using_stream == nullptr; - }); - - if (freeChunkBuffer == m_chunk_buffers.end()) + ~IPakStreamManagerImpl() override { - reservedChunkBuffer = new ChunkBuffer(); - m_chunk_buffers.push_back(reservedChunkBuffer); - } - else - reservedChunkBuffer = *freeChunkBuffer; + m_stream_mutex.lock(); - auto ipakEntryStream = std::make_unique(m_stream, this, reservedChunkBuffer->m_buffer, startPosition, length); - - reservedChunkBuffer->m_using_stream = ipakEntryStream.get(); - - m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer); - - m_stream_mutex.unlock(); - - return std::make_unique(std::move(ipakEntryStream)); - } - - void StartReading() override - { - m_read_mutex.lock(); - } - - void StopReading() override - { - m_read_mutex.unlock(); - } - - void CloseStream(objbuf* stream) override - { - m_stream_mutex.lock(); - - const auto openStreamEntry = std::ranges::find_if(m_open_streams, - [stream](const ManagedStream& managedStream) - { - return managedStream.m_stream == stream; - }); - - if (openStreamEntry != m_open_streams.end()) - { - auto* chunkBuffer = openStreamEntry->m_chunk_buffer; - m_open_streams.erase(openStreamEntry); - chunkBuffer->m_using_stream = nullptr; - - // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers - if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT) + for (const auto& openStream : m_open_streams) { - const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer); + openStream.m_stream->close(); + } + m_open_streams.clear(); - if (chunkBufferEntry != m_chunk_buffers.end()) + m_stream_mutex.unlock(); + } + + std::unique_ptr OpenStream(const int64_t startPosition, const size_t length) override + { + m_stream_mutex.lock(); + + ChunkBuffer* reservedChunkBuffer; + const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers, + [](ChunkBuffer* chunkBuffer) + { + return chunkBuffer->m_using_stream == nullptr; + }); + + if (freeChunkBuffer == m_chunk_buffers.end()) + { + reservedChunkBuffer = new ChunkBuffer(); + m_chunk_buffers.push_back(reservedChunkBuffer); + } + else + reservedChunkBuffer = *freeChunkBuffer; + + auto ipakEntryStream = std::make_unique(m_stream, m_little_endian, this, reservedChunkBuffer->m_buffer, startPosition, length); + + reservedChunkBuffer->m_using_stream = ipakEntryStream.get(); + + m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer); + + m_stream_mutex.unlock(); + + return std::make_unique(std::move(ipakEntryStream)); + } + + void StartReading() override + { + m_read_mutex.lock(); + } + + void StopReading() override + { + m_read_mutex.unlock(); + } + + void CloseStream(objbuf* stream) override + { + m_stream_mutex.lock(); + + const auto openStreamEntry = std::ranges::find_if(m_open_streams, + [stream](const ManagedStream& managedStream) + { + return managedStream.m_stream == stream; + }); + + if (openStreamEntry != m_open_streams.end()) + { + auto* chunkBuffer = openStreamEntry->m_chunk_buffer; + m_open_streams.erase(openStreamEntry); + chunkBuffer->m_using_stream = nullptr; + + // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers + if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT) { - m_chunk_buffers.erase(chunkBufferEntry); - delete chunkBuffer; + const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer); + + if (chunkBufferEntry != m_chunk_buffers.end()) + { + m_chunk_buffers.erase(chunkBufferEntry); + delete chunkBuffer; + } } } + + m_stream_mutex.unlock(); } - m_stream_mutex.unlock(); - } -}; + private: + std::istream& m_stream; + bool m_little_endian; -IPakStreamManager::IPakStreamManager(std::istream& stream) - : m_impl(new Impl(stream)) -{ -} + std::mutex m_read_mutex; + std::mutex m_stream_mutex; -IPakStreamManager::~IPakStreamManager() -{ - delete m_impl; - m_impl = nullptr; -} + std::vector m_open_streams; + std::vector m_chunk_buffers; + }; +} // namespace -std::unique_ptr IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) const +std::unique_ptr IPakStreamManager::Create(std::istream& stream, const bool isLittleEndian) { - return m_impl->OpenStream(startPosition, length); + return std::make_unique(stream, isLittleEndian); } diff --git a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h index b443e80e..6877b2d8 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h @@ -4,6 +4,7 @@ #include #include +#include #include class IPakStreamManagerActions @@ -17,17 +18,11 @@ public: class IPakStreamManager { - class Impl; - Impl* m_impl; - public: - explicit IPakStreamManager(std::istream& stream); - IPakStreamManager(const IPakStreamManager& other) = delete; - IPakStreamManager(IPakStreamManager&& other) noexcept = delete; - ~IPakStreamManager(); + IPakStreamManager() = default; + virtual ~IPakStreamManager() = default; - IPakStreamManager& operator=(const IPakStreamManager& other) = delete; - IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete; + static std::unique_ptr Create(std::istream& stream, bool isLittleEndian); - [[nodiscard]] std::unique_ptr OpenStream(int64_t startPosition, size_t length) const; + [[nodiscard]] virtual std::unique_ptr OpenStream(int64_t startPosition, size_t length) = 0; }; From 71ca182524e52b2065c9a1a888b697734a67762d Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Tue, 12 May 2026 22:02:52 +0200 Subject: [PATCH 5/6] feat: properly parse data from xenon ipaks --- premake5.lua | 2 + src/ImageConverter/ImageConverter.cpp | 8 +- src/ObjCommon/ObjContainer/IPak/IPakTypes.h | 11 +- src/ObjLoading.lua | 2 + .../ObjContainer/IPak/IPakEntryReadStream.cpp | 22 ++- .../ObjContainer/IPak/IPakEntryReadStream.h | 2 + src/XMemCompress.lua | 49 ++++++ src/XMemCompress/XMemDecompress.cpp | 141 ++++++++++++++++++ src/XMemCompress/XMemDecompress.h | 21 +++ src/ZoneCommon.lua | 4 +- .../XChunk/XChunkProcessorLzxDecompress.cpp | 124 +-------------- .../XChunk/XChunkProcessorLzxDecompress.h | 4 +- 12 files changed, 261 insertions(+), 129 deletions(-) create mode 100644 src/XMemCompress.lua create mode 100644 src/XMemCompress/XMemDecompress.cpp create mode 100644 src/XMemCompress/XMemDecompress.h diff --git a/premake5.lua b/premake5.lua index 5cfb7552..ef0a3d56 100644 --- a/premake5.lua +++ b/premake5.lua @@ -136,6 +136,7 @@ include "src/RawTemplater.lua" include "src/UnlinkerCli.lua" include "src/Unlinking.lua" include "src/Utils.lua" +include "src/XMemCompress.lua" include "src/ZoneCode.lua" include "src/ZoneCodeGeneratorLib.lua" include "src/ZoneCodeGenerator.lua" @@ -156,6 +157,7 @@ group "Components" Cryptography:project() Parser:project() Utils:project() + XMemCompress:project() ZoneCode:project() ZoneCodeGeneratorLib:project() ZoneCommon:project() diff --git a/src/ImageConverter/ImageConverter.cpp b/src/ImageConverter/ImageConverter.cpp index 5b5694cb..75ea81e9 100644 --- a/src/ImageConverter/ImageConverter.cpp +++ b/src/ImageConverter/ImageConverter.cpp @@ -200,7 +200,7 @@ namespace for (const auto& indexEntry : ipak->GetIndexEntries()) { - const auto fileName = std::format("{:6x}_{:6x}.iwi", indexEntry.key.dataHash, indexEntry.key.nameHash); + const auto fileName = std::format("{:0>6x}_{:0>6x}.iwi", indexEntry.key.dataHash, indexEntry.key.nameHash); std::ofstream outFile(outDir / fileName, std::ios::out | std::ios::binary); if (!outFile.is_open()) { @@ -216,18 +216,20 @@ namespace } char buffer[0x2000]; - entryStream->read(buffer, 0x2000); + entryStream->read(buffer, sizeof(buffer)); auto readCount = entryStream->gcount(); while (readCount > 0) { outFile.write(buffer, readCount); - entryStream->read(buffer, 0x2000); + entryStream->read(buffer, sizeof(buffer)); readCount = entryStream->gcount(); } entryStream->close(); outFile.close(); + + con::info("Dumped {}", fileName); } return true; diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index 6f08cc9e..88c2a05a 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -74,10 +74,15 @@ union IPakDataBlockCountAndOffset static_assert(sizeof(IPakDataBlockCountAndOffset) == 4); -struct IPakDataBlockCommand +union IPakDataBlockCommand { - uint32_t size : 24; - uint32_t compressed : 8; + struct + { + uint32_t size : 24; + uint32_t compressed : 8; + }; + + uint32_t raw; }; static_assert(sizeof(IPakDataBlockCommand) == 4); diff --git a/src/ObjLoading.lua b/src/ObjLoading.lua index 9325ca88..153bb97a 100644 --- a/src/ObjLoading.lua +++ b/src/ObjLoading.lua @@ -16,6 +16,7 @@ function ObjLoading:link(links) links:linkto(Utils) links:linkto(ObjCommon) links:linkto(ObjImage) + links:linkto(XMemCompress) links:linkto(ZoneCommon) links:linkto(minilzo) links:linkto(minizip) @@ -58,6 +59,7 @@ function ObjLoading:project() self:include(includes) Cryptography:include(includes) Utils:include(includes) + XMemCompress:include(includes) minilzo:include(includes) minizip:include(includes) zlib:include(includes) diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index 72a5ec21..324a6d83 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -206,8 +206,12 @@ bool IPakEntryReadStream::NextBlock() SwapBytesIfNecessary(m_current_block->countAndOffset.raw); for (auto& command : m_current_block->commands) { + if (!m_little_endian) + command.raw &= 0xFFFFFFDF; // ? idk, the game seems to do this? halp + + SwapBytesIfNecessary(command.raw); + auto size = command.size; - SwapBytesIfNecessary(size); command.size = size; } @@ -245,8 +249,20 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com } else if (compressed == 2) { - // This seems to use XMemDecompress - assert(false); + m_xmemdecompress_context.Reset(); + const auto maybeDecompressSize = m_xmemdecompress_context.Process( + &m_chunk_buffer[m_pos - m_buffer_start_pos], static_cast(commandSize), m_decompress_buffer, sizeof(m_decompress_buffer)); + + if (!maybeDecompressSize.has_value()) + { + con::error("Decompressing block with XMemDecompress failed!"); + return false; + } + + m_current_command_buffer = m_decompress_buffer; + m_current_command_length = *maybeDecompressSize; + m_current_command_offset = 0; + m_file_head += static_cast(*maybeDecompressSize); } else { diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h index 5e574be8..46261ceb 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h @@ -4,6 +4,7 @@ #include "ObjContainer/IPak/IPakTypes.h" #include "Utils/Endianness.h" #include "Utils/ObjStream.h" +#include "XMemDecompress.h" #include #include @@ -102,6 +103,7 @@ private: uint8_t* m_chunk_buffer; bool m_little_endian; + XMemDecompressContext m_xmemdecompress_context; std::istream& m_stream; IPakStreamManagerActions* m_stream_manager_actions; diff --git a/src/XMemCompress.lua b/src/XMemCompress.lua new file mode 100644 index 00000000..60720cf5 --- /dev/null +++ b/src/XMemCompress.lua @@ -0,0 +1,49 @@ +XMemCompress = {} + +function XMemCompress:include(includes) + if includes:handle(self:name()) then + includedirs { + path.join(ProjectFolder(), "XMemCompress") + } + end +end + +function XMemCompress:link(links) + links:add(self:name()) + links:linkto(Utils) + links:linkto(lzx) +end + +function XMemCompress:use() + +end + +function XMemCompress:name() + return "XMemCompress" +end + +function XMemCompress:project() + local folder = ProjectFolder() + local includes = Includes:create() + + project(self:name()) + targetdir(TargetDirectoryLib) + location "%{wks.location}/src/%{prj.name}" + kind "StaticLib" + language "C++" + + files { + path.join(folder, "XMemCompress/**.h"), + path.join(folder, "XMemCompress/**.cpp") + } + + vpaths { + ["*"] = { + path.join(folder, "XMemCompress") + } + } + + self:include(includes) + Utils:include(includes) + lzx:include(includes) +end diff --git a/src/XMemCompress/XMemDecompress.cpp b/src/XMemCompress/XMemDecompress.cpp new file mode 100644 index 00000000..4953c6e7 --- /dev/null +++ b/src/XMemCompress/XMemDecompress.cpp @@ -0,0 +1,141 @@ +#include "XMemDecompress.h" + +#include "Utils/Logging/Log.h" + +#include +#include + +namespace +{ + uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize) + { + const auto value = input[offset]; + offset++; + remainingSize--; + return value; + } + + uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte) + { + return static_cast(static_cast(static_cast(highByte) << 8u) | static_cast(lowByte)); + } + + void LogErrorHeaderSpace(size_t remainingInputSize) + { + con::error("XMemDecompress: Not enough data for header: {}", remainingInputSize); + } +} // namespace + +XMemDecompressContext::XMemDecompressContext() + : m_lzx_state(lzx_init(17)) +{ +} + +XMemDecompressContext::~XMemDecompressContext() +{ + if (m_lzx_state) + lzx_teardown(static_cast(m_lzx_state)); +} + +XMemDecompressContext::XMemDecompressContext(XMemDecompressContext&& other) noexcept + : m_lzx_state(other.m_lzx_state) +{ + other.m_lzx_state = nullptr; +} + +XMemDecompressContext& XMemDecompressContext::operator=(XMemDecompressContext&& other) noexcept +{ + m_lzx_state = other.m_lzx_state; + other.m_lzx_state = nullptr; + return *this; +} + +void XMemDecompressContext::Reset() const +{ + lzx_reset(static_cast(m_lzx_state)); +} + +std::optional XMemDecompressContext::Process(const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) const +{ + size_t curInputOffset = 0uz; + size_t curInputSize = inputLength; + + size_t curOutputOffset = 0uz; + size_t curOutputSize = outputBufferSize; + + uint8_t lowByte; + uint16_t dstSize, srcSize; + + while (curInputSize > 0) + { + uint8_t highByte = NextByte(input, curInputOffset, curInputSize); + + uint8_t suffixSize; + if (highByte == 0xFF) // magic number: output is smaller than 0x8000 + { + if (curInputSize < 4) + { + LogErrorHeaderSpace(curInputSize); + return std::nullopt; + } + + highByte = NextByte(input, curInputOffset, curInputSize); + lowByte = NextByte(input, curInputOffset, curInputSize); + dstSize = CombineHighLow(highByte, lowByte); + + highByte = NextByte(input, curInputOffset, curInputSize); + lowByte = NextByte(input, curInputOffset, curInputSize); + srcSize = CombineHighLow(highByte, lowByte); + + // The game seems to skip a 5 byte suffix after these blocks, not sure why. + suffixSize = 5u; + } + else + { + if (curInputSize < 1) + { + LogErrorHeaderSpace(curInputSize); + return std::nullopt; + } + + dstSize = 0x8000u; + lowByte = NextByte(input, curInputOffset, curInputSize); + srcSize = CombineHighLow(highByte, lowByte); + + suffixSize = 0u; + } + + if (srcSize == 0 || dstSize == 0) + { + // Other implementations do not handle this as a failure, game code suggests otherwise though + con::error("XMemDecompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize); + return std::nullopt; + } + + if (static_cast(srcSize) + suffixSize > curInputSize) + { + con::error("XMemDecompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize); + return std::nullopt; + } + + if (dstSize > curOutputSize) + { + con::error("XMemDecompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize); + return std::nullopt; + } + + auto ret = lzx_decompress(static_cast(m_lzx_state), &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize); + curInputOffset += srcSize + suffixSize; + curInputSize -= (srcSize + suffixSize); + curOutputOffset += dstSize; + curOutputSize -= srcSize; + + if (ret != DECR_OK) + { + con::error("XMemDecompress: lzx decompression failed: {}", ret); + return std::nullopt; + } + } + + return curOutputOffset; +} diff --git a/src/XMemCompress/XMemDecompress.h b/src/XMemCompress/XMemDecompress.h new file mode 100644 index 00000000..d350f615 --- /dev/null +++ b/src/XMemCompress/XMemDecompress.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +class XMemDecompressContext +{ +public: + XMemDecompressContext(); + ~XMemDecompressContext(); + XMemDecompressContext(const XMemDecompressContext& other) = delete; + XMemDecompressContext(XMemDecompressContext&& other) noexcept; + XMemDecompressContext& operator=(const XMemDecompressContext& other) = delete; + XMemDecompressContext& operator=(XMemDecompressContext&& other) noexcept; + + void Reset() const; + std::optional Process(const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) const; + +private: + void* m_lzx_state; +}; diff --git a/src/ZoneCommon.lua b/src/ZoneCommon.lua index 31cac7d4..2ea7e8ad 100644 --- a/src/ZoneCommon.lua +++ b/src/ZoneCommon.lua @@ -6,6 +6,7 @@ function ZoneCommon:include(includes) path.join(ProjectFolder(), "ZoneCommon") } Utils:include(includes) + XMemCompress:include(includes) Common:include(includes) ObjCommon:include(includes) Parser:include(includes) @@ -22,7 +23,7 @@ function ZoneCommon:link(links) links:linkto(ObjCommon) links:linkto(Parser) links:linkto(Utils) - links:linkto(lzx) + links:linkto(XMemCompress) ZoneCode:use() end @@ -58,7 +59,6 @@ function ZoneCommon:project() } self:include(includes) - lzx:include(includes) ZoneCode:include(includes) ZoneCode:use() diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp index 6778150e..1444f3d0 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp @@ -2,133 +2,25 @@ #include "Utils/Logging/Log.h" -#include -#include -#include -#include - -namespace -{ - uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize) - { - const auto value = input[offset]; - offset++; - remainingSize--; - return value; - } - - uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte) - { - return static_cast(static_cast(static_cast(highByte) << 8u) | static_cast(lowByte)); - } - - void LogErrorHeaderSpace(size_t remainingInputSize) - { - con::error("XMemCompress: Not enough data for header: {}", remainingInputSize); - } -} // namespace - XChunkProcessorLzxDecompress::XChunkProcessorLzxDecompress(const unsigned streamCount) - : m_lzx_states(streamCount) + : m_xmemdecompress_contexts(streamCount) { - // T6 uses 17 for window bits - for (auto& lzxState : m_lzx_states) - lzxState = lzx_init(17); -} - -XChunkProcessorLzxDecompress::~XChunkProcessorLzxDecompress() -{ - for (auto* lzxState : m_lzx_states) - lzx_teardown(static_cast(lzxState)); } size_t XChunkProcessorLzxDecompress::Process( const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) { - auto* state = static_cast(m_lzx_states[streamNumber]); + const auto& xMemDecompress = m_xmemdecompress_contexts[streamNumber]; // lzx state is reset before each chunk - lzx_reset(state); + xMemDecompress.Reset(); - size_t curInputOffset = 0uz; - size_t curInputSize = inputLength; - - size_t curOutputOffset = 0uz; - size_t curOutputSize = outputBufferSize; - - uint8_t lowByte; - uint16_t dstSize, srcSize; - - while (curInputSize > 0) + const auto maybeDecompressedSize = xMemDecompress.Process(input, inputLength, output, outputBufferSize); + if (!maybeDecompressedSize.has_value()) { - uint8_t highByte = NextByte(input, curInputOffset, curInputSize); - - uint8_t suffixSize; - if (highByte == 0xFF) // magic number: output is smaller than 0x8000 - { - if (curInputSize < 4) - { - LogErrorHeaderSpace(curInputSize); - return curOutputOffset; - } - - highByte = NextByte(input, curInputOffset, curInputSize); - lowByte = NextByte(input, curInputOffset, curInputSize); - dstSize = CombineHighLow(highByte, lowByte); - - highByte = NextByte(input, curInputOffset, curInputSize); - lowByte = NextByte(input, curInputOffset, curInputSize); - srcSize = CombineHighLow(highByte, lowByte); - - // The game seems to skip a 5 byte suffix after these blocks, not sure why. - suffixSize = 5u; - } - else - { - if (curInputSize < 1) - { - LogErrorHeaderSpace(curInputSize); - return curOutputOffset; - } - - dstSize = 0x8000u; - lowByte = NextByte(input, curInputOffset, curInputSize); - srcSize = CombineHighLow(highByte, lowByte); - - suffixSize = 0u; - } - - if (srcSize == 0 || dstSize == 0) - { - // Other implementations do not handle this as a failure, game code suggests otherwise though - con::error("XMemCompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize); - return curOutputOffset; - } - - if (static_cast(srcSize) + suffixSize > curInputSize) - { - con::error("XMemCompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize); - return curOutputOffset; - } - - if (dstSize > curOutputSize) - { - con::error("XMemCompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize); - return curOutputOffset; - } - - auto ret = lzx_decompress(state, &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize); - curInputOffset += srcSize + suffixSize; - curInputSize -= (srcSize + suffixSize); - curOutputOffset += dstSize; - curOutputSize -= srcSize; - - if (ret != DECR_OK) - { - con::error("XMemCompress: lzx decompression failed: {}", ret); - return curOutputOffset; - } + con::error("Failed to decompress xchunk with XMemDecompress"); + return 0; } - return curOutputOffset; + return *maybeDecompressedSize; } diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h index e32ee92a..1ba24728 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h @@ -1,6 +1,7 @@ #pragma once #include "IXChunkProcessor.h" +#include "XMemDecompress.h" #include @@ -8,9 +9,8 @@ class XChunkProcessorLzxDecompress final : public IXChunkProcessor { public: explicit XChunkProcessorLzxDecompress(unsigned streamCount); - ~XChunkProcessorLzxDecompress(); size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; private: - std::vector m_lzx_states; + std::vector m_xmemdecompress_contexts; }; From f7f4deeeccd45dac565b1667ffa4a586e66c071f Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Fri, 15 May 2026 13:18:37 +0200 Subject: [PATCH 6/6] chore: choose appropriate extension for ipak file --- src/ImageConverter/ImageConverter.cpp | 32 +++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/ImageConverter/ImageConverter.cpp b/src/ImageConverter/ImageConverter.cpp index 75ea81e9..48ff43fc 100644 --- a/src/ImageConverter/ImageConverter.cpp +++ b/src/ImageConverter/ImageConverter.cpp @@ -14,6 +14,7 @@ #include "Utils/StringUtils.h" #include +#include #include #include #include @@ -28,6 +29,15 @@ namespace constexpr auto EXTENSION_IWI = ".iwi"; constexpr auto EXTENSION_DDS = ".dds"; constexpr auto EXTENSION_IPAK = ".ipak"; + constexpr char IWI_MAGIC[3]{'I', 'W', 'i'}; + + const char* DetermineExtensionForBytes(const void* bytes, const size_t byteCount) + { + if (byteCount >= sizeof(IWI_MAGIC) && !memcmp(bytes, IWI_MAGIC, sizeof(IWI_MAGIC))) + return ".iwi"; + + return ""; + } class ImageConverter { @@ -200,24 +210,28 @@ namespace for (const auto& indexEntry : ipak->GetIndexEntries()) { - const auto fileName = std::format("{:0>6x}_{:0>6x}.iwi", indexEntry.key.dataHash, indexEntry.key.nameHash); - std::ofstream outFile(outDir / fileName, std::ios::out | std::ios::binary); - if (!outFile.is_open()) - { - con::error("Failed to open ipak file {}", fileName); - return false; - } + const auto baseFileName = std::format("{:0>6x}_{:0>6x}", indexEntry.key.dataHash, indexEntry.key.nameHash); auto entryStream = ipak->GetEntryStream(indexEntry.key.nameHash, indexEntry.key.dataHash); if (!entryStream) { - con::error("Failed to open entry stream for {}", fileName); + con::error("Failed to open entry stream for {}", baseFileName); return false; } char buffer[0x2000]; entryStream->read(buffer, sizeof(buffer)); auto readCount = entryStream->gcount(); + + const auto extension = DetermineExtensionForBytes(buffer, static_cast(readCount)); + const auto fileNameWithExtension = std::format("{}{}", baseFileName, extension); + std::ofstream outFile(outDir / fileNameWithExtension, std::ios::out | std::ios::binary); + if (!outFile.is_open()) + { + con::error("Failed to open ipak file {}", fileNameWithExtension); + return false; + } + while (readCount > 0) { outFile.write(buffer, readCount); @@ -229,7 +243,7 @@ namespace entryStream->close(); outFile.close(); - con::info("Dumped {}", fileName); + con::info("Dumped {}", fileNameWithExtension); } return true;