From 71ca182524e52b2065c9a1a888b697734a67762d Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Tue, 12 May 2026 22:02:52 +0200 Subject: [PATCH] feat: properly parse data from xenon ipaks --- premake5.lua | 2 + src/ImageConverter/ImageConverter.cpp | 8 +- src/ObjCommon/ObjContainer/IPak/IPakTypes.h | 11 +- src/ObjLoading.lua | 2 + .../ObjContainer/IPak/IPakEntryReadStream.cpp | 22 ++- .../ObjContainer/IPak/IPakEntryReadStream.h | 2 + src/XMemCompress.lua | 49 ++++++ src/XMemCompress/XMemDecompress.cpp | 141 ++++++++++++++++++ src/XMemCompress/XMemDecompress.h | 21 +++ src/ZoneCommon.lua | 4 +- .../XChunk/XChunkProcessorLzxDecompress.cpp | 124 +-------------- .../XChunk/XChunkProcessorLzxDecompress.h | 4 +- 12 files changed, 261 insertions(+), 129 deletions(-) create mode 100644 src/XMemCompress.lua create mode 100644 src/XMemCompress/XMemDecompress.cpp create mode 100644 src/XMemCompress/XMemDecompress.h diff --git a/premake5.lua b/premake5.lua index 5cfb7552..ef0a3d56 100644 --- a/premake5.lua +++ b/premake5.lua @@ -136,6 +136,7 @@ include "src/RawTemplater.lua" include "src/UnlinkerCli.lua" include "src/Unlinking.lua" include "src/Utils.lua" +include "src/XMemCompress.lua" include "src/ZoneCode.lua" include "src/ZoneCodeGeneratorLib.lua" include "src/ZoneCodeGenerator.lua" @@ -156,6 +157,7 @@ group "Components" Cryptography:project() Parser:project() Utils:project() + XMemCompress:project() ZoneCode:project() ZoneCodeGeneratorLib:project() ZoneCommon:project() diff --git a/src/ImageConverter/ImageConverter.cpp b/src/ImageConverter/ImageConverter.cpp index 5b5694cb..75ea81e9 100644 --- a/src/ImageConverter/ImageConverter.cpp +++ b/src/ImageConverter/ImageConverter.cpp @@ -200,7 +200,7 @@ namespace for (const auto& indexEntry : ipak->GetIndexEntries()) { - const auto fileName = std::format("{:6x}_{:6x}.iwi", indexEntry.key.dataHash, indexEntry.key.nameHash); + const auto fileName = std::format("{:0>6x}_{:0>6x}.iwi", indexEntry.key.dataHash, indexEntry.key.nameHash); std::ofstream outFile(outDir / fileName, std::ios::out | std::ios::binary); if (!outFile.is_open()) { @@ -216,18 +216,20 @@ namespace } char buffer[0x2000]; - entryStream->read(buffer, 0x2000); + entryStream->read(buffer, sizeof(buffer)); auto readCount = entryStream->gcount(); while (readCount > 0) { outFile.write(buffer, readCount); - entryStream->read(buffer, 0x2000); + entryStream->read(buffer, sizeof(buffer)); readCount = entryStream->gcount(); } entryStream->close(); outFile.close(); + + con::info("Dumped {}", fileName); } return true; diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index 6f08cc9e..88c2a05a 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -74,10 +74,15 @@ union IPakDataBlockCountAndOffset static_assert(sizeof(IPakDataBlockCountAndOffset) == 4); -struct IPakDataBlockCommand +union IPakDataBlockCommand { - uint32_t size : 24; - uint32_t compressed : 8; + struct + { + uint32_t size : 24; + uint32_t compressed : 8; + }; + + uint32_t raw; }; static_assert(sizeof(IPakDataBlockCommand) == 4); diff --git a/src/ObjLoading.lua b/src/ObjLoading.lua index 9325ca88..153bb97a 100644 --- a/src/ObjLoading.lua +++ b/src/ObjLoading.lua @@ -16,6 +16,7 @@ function ObjLoading:link(links) links:linkto(Utils) links:linkto(ObjCommon) links:linkto(ObjImage) + links:linkto(XMemCompress) links:linkto(ZoneCommon) links:linkto(minilzo) links:linkto(minizip) @@ -58,6 +59,7 @@ function ObjLoading:project() self:include(includes) Cryptography:include(includes) Utils:include(includes) + XMemCompress:include(includes) minilzo:include(includes) minizip:include(includes) zlib:include(includes) diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index 72a5ec21..324a6d83 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -206,8 +206,12 @@ bool IPakEntryReadStream::NextBlock() SwapBytesIfNecessary(m_current_block->countAndOffset.raw); for (auto& command : m_current_block->commands) { + if (!m_little_endian) + command.raw &= 0xFFFFFFDF; // ? idk, the game seems to do this? halp + + SwapBytesIfNecessary(command.raw); + auto size = command.size; - SwapBytesIfNecessary(size); command.size = size; } @@ -245,8 +249,20 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com } else if (compressed == 2) { - // This seems to use XMemDecompress - assert(false); + m_xmemdecompress_context.Reset(); + const auto maybeDecompressSize = m_xmemdecompress_context.Process( + &m_chunk_buffer[m_pos - m_buffer_start_pos], static_cast(commandSize), m_decompress_buffer, sizeof(m_decompress_buffer)); + + if (!maybeDecompressSize.has_value()) + { + con::error("Decompressing block with XMemDecompress failed!"); + return false; + } + + m_current_command_buffer = m_decompress_buffer; + m_current_command_length = *maybeDecompressSize; + m_current_command_offset = 0; + m_file_head += static_cast(*maybeDecompressSize); } else { diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h index 5e574be8..46261ceb 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h @@ -4,6 +4,7 @@ #include "ObjContainer/IPak/IPakTypes.h" #include "Utils/Endianness.h" #include "Utils/ObjStream.h" +#include "XMemDecompress.h" #include #include @@ -102,6 +103,7 @@ private: uint8_t* m_chunk_buffer; bool m_little_endian; + XMemDecompressContext m_xmemdecompress_context; std::istream& m_stream; IPakStreamManagerActions* m_stream_manager_actions; diff --git a/src/XMemCompress.lua b/src/XMemCompress.lua new file mode 100644 index 00000000..60720cf5 --- /dev/null +++ b/src/XMemCompress.lua @@ -0,0 +1,49 @@ +XMemCompress = {} + +function XMemCompress:include(includes) + if includes:handle(self:name()) then + includedirs { + path.join(ProjectFolder(), "XMemCompress") + } + end +end + +function XMemCompress:link(links) + links:add(self:name()) + links:linkto(Utils) + links:linkto(lzx) +end + +function XMemCompress:use() + +end + +function XMemCompress:name() + return "XMemCompress" +end + +function XMemCompress:project() + local folder = ProjectFolder() + local includes = Includes:create() + + project(self:name()) + targetdir(TargetDirectoryLib) + location "%{wks.location}/src/%{prj.name}" + kind "StaticLib" + language "C++" + + files { + path.join(folder, "XMemCompress/**.h"), + path.join(folder, "XMemCompress/**.cpp") + } + + vpaths { + ["*"] = { + path.join(folder, "XMemCompress") + } + } + + self:include(includes) + Utils:include(includes) + lzx:include(includes) +end diff --git a/src/XMemCompress/XMemDecompress.cpp b/src/XMemCompress/XMemDecompress.cpp new file mode 100644 index 00000000..4953c6e7 --- /dev/null +++ b/src/XMemCompress/XMemDecompress.cpp @@ -0,0 +1,141 @@ +#include "XMemDecompress.h" + +#include "Utils/Logging/Log.h" + +#include +#include + +namespace +{ + uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize) + { + const auto value = input[offset]; + offset++; + remainingSize--; + return value; + } + + uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte) + { + return static_cast(static_cast(static_cast(highByte) << 8u) | static_cast(lowByte)); + } + + void LogErrorHeaderSpace(size_t remainingInputSize) + { + con::error("XMemDecompress: Not enough data for header: {}", remainingInputSize); + } +} // namespace + +XMemDecompressContext::XMemDecompressContext() + : m_lzx_state(lzx_init(17)) +{ +} + +XMemDecompressContext::~XMemDecompressContext() +{ + if (m_lzx_state) + lzx_teardown(static_cast(m_lzx_state)); +} + +XMemDecompressContext::XMemDecompressContext(XMemDecompressContext&& other) noexcept + : m_lzx_state(other.m_lzx_state) +{ + other.m_lzx_state = nullptr; +} + +XMemDecompressContext& XMemDecompressContext::operator=(XMemDecompressContext&& other) noexcept +{ + m_lzx_state = other.m_lzx_state; + other.m_lzx_state = nullptr; + return *this; +} + +void XMemDecompressContext::Reset() const +{ + lzx_reset(static_cast(m_lzx_state)); +} + +std::optional XMemDecompressContext::Process(const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) const +{ + size_t curInputOffset = 0uz; + size_t curInputSize = inputLength; + + size_t curOutputOffset = 0uz; + size_t curOutputSize = outputBufferSize; + + uint8_t lowByte; + uint16_t dstSize, srcSize; + + while (curInputSize > 0) + { + uint8_t highByte = NextByte(input, curInputOffset, curInputSize); + + uint8_t suffixSize; + if (highByte == 0xFF) // magic number: output is smaller than 0x8000 + { + if (curInputSize < 4) + { + LogErrorHeaderSpace(curInputSize); + return std::nullopt; + } + + highByte = NextByte(input, curInputOffset, curInputSize); + lowByte = NextByte(input, curInputOffset, curInputSize); + dstSize = CombineHighLow(highByte, lowByte); + + highByte = NextByte(input, curInputOffset, curInputSize); + lowByte = NextByte(input, curInputOffset, curInputSize); + srcSize = CombineHighLow(highByte, lowByte); + + // The game seems to skip a 5 byte suffix after these blocks, not sure why. + suffixSize = 5u; + } + else + { + if (curInputSize < 1) + { + LogErrorHeaderSpace(curInputSize); + return std::nullopt; + } + + dstSize = 0x8000u; + lowByte = NextByte(input, curInputOffset, curInputSize); + srcSize = CombineHighLow(highByte, lowByte); + + suffixSize = 0u; + } + + if (srcSize == 0 || dstSize == 0) + { + // Other implementations do not handle this as a failure, game code suggests otherwise though + con::error("XMemDecompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize); + return std::nullopt; + } + + if (static_cast(srcSize) + suffixSize > curInputSize) + { + con::error("XMemDecompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize); + return std::nullopt; + } + + if (dstSize > curOutputSize) + { + con::error("XMemDecompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize); + return std::nullopt; + } + + auto ret = lzx_decompress(static_cast(m_lzx_state), &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize); + curInputOffset += srcSize + suffixSize; + curInputSize -= (srcSize + suffixSize); + curOutputOffset += dstSize; + curOutputSize -= srcSize; + + if (ret != DECR_OK) + { + con::error("XMemDecompress: lzx decompression failed: {}", ret); + return std::nullopt; + } + } + + return curOutputOffset; +} diff --git a/src/XMemCompress/XMemDecompress.h b/src/XMemCompress/XMemDecompress.h new file mode 100644 index 00000000..d350f615 --- /dev/null +++ b/src/XMemCompress/XMemDecompress.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +class XMemDecompressContext +{ +public: + XMemDecompressContext(); + ~XMemDecompressContext(); + XMemDecompressContext(const XMemDecompressContext& other) = delete; + XMemDecompressContext(XMemDecompressContext&& other) noexcept; + XMemDecompressContext& operator=(const XMemDecompressContext& other) = delete; + XMemDecompressContext& operator=(XMemDecompressContext&& other) noexcept; + + void Reset() const; + std::optional Process(const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) const; + +private: + void* m_lzx_state; +}; diff --git a/src/ZoneCommon.lua b/src/ZoneCommon.lua index 31cac7d4..2ea7e8ad 100644 --- a/src/ZoneCommon.lua +++ b/src/ZoneCommon.lua @@ -6,6 +6,7 @@ function ZoneCommon:include(includes) path.join(ProjectFolder(), "ZoneCommon") } Utils:include(includes) + XMemCompress:include(includes) Common:include(includes) ObjCommon:include(includes) Parser:include(includes) @@ -22,7 +23,7 @@ function ZoneCommon:link(links) links:linkto(ObjCommon) links:linkto(Parser) links:linkto(Utils) - links:linkto(lzx) + links:linkto(XMemCompress) ZoneCode:use() end @@ -58,7 +59,6 @@ function ZoneCommon:project() } self:include(includes) - lzx:include(includes) ZoneCode:include(includes) ZoneCode:use() diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp index 6778150e..1444f3d0 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp @@ -2,133 +2,25 @@ #include "Utils/Logging/Log.h" -#include -#include -#include -#include - -namespace -{ - uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize) - { - const auto value = input[offset]; - offset++; - remainingSize--; - return value; - } - - uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte) - { - return static_cast(static_cast(static_cast(highByte) << 8u) | static_cast(lowByte)); - } - - void LogErrorHeaderSpace(size_t remainingInputSize) - { - con::error("XMemCompress: Not enough data for header: {}", remainingInputSize); - } -} // namespace - XChunkProcessorLzxDecompress::XChunkProcessorLzxDecompress(const unsigned streamCount) - : m_lzx_states(streamCount) + : m_xmemdecompress_contexts(streamCount) { - // T6 uses 17 for window bits - for (auto& lzxState : m_lzx_states) - lzxState = lzx_init(17); -} - -XChunkProcessorLzxDecompress::~XChunkProcessorLzxDecompress() -{ - for (auto* lzxState : m_lzx_states) - lzx_teardown(static_cast(lzxState)); } size_t XChunkProcessorLzxDecompress::Process( const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) { - auto* state = static_cast(m_lzx_states[streamNumber]); + const auto& xMemDecompress = m_xmemdecompress_contexts[streamNumber]; // lzx state is reset before each chunk - lzx_reset(state); + xMemDecompress.Reset(); - size_t curInputOffset = 0uz; - size_t curInputSize = inputLength; - - size_t curOutputOffset = 0uz; - size_t curOutputSize = outputBufferSize; - - uint8_t lowByte; - uint16_t dstSize, srcSize; - - while (curInputSize > 0) + const auto maybeDecompressedSize = xMemDecompress.Process(input, inputLength, output, outputBufferSize); + if (!maybeDecompressedSize.has_value()) { - uint8_t highByte = NextByte(input, curInputOffset, curInputSize); - - uint8_t suffixSize; - if (highByte == 0xFF) // magic number: output is smaller than 0x8000 - { - if (curInputSize < 4) - { - LogErrorHeaderSpace(curInputSize); - return curOutputOffset; - } - - highByte = NextByte(input, curInputOffset, curInputSize); - lowByte = NextByte(input, curInputOffset, curInputSize); - dstSize = CombineHighLow(highByte, lowByte); - - highByte = NextByte(input, curInputOffset, curInputSize); - lowByte = NextByte(input, curInputOffset, curInputSize); - srcSize = CombineHighLow(highByte, lowByte); - - // The game seems to skip a 5 byte suffix after these blocks, not sure why. - suffixSize = 5u; - } - else - { - if (curInputSize < 1) - { - LogErrorHeaderSpace(curInputSize); - return curOutputOffset; - } - - dstSize = 0x8000u; - lowByte = NextByte(input, curInputOffset, curInputSize); - srcSize = CombineHighLow(highByte, lowByte); - - suffixSize = 0u; - } - - if (srcSize == 0 || dstSize == 0) - { - // Other implementations do not handle this as a failure, game code suggests otherwise though - con::error("XMemCompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize); - return curOutputOffset; - } - - if (static_cast(srcSize) + suffixSize > curInputSize) - { - con::error("XMemCompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize); - return curOutputOffset; - } - - if (dstSize > curOutputSize) - { - con::error("XMemCompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize); - return curOutputOffset; - } - - auto ret = lzx_decompress(state, &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize); - curInputOffset += srcSize + suffixSize; - curInputSize -= (srcSize + suffixSize); - curOutputOffset += dstSize; - curOutputSize -= srcSize; - - if (ret != DECR_OK) - { - con::error("XMemCompress: lzx decompression failed: {}", ret); - return curOutputOffset; - } + con::error("Failed to decompress xchunk with XMemDecompress"); + return 0; } - return curOutputOffset; + return *maybeDecompressedSize; } diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h index e32ee92a..1ba24728 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h @@ -1,6 +1,7 @@ #pragma once #include "IXChunkProcessor.h" +#include "XMemDecompress.h" #include @@ -8,9 +9,8 @@ class XChunkProcessorLzxDecompress final : public IXChunkProcessor { public: explicit XChunkProcessorLzxDecompress(unsigned streamCount); - ~XChunkProcessorLzxDecompress(); size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; private: - std::vector m_lzx_states; + std::vector m_xmemdecompress_contexts; };