feat: properly parse data from xenon ipaks

premake5.lua

@@ -136,6 +136,7 @@ include "src/RawTemplater.lua"
 include "src/UnlinkerCli.lua"
 include "src/Unlinking.lua"
 include "src/Utils.lua"
+include "src/XMemCompress.lua"
 include "src/ZoneCode.lua"
 include "src/ZoneCodeGeneratorLib.lua"
 include "src/ZoneCodeGenerator.lua"
@@ -156,6 +157,7 @@ group "Components"
     Cryptography:project()
     Parser:project()
     Utils:project()
+    XMemCompress:project()
     ZoneCode:project()
     ZoneCodeGeneratorLib:project()
     ZoneCommon:project()

src/ImageConverter/ImageConverter.cpp

@@ -200,7 +200,7 @@ namespace
 
             for (const auto& indexEntry : ipak->GetIndexEntries())
             {
-                const auto fileName = std::format("{:6x}_{:6x}.iwi", indexEntry.key.dataHash, indexEntry.key.nameHash);
+                const auto fileName = std::format("{:0>6x}_{:0>6x}.iwi", indexEntry.key.dataHash, indexEntry.key.nameHash);
                 std::ofstream outFile(outDir / fileName, std::ios::out | std::ios::binary);
                 if (!outFile.is_open())
                 {
@@ -216,18 +216,20 @@ namespace
                 }
 
                 char buffer[0x2000];
-                entryStream->read(buffer, 0x2000);
+                entryStream->read(buffer, sizeof(buffer));
                 auto readCount = entryStream->gcount();
                 while (readCount > 0)
                 {
                     outFile.write(buffer, readCount);
 
-                    entryStream->read(buffer, 0x2000);
+                    entryStream->read(buffer, sizeof(buffer));
                     readCount = entryStream->gcount();
                 }
 
                 entryStream->close();
                 outFile.close();
+
+                con::info("Dumped {}", fileName);
             }
 
             return true;

src/ObjCommon/ObjContainer/IPak/IPakTypes.h

@@ -74,10 +74,15 @@ union IPakDataBlockCountAndOffset
 
 static_assert(sizeof(IPakDataBlockCountAndOffset) == 4);
 
-struct IPakDataBlockCommand
+union IPakDataBlockCommand
 {
-    uint32_t size : 24;
+    struct
-    uint32_t compressed : 8;
+    {
+        uint32_t size : 24;
+        uint32_t compressed : 8;
+    };
+
+    uint32_t raw;
 };
 
 static_assert(sizeof(IPakDataBlockCommand) == 4);

src/ObjLoading.lua

@@ -16,6 +16,7 @@ function ObjLoading:link(links)
 	links:linkto(Utils)
 	links:linkto(ObjCommon)
 	links:linkto(ObjImage)
+	links:linkto(XMemCompress)
 	links:linkto(ZoneCommon)
 	links:linkto(minilzo)
 	links:linkto(minizip)
@@ -58,6 +59,7 @@ function ObjLoading:project()
 		self:include(includes)
 		Cryptography:include(includes)
 		Utils:include(includes)
+		XMemCompress:include(includes)
 		minilzo:include(includes)
 		minizip:include(includes)
 		zlib:include(includes)

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp

@@ -206,8 +206,12 @@ bool IPakEntryReadStream::NextBlock()
     SwapBytesIfNecessary(m_current_block->countAndOffset.raw);
     for (auto& command : m_current_block->commands)
     {
+        if (!m_little_endian)
+            command.raw &= 0xFFFFFFDF; // ? idk, the game seems to do this? halp
+
+        SwapBytesIfNecessary(command.raw);
+
         auto size = command.size;
-        SwapBytesIfNecessary(size);
         command.size = size;
     }
 
@@ -245,8 +249,20 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com
         }
         else if (compressed == 2)
         {
-            // This seems to use XMemDecompress
+            m_xmemdecompress_context.Reset();
-            assert(false);
+            const auto maybeDecompressSize = m_xmemdecompress_context.Process(
+                &m_chunk_buffer[m_pos - m_buffer_start_pos], static_cast<int>(commandSize), m_decompress_buffer, sizeof(m_decompress_buffer));
+
+            if (!maybeDecompressSize.has_value())
+            {
+                con::error("Decompressing block with XMemDecompress failed!");
+                return false;
+            }
+
+            m_current_command_buffer = m_decompress_buffer;
+            m_current_command_length = *maybeDecompressSize;
+            m_current_command_offset = 0;
+            m_file_head += static_cast<int64_t>(*maybeDecompressSize);
         }
         else
         {

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h

@@ -4,6 +4,7 @@
 #include "ObjContainer/IPak/IPakTypes.h"
 #include "Utils/Endianness.h"
 #include "Utils/ObjStream.h"
+#include "XMemDecompress.h"
 
 #include <concepts>
 #include <istream>
@@ -102,6 +103,7 @@ private:
     uint8_t* m_chunk_buffer;
 
     bool m_little_endian;
+    XMemDecompressContext m_xmemdecompress_context;
 
     std::istream& m_stream;
     IPakStreamManagerActions* m_stream_manager_actions;

src/XMemCompress.lua

@@ -0,0 +1,49 @@
+XMemCompress = {}
+
+function XMemCompress:include(includes)
+	if includes:handle(self:name()) then
+		includedirs {
+			path.join(ProjectFolder(), "XMemCompress")
+		}
+	end
+end
+
+function XMemCompress:link(links)
+	links:add(self:name())
+	links:linkto(Utils)
+	links:linkto(lzx)
+end
+
+function XMemCompress:use()
+	
+end
+
+function XMemCompress:name()
+	return "XMemCompress"
+end
+
+function XMemCompress:project()
+	local folder = ProjectFolder()
+	local includes = Includes:create()
+
+	project(self:name())
+        targetdir(TargetDirectoryLib)
+		location "%{wks.location}/src/%{prj.name}"
+		kind "StaticLib"
+		language "C++"
+		
+		files {
+			path.join(folder, "XMemCompress/**.h"), 
+			path.join(folder, "XMemCompress/**.cpp")
+		}
+		
+        vpaths {
+			["*"] = {
+				path.join(folder, "XMemCompress")
+			}
+		}
+		
+		self:include(includes)
+		Utils:include(includes)
+		lzx:include(includes)
+end

src/XMemCompress/XMemDecompress.cpp

@@ -0,0 +1,141 @@
+#include "XMemDecompress.h"
+
+#include "Utils/Logging/Log.h"
+
+#include <format>
+#include <lzx.h>
+
+namespace
+{
+    uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize)
+    {
+        const auto value = input[offset];
+        offset++;
+        remainingSize--;
+        return value;
+    }
+
+    uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte)
+    {
+        return static_cast<uint16_t>(static_cast<uint16_t>(static_cast<uint16_t>(highByte) << 8u) | static_cast<uint16_t>(lowByte));
+    }
+
+    void LogErrorHeaderSpace(size_t remainingInputSize)
+    {
+        con::error("XMemDecompress: Not enough data for header: {}", remainingInputSize);
+    }
+} // namespace
+
+XMemDecompressContext::XMemDecompressContext()
+    : m_lzx_state(lzx_init(17))
+{
+}
+
+XMemDecompressContext::~XMemDecompressContext()
+{
+    if (m_lzx_state)
+        lzx_teardown(static_cast<lzx_state*>(m_lzx_state));
+}
+
+XMemDecompressContext::XMemDecompressContext(XMemDecompressContext&& other) noexcept
+    : m_lzx_state(other.m_lzx_state)
+{
+    other.m_lzx_state = nullptr;
+}
+
+XMemDecompressContext& XMemDecompressContext::operator=(XMemDecompressContext&& other) noexcept
+{
+    m_lzx_state = other.m_lzx_state;
+    other.m_lzx_state = nullptr;
+    return *this;
+}
+
+void XMemDecompressContext::Reset() const
+{
+    lzx_reset(static_cast<lzx_state*>(m_lzx_state));
+}
+
+std::optional<size_t> XMemDecompressContext::Process(const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) const
+{
+    size_t curInputOffset = 0uz;
+    size_t curInputSize = inputLength;
+
+    size_t curOutputOffset = 0uz;
+    size_t curOutputSize = outputBufferSize;
+
+    uint8_t lowByte;
+    uint16_t dstSize, srcSize;
+
+    while (curInputSize > 0)
+    {
+        uint8_t highByte = NextByte(input, curInputOffset, curInputSize);
+
+        uint8_t suffixSize;
+        if (highByte == 0xFF) // magic number: output is smaller than 0x8000
+        {
+            if (curInputSize < 4)
+            {
+                LogErrorHeaderSpace(curInputSize);
+                return std::nullopt;
+            }
+
+            highByte = NextByte(input, curInputOffset, curInputSize);
+            lowByte = NextByte(input, curInputOffset, curInputSize);
+            dstSize = CombineHighLow(highByte, lowByte);
+
+            highByte = NextByte(input, curInputOffset, curInputSize);
+            lowByte = NextByte(input, curInputOffset, curInputSize);
+            srcSize = CombineHighLow(highByte, lowByte);
+
+            // The game seems to skip a 5 byte suffix after these blocks, not sure why.
+            suffixSize = 5u;
+        }
+        else
+        {
+            if (curInputSize < 1)
+            {
+                LogErrorHeaderSpace(curInputSize);
+                return std::nullopt;
+            }
+
+            dstSize = 0x8000u;
+            lowByte = NextByte(input, curInputOffset, curInputSize);
+            srcSize = CombineHighLow(highByte, lowByte);
+
+            suffixSize = 0u;
+        }
+
+        if (srcSize == 0 || dstSize == 0)
+        {
+            // Other implementations do not handle this as a failure, game code suggests otherwise though
+            con::error("XMemDecompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize);
+            return std::nullopt;
+        }
+
+        if (static_cast<size_t>(srcSize) + suffixSize > curInputSize)
+        {
+            con::error("XMemDecompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize);
+            return std::nullopt;
+        }
+
+        if (dstSize > curOutputSize)
+        {
+            con::error("XMemDecompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize);
+            return std::nullopt;
+        }
+
+        auto ret = lzx_decompress(static_cast<lzx_state*>(m_lzx_state), &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize);
+        curInputOffset += srcSize + suffixSize;
+        curInputSize -= (srcSize + suffixSize);
+        curOutputOffset += dstSize;
+        curOutputSize -= srcSize;
+
+        if (ret != DECR_OK)
+        {
+            con::error("XMemDecompress: lzx decompression failed: {}", ret);
+            return std::nullopt;
+        }
+    }
+
+    return curOutputOffset;
+}

src/XMemCompress/XMemDecompress.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <cstdint>
+#include <optional>
+
+class XMemDecompressContext
+{
+public:
+    XMemDecompressContext();
+    ~XMemDecompressContext();
+    XMemDecompressContext(const XMemDecompressContext& other) = delete;
+    XMemDecompressContext(XMemDecompressContext&& other) noexcept;
+    XMemDecompressContext& operator=(const XMemDecompressContext& other) = delete;
+    XMemDecompressContext& operator=(XMemDecompressContext&& other) noexcept;
+
+    void Reset() const;
+    std::optional<size_t> Process(const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) const;
+
+private:
+    void* m_lzx_state;
+};

src/ZoneCommon.lua

@@ -6,6 +6,7 @@ function ZoneCommon:include(includes)
 			path.join(ProjectFolder(), "ZoneCommon")
 		}
 		Utils:include(includes)
+		XMemCompress:include(includes)
 		Common:include(includes)
 		ObjCommon:include(includes)
 		Parser:include(includes)
@@ -22,7 +23,7 @@ function ZoneCommon:link(links)
 	links:linkto(ObjCommon)
 	links:linkto(Parser)
 	links:linkto(Utils)
-	links:linkto(lzx)
+	links:linkto(XMemCompress)
 	ZoneCode:use()
 end
 
@@ -58,7 +59,6 @@ function ZoneCommon:project()
 		}
 		
         self:include(includes)
-		lzx:include(includes)
 		ZoneCode:include(includes)
 
 		ZoneCode:use()

src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp

@@ -2,133 +2,25 @@
 
 #include "Utils/Logging/Log.h"
 
-#include <cstring>
-#include <format>
-#include <iostream>
-#include <lzx.h>
-
-namespace
-{
-    uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize)
-    {
-        const auto value = input[offset];
-        offset++;
-        remainingSize--;
-        return value;
-    }
-
-    uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte)
-    {
-        return static_cast<uint16_t>(static_cast<uint16_t>(static_cast<uint16_t>(highByte) << 8u) | static_cast<uint16_t>(lowByte));
-    }
-
-    void LogErrorHeaderSpace(size_t remainingInputSize)
-    {
-        con::error("XMemCompress: Not enough data for header: {}", remainingInputSize);
-    }
-} // namespace
-
 XChunkProcessorLzxDecompress::XChunkProcessorLzxDecompress(const unsigned streamCount)
-    : m_lzx_states(streamCount)
+    : m_xmemdecompress_contexts(streamCount)
 {
-    // T6 uses 17 for window bits
-    for (auto& lzxState : m_lzx_states)
-        lzxState = lzx_init(17);
-}
-
-XChunkProcessorLzxDecompress::~XChunkProcessorLzxDecompress()
-{
-    for (auto* lzxState : m_lzx_states)
-        lzx_teardown(static_cast<lzx_state*>(lzxState));
 }
 
 size_t XChunkProcessorLzxDecompress::Process(
     const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
 {
-    auto* state = static_cast<lzx_state*>(m_lzx_states[streamNumber]);
+    const auto& xMemDecompress = m_xmemdecompress_contexts[streamNumber];
 
     // lzx state is reset before each chunk
-    lzx_reset(state);
+    xMemDecompress.Reset();
 
-    size_t curInputOffset = 0uz;
+    const auto maybeDecompressedSize = xMemDecompress.Process(input, inputLength, output, outputBufferSize);
-    size_t curInputSize = inputLength;
+    if (!maybeDecompressedSize.has_value())
-
-    size_t curOutputOffset = 0uz;
-    size_t curOutputSize = outputBufferSize;
-
-    uint8_t lowByte;
-    uint16_t dstSize, srcSize;
-
-    while (curInputSize > 0)
     {
-        uint8_t highByte = NextByte(input, curInputOffset, curInputSize);
+        con::error("Failed to decompress xchunk with XMemDecompress");
-
+        return 0;
-        uint8_t suffixSize;
-        if (highByte == 0xFF) // magic number: output is smaller than 0x8000
-        {
-            if (curInputSize < 4)
-            {
-                LogErrorHeaderSpace(curInputSize);
-                return curOutputOffset;
-            }
-
-            highByte = NextByte(input, curInputOffset, curInputSize);
-            lowByte = NextByte(input, curInputOffset, curInputSize);
-            dstSize = CombineHighLow(highByte, lowByte);
-
-            highByte = NextByte(input, curInputOffset, curInputSize);
-            lowByte = NextByte(input, curInputOffset, curInputSize);
-            srcSize = CombineHighLow(highByte, lowByte);
-
-            // The game seems to skip a 5 byte suffix after these blocks, not sure why.
-            suffixSize = 5u;
-        }
-        else
-        {
-            if (curInputSize < 1)
-            {
-                LogErrorHeaderSpace(curInputSize);
-                return curOutputOffset;
-            }
-
-            dstSize = 0x8000u;
-            lowByte = NextByte(input, curInputOffset, curInputSize);
-            srcSize = CombineHighLow(highByte, lowByte);
-
-            suffixSize = 0u;
-        }
-
-        if (srcSize == 0 || dstSize == 0)
-        {
-            // Other implementations do not handle this as a failure, game code suggests otherwise though
-            con::error("XMemCompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize);
-            return curOutputOffset;
-        }
-
-        if (static_cast<size_t>(srcSize) + suffixSize > curInputSize)
-        {
-            con::error("XMemCompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize);
-            return curOutputOffset;
-        }
-
-        if (dstSize > curOutputSize)
-        {
-            con::error("XMemCompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize);
-            return curOutputOffset;
-        }
-
-        auto ret = lzx_decompress(state, &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize);
-        curInputOffset += srcSize + suffixSize;
-        curInputSize -= (srcSize + suffixSize);
-        curOutputOffset += dstSize;
-        curOutputSize -= srcSize;
-
-        if (ret != DECR_OK)
-        {
-            con::error("XMemCompress: lzx decompression failed: {}", ret);
-            return curOutputOffset;
-        }
     }
 
-    return curOutputOffset;
+    return *maybeDecompressedSize;
 }

src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "IXChunkProcessor.h"
+#include "XMemDecompress.h"
 
 #include <vector>
 
@@ -8,9 +9,8 @@ class XChunkProcessorLzxDecompress final : public IXChunkProcessor
 {
 public:
     explicit XChunkProcessorLzxDecompress(unsigned streamCount);
-    ~XChunkProcessorLzxDecompress();
     size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
 
 private:
-    std::vector<void*> m_lzx_states;
+    std::vector<XMemDecompressContext> m_xmemdecompress_contexts;
 };