From 802b0f244a4d689af467d896169f73b06f13621f Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Tue, 19 Aug 2025 13:22:39 +0200 Subject: [PATCH 1/4] chore: dump fastfile data when xenon t6 fastfile is detected --- src/Common/Game/IGame.h | 12 ++ src/ZoneCommon/Game/T6/ZoneConstantsT6.h | 28 ++-- .../Zone/XChunk/AbstractSalsa20Processor.cpp | 34 ++-- .../Zone/XChunk/AbstractSalsa20Processor.h | 51 +++--- src/ZoneCommon/Zone/XChunk/IXChunkProcessor.h | 8 +- .../Zone/XChunk/XChunkProcessorDeflate.cpp | 2 +- .../Zone/XChunk/XChunkProcessorDeflate.h | 2 +- .../Zone/XChunk/XChunkProcessorInflate.cpp | 9 +- .../Zone/XChunk/XChunkProcessorInflate.h | 2 +- .../XChunkProcessorSalsa20Decryption.cpp | 10 +- .../XChunk/XChunkProcessorSalsa20Decryption.h | 4 +- .../XChunkProcessorSalsa20Encryption.cpp | 8 +- .../XChunk/XChunkProcessorSalsa20Encryption.h | 4 +- .../Game/T6/ZoneLoaderFactoryT6.cpp | 152 ++++++++++-------- .../Loading/Processor/ProcessorXChunks.cpp | 21 ++- .../Loading/Processor/ProcessorXChunks.h | 6 +- .../Loading/Steps/StepDumpData.cpp | 14 +- src/ZoneLoading/Loading/Steps/StepDumpData.h | 3 +- .../Game/T6/ZoneWriterFactoryT6.cpp | 8 +- 19 files changed, 226 insertions(+), 152 deletions(-) diff --git a/src/Common/Game/IGame.h b/src/Common/Game/IGame.h index 449005f8..d9868724 100644 --- a/src/Common/Game/IGame.h +++ b/src/Common/Game/IGame.h @@ -16,6 +16,18 @@ enum class GameId COUNT }; +enum class GameEndianness +{ + LITTLE_ENDIAN, + BIG_ENDIAN +}; + +enum class GameWordSize +{ + ARCH_32, + ARCH_64 +}; + static constexpr const char* GameId_Names[]{ "IW3", "IW4", diff --git a/src/ZoneCommon/Game/T6/ZoneConstantsT6.h b/src/ZoneCommon/Game/T6/ZoneConstantsT6.h index 10213294..785c3742 100644 --- a/src/ZoneCommon/Game/T6/ZoneConstantsT6.h +++ b/src/ZoneCommon/Game/T6/ZoneConstantsT6.h @@ -12,21 +12,25 @@ namespace T6 ZoneConstants() = default; public: - static constexpr const char* MAGIC_SIGNED_TREYARCH = "TAff0100"; - static constexpr const char* MAGIC_SIGNED_OAT = "ABff0100"; + static constexpr const char* MAGIC_SIGNED_PC_TREYARCH = "TAff0100"; + static constexpr const char* MAGIC_SIGNED_XENON_TREYARCH = "TAffx100"; + static constexpr const char* MAGIC_SIGNED_PC_OAT = "ABff0100"; static constexpr const char* MAGIC_UNSIGNED = "TAffu100"; static constexpr const char* MAGIC_UNSIGNED_SERVER = "TAsvu100"; - static_assert(std::char_traits::length(MAGIC_SIGNED_TREYARCH) == sizeof(ZoneHeader::m_magic)); - static_assert(std::char_traits::length(MAGIC_SIGNED_OAT) == sizeof(ZoneHeader::m_magic)); + static_assert(std::char_traits::length(MAGIC_SIGNED_PC_TREYARCH) == sizeof(ZoneHeader::m_magic)); + static_assert(std::char_traits::length(MAGIC_SIGNED_XENON_TREYARCH) == sizeof(ZoneHeader::m_magic)); + static_assert(std::char_traits::length(MAGIC_SIGNED_PC_OAT) == sizeof(ZoneHeader::m_magic)); static_assert(std::char_traits::length(MAGIC_UNSIGNED) == sizeof(ZoneHeader::m_magic)); static_assert(std::char_traits::length(MAGIC_UNSIGNED_SERVER) == sizeof(ZoneHeader::m_magic)); - static constexpr int ZONE_VERSION = 147; - static constexpr int STREAM_COUNT = 4; - static constexpr int XCHUNK_SIZE = 0x8000; - static constexpr int XCHUNK_MAX_WRITE_SIZE = XCHUNK_SIZE - 0x40; - static constexpr int VANILLA_BUFFER_SIZE = 0x80000; + static constexpr unsigned ZONE_VERSION_PC = 147; + static constexpr unsigned ZONE_VERSION_XENON = 146; + + static constexpr unsigned STREAM_COUNT = 4; + static constexpr unsigned XCHUNK_SIZE = 0x8000; + static constexpr unsigned XCHUNK_MAX_WRITE_SIZE = XCHUNK_SIZE - 0x40; + static constexpr unsigned VANILLA_BUFFER_SIZE = 0x80000; static constexpr unsigned OFFSET_BLOCK_BIT_COUNT = 3u; static constexpr block_t INSERT_BLOCK = XFILE_BLOCK_VIRTUAL; @@ -34,10 +38,14 @@ namespace T6 static constexpr size_t FILE_SUFFIX_ZERO_ALIGN = 0x40; static constexpr const char* MAGIC_AUTH_HEADER = "PHEEBs71"; - inline static const uint8_t SALSA20_KEY_TREYARCH[]{ + inline static const uint8_t SALSA20_KEY_TREYARCH_PC[]{ 0x64, 0x1D, 0x8A, 0x2F, 0xE3, 0x1D, 0x3A, 0xA6, 0x36, 0x22, 0xBB, 0xC9, 0xCE, 0x85, 0x87, 0x22, 0x9D, 0x42, 0xB0, 0xF8, 0xED, 0x9B, 0x92, 0x41, 0x30, 0xBF, 0x88, 0xB6, 0x5E, 0xDC, 0x50, 0xBE, }; + inline static const uint8_t SALSA20_KEY_TREYARCH_XENON[]{ + 0x0E, 0x50, 0xF4, 0x9F, 0x41, 0x23, 0x17, 0x09, 0x60, 0x38, 0x66, 0x56, 0x22, 0xDD, 0x09, 0x13, + 0x32, 0xA2, 0x09, 0xBA, 0x0A, 0x05, 0xA0, 0x0E, 0x13, 0x77, 0xCE, 0xDB, 0x0A, 0x3C, 0xB1, 0xD3, + }; inline static const uint8_t RSA_PUBLIC_KEY_TREYARCH[]{ 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, 0x00, 0xc7, 0x9d, 0x33, 0xe0, 0x75, 0xaf, 0xef, 0x08, 0x08, 0x2b, 0x89, 0xd9, 0x3b, 0xf3, diff --git a/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.cpp b/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.cpp index 377a74be..63018e4c 100644 --- a/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.cpp +++ b/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.cpp @@ -2,24 +2,16 @@ #include -AbstractSalsa20Processor::AbstractSalsa20Processor(const int streamCount, const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize) +AbstractSalsa20Processor::AbstractSalsa20Processor(const unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, const unsigned keySize) : m_stream_count(streamCount), - m_stream_contexts(std::make_unique(streamCount)), - m_stream_block_indices(std::make_unique(streamCount)) + m_stream_contexts(streamCount), + m_block_hashes(BLOCK_HASHES_COUNT * streamCount * SHA1_HASH_SIZE), + m_stream_block_indices(streamCount) { - m_block_hashes = std::make_unique(BLOCK_HASHES_COUNT * streamCount * SHA1_HASH_SIZE); InitStreams(zoneName, salsa20Key, keySize); } -uint8_t* AbstractSalsa20Processor::GetHashBlock(const int streamNumber) const -{ - const auto blockIndexOffset = m_stream_block_indices[streamNumber] * m_stream_count * SHA1_HASH_SIZE; - const auto streamOffset = static_cast(streamNumber) * SHA1_HASH_SIZE; - - return &m_block_hashes[blockIndexOffset + streamOffset]; -} - -void AbstractSalsa20Processor::InitStreams(const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize) const +void AbstractSalsa20Processor::InitStreams(const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize) { // Original buffer must have been 32 bytes because the zoneName can at most be 31 characters be long before being cut off const auto zoneNameLength = std::min(zoneName.length(), 31uz); @@ -29,14 +21,14 @@ void AbstractSalsa20Processor::InitStreams(const std::string& zoneName, const ui assert(blockHashBufferSize % 4 == 0); size_t zoneNameOffset = 0; - for (size_t i = 0; i < blockHashBufferSize; i += 4) + for (auto i = 0uz; i < blockHashBufferSize; i += 4) { - *reinterpret_cast(&m_block_hashes[i]) = 0x1010101 * zoneName[zoneNameOffset++]; + memset(&m_block_hashes[i], zoneName[zoneNameOffset++], 4u); zoneNameOffset %= zoneNameLength; } - for (auto stream = 0; stream < m_stream_count; stream++) + for (auto stream = 0u; stream < m_stream_count; stream++) { m_stream_block_indices[stream] = 0; @@ -45,11 +37,19 @@ void AbstractSalsa20Processor::InitStreams(const std::string& zoneName, const ui } } +uint8_t* AbstractSalsa20Processor::GetHashBlock(const unsigned streamNumber) +{ + const auto blockIndexOffset = m_stream_block_indices[streamNumber] * m_stream_count * SHA1_HASH_SIZE; + const auto streamOffset = static_cast(streamNumber) * SHA1_HASH_SIZE; + + return &m_block_hashes[blockIndexOffset + streamOffset]; +} + void AbstractSalsa20Processor::GetCapturedData(const uint8_t** pCapturedData, size_t* pSize) { assert(pCapturedData != nullptr); assert(pSize != nullptr); - *pCapturedData = m_block_hashes.get(); + *pCapturedData = m_block_hashes.data(); *pSize = BLOCK_HASHES_COUNT * m_stream_count * SHA1_HASH_SIZE; } diff --git a/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.h b/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.h index 01d56a63..b5045c54 100644 --- a/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.h +++ b/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.h @@ -1,40 +1,22 @@ #pragma once #include "Cryptography.h" -#include "Utils/ClassUtils.h" #include "Utils/ICapturedDataProvider.h" #include #include #include +#include + +class Salsa20StreamContext +{ +public: + std::unique_ptr m_salsa20; + std::unique_ptr m_sha1; +}; class AbstractSalsa20Processor : public ICapturedDataProvider { -protected: - static constexpr int BLOCK_HASHES_COUNT = 200; - static constexpr int SHA1_HASH_SIZE = 20; - static constexpr int SALSA20_IV_SIZE = 8; - - class StreamContext - { - public: - std::unique_ptr m_salsa20; - std::unique_ptr m_sha1; - }; - - int m_stream_count; - std::unique_ptr m_stream_contexts; - - // m_block_hashes[BLOCK_HASHES_COUNT][numStreams][HASH_SIZE] - std::unique_ptr m_block_hashes; - std::unique_ptr m_stream_block_indices; - - AbstractSalsa20Processor(int streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); - - _NODISCARD uint8_t* GetHashBlock(int streamNumber) const; - - void InitStreams(const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize) const; - public: virtual ~AbstractSalsa20Processor() = default; AbstractSalsa20Processor(const AbstractSalsa20Processor& other) = delete; @@ -43,4 +25,21 @@ public: AbstractSalsa20Processor& operator=(AbstractSalsa20Processor&& other) noexcept = default; void GetCapturedData(const uint8_t** pCapturedData, size_t* pSize) override; + +protected: + static constexpr auto BLOCK_HASHES_COUNT = 200u; + static constexpr auto SHA1_HASH_SIZE = 20u; + static constexpr auto SALSA20_IV_SIZE = 8u; + + AbstractSalsa20Processor(unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, unsigned keySize); + void InitStreams(const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); + + [[nodiscard]] uint8_t* GetHashBlock(unsigned streamNumber); + + unsigned m_stream_count; + std::vector m_stream_contexts; + + // m_block_hashes[BLOCK_HASHES_COUNT][numStreams][SHA1_HASH_SIZE] + std::vector m_block_hashes; + std::vector m_stream_block_indices; }; diff --git a/src/ZoneCommon/Zone/XChunk/IXChunkProcessor.h b/src/ZoneCommon/Zone/XChunk/IXChunkProcessor.h index 22e444b1..bcc744ff 100644 --- a/src/ZoneCommon/Zone/XChunk/IXChunkProcessor.h +++ b/src/ZoneCommon/Zone/XChunk/IXChunkProcessor.h @@ -6,6 +6,12 @@ class IXChunkProcessor { public: + IXChunkProcessor() = default; virtual ~IXChunkProcessor() = default; - virtual size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) = 0; + IXChunkProcessor(const IXChunkProcessor& other) = default; + IXChunkProcessor(IXChunkProcessor&& other) noexcept = default; + IXChunkProcessor& operator=(const IXChunkProcessor& other) = default; + IXChunkProcessor& operator=(IXChunkProcessor&& other) noexcept = default; + + virtual size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) = 0; }; diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp index 5cff8933..ed546ab3 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp @@ -6,7 +6,7 @@ #include #include -size_t XChunkProcessorDeflate::Process(int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) +size_t XChunkProcessorDeflate::Process(unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) { z_stream stream{}; stream.zalloc = Z_NULL; diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.h b/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.h index 7c158c86..9dda1869 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.h +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.h @@ -4,5 +4,5 @@ class XChunkProcessorDeflate final : public IXChunkProcessor { public: - size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; + size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; }; diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.cpp index bc8420ff..a9fdec9d 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.cpp +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.cpp @@ -2,10 +2,12 @@ #include "XChunkException.h" +#include +#include #include #include -size_t XChunkProcessorInflate::Process(int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) +size_t XChunkProcessorInflate::Process(unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) { z_stream stream{}; stream.zalloc = Z_NULL; @@ -23,7 +25,10 @@ size_t XChunkProcessorInflate::Process(int streamNumber, const uint8_t* input, c ret = inflate(&stream, Z_FULL_FLUSH); if (ret != Z_STREAM_END) - throw XChunkException("Zone has invalid or unsupported compression. Inflate failed"); + { + std::cerr << std::format("inflate of stream failed with error code {}: {}\n", streamNumber, ret, stream.msg); + throw XChunkException(std::format("Zone has invalid or unsupported compression: {}", stream.msg)); + } const size_t outputSize = stream.total_out; diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.h b/src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.h index c328219f..c6e24a1c 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.h +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.h @@ -4,5 +4,5 @@ class XChunkProcessorInflate final : public IXChunkProcessor { public: - size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; + size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; }; diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp index cf1bfb05..72de9270 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp @@ -4,9 +4,11 @@ #include "Cryptography.h" #include +#include +#include -XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const int streamCount, - std::string& zoneName, +XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const unsigned streamCount, + const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize) : AbstractSalsa20Processor(streamCount, zoneName, salsa20Key, keySize) @@ -14,9 +16,9 @@ XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const int str } size_t XChunkProcessorSalsa20Decryption::Process( - const int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) + const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) { - assert(streamNumber >= 0 && streamNumber < m_stream_count); + assert(streamNumber < m_stream_count); assert(input != nullptr); assert(output != nullptr); assert(inputLength <= outputBufferSize); diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.h b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.h index c5c57d51..fa32e9b0 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.h +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.h @@ -7,7 +7,7 @@ class XChunkProcessorSalsa20Decryption final : public IXChunkProcessor, public AbstractSalsa20Processor { public: - XChunkProcessorSalsa20Decryption(int streamCount, std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); + XChunkProcessorSalsa20Decryption(unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); - size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; + size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; }; diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.cpp index 80ca8928..43899e18 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.cpp +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.cpp @@ -2,7 +2,7 @@ #include -XChunkProcessorSalsa20Encryption::XChunkProcessorSalsa20Encryption(const int streamCount, +XChunkProcessorSalsa20Encryption::XChunkProcessorSalsa20Encryption(const unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize) @@ -11,14 +11,14 @@ XChunkProcessorSalsa20Encryption::XChunkProcessorSalsa20Encryption(const int str } size_t XChunkProcessorSalsa20Encryption::Process( - const int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) + const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) { - assert(streamNumber >= 0 && streamNumber < m_stream_count); + assert(streamNumber < m_stream_count); assert(input != nullptr); assert(output != nullptr); assert(inputLength <= outputBufferSize); - auto& streamContext = m_stream_contexts[streamNumber]; + const auto& streamContext = m_stream_contexts[streamNumber]; // Hash not yet encrypted XChunk uint8_t blockSha1Hash[SHA1_HASH_SIZE]; diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.h b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.h index 89272df3..12249613 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.h +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.h @@ -8,7 +8,7 @@ class XChunkProcessorSalsa20Encryption final : public IXChunkProcessor, public AbstractSalsa20Processor { public: - XChunkProcessorSalsa20Encryption(int streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); + XChunkProcessorSalsa20Encryption(unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); - size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; + size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; }; diff --git a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp index f014f4f6..7c0a9718 100644 --- a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp +++ b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp @@ -9,6 +9,7 @@ #include "Loading/Processor/ProcessorXChunks.h" #include "Loading/Steps/StepAddProcessor.h" #include "Loading/Steps/StepAllocXBlocks.h" +#include "Loading/Steps/StepDumpData.h" #include "Loading/Steps/StepLoadSignature.h" #include "Loading/Steps/StepLoadZoneContent.h" #include "Loading/Steps/StepLoadZoneSizes.h" @@ -16,16 +17,20 @@ #include "Loading/Steps/StepVerifyFileName.h" #include "Loading/Steps/StepVerifyMagic.h" #include "Loading/Steps/StepVerifySignature.h" -#include "Utils/ClassUtils.h" +#include "Utils/Endianness.h" #include "Zone/XChunk/XChunkProcessorInflate.h" #include "Zone/XChunk/XChunkProcessorSalsa20Decryption.h" #include +#include #include +#include +#include #include #include using namespace T6; +namespace fs = std::filesystem; namespace { @@ -44,46 +49,53 @@ namespace return GameLanguage::LANGUAGE_NONE; } - bool CanLoad(const ZoneHeader& header, bool* isSecure, bool* isOfficial, bool* isEncrypted) + bool CanLoad(const ZoneHeader& header, bool& isBigEndian, bool& isSecure, bool& isOfficial, bool& isEncrypted) { - assert(isSecure != nullptr); - assert(isOfficial != nullptr); - - if (header.m_version != ZoneConstants::ZONE_VERSION) + if (endianness::FromLittleEndian(header.m_version) == ZoneConstants::ZONE_VERSION_PC) { - return false; + isBigEndian = false; + if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_PC_TREYARCH, 8)) + { + isSecure = true; + isOfficial = true; + isEncrypted = true; + return true; + } + + if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_PC_OAT, 8)) + { + isSecure = true; + isOfficial = false; + isEncrypted = true; + return true; + } + + if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED, 8)) + { + isSecure = false; + isOfficial = true; + isEncrypted = true; + return true; + } + + if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED_SERVER, 8)) + { + isSecure = false; + isOfficial = true; + isEncrypted = false; + return true; + } } - - if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_TREYARCH, 8)) + else if (endianness::FromBigEndian(header.m_version) == ZoneConstants::ZONE_VERSION_XENON) { - *isSecure = true; - *isOfficial = true; - *isEncrypted = true; - return true; - } - - if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_OAT, 8)) - { - *isSecure = true; - *isOfficial = false; - *isEncrypted = true; - return true; - } - - if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED, 8)) - { - *isSecure = false; - *isOfficial = true; - *isEncrypted = true; - return true; - } - - if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED_SERVER, 8)) - { - *isSecure = false; - *isOfficial = true; - *isEncrypted = false; - return true; + isBigEndian = true; + if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_XENON_TREYARCH, 8)) + { + isSecure = true; + isOfficial = true; + isEncrypted = true; + return true; + } } return false; @@ -145,16 +157,21 @@ namespace return signatureLoadStepPtr; } - ICapturedDataProvider* AddXChunkProcessor(const bool isEncrypted, ZoneLoader& zoneLoader, std::string& fileName) + ICapturedDataProvider* AddXChunkProcessor(const bool isBigEndian, const bool isEncrypted, ZoneLoader& zoneLoader, std::string& fileName) { ICapturedDataProvider* result = nullptr; - auto xChunkProcessor = processor::CreateProcessorXChunks(ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, ZoneConstants::VANILLA_BUFFER_SIZE); + auto xChunkProcessor = processor::CreateProcessorXChunks(ZoneConstants::STREAM_COUNT, + ZoneConstants::XCHUNK_SIZE, + isBigEndian ? GameEndianness::BIG_ENDIAN : GameEndianness::LITTLE_ENDIAN, + ZoneConstants::VANILLA_BUFFER_SIZE); + + const uint8_t (&salsa20Key)[32] = isBigEndian ? ZoneConstants::SALSA20_KEY_TREYARCH_XENON : ZoneConstants::SALSA20_KEY_TREYARCH_PC; if (isEncrypted) { // If zone is encrypted, the decryption is applied before the decompression. T6 Zones always use Salsa20. - auto chunkProcessorSalsa20 = std::make_unique( - ZoneConstants::STREAM_COUNT, fileName, ZoneConstants::SALSA20_KEY_TREYARCH, sizeof(ZoneConstants::SALSA20_KEY_TREYARCH)); + auto chunkProcessorSalsa20 = + std::make_unique(ZoneConstants::STREAM_COUNT, fileName, salsa20Key, sizeof(salsa20Key)); result = chunkProcessorSalsa20.get(); xChunkProcessor->AddChunkProcessor(std::move(chunkProcessorSalsa20)); } @@ -170,12 +187,10 @@ namespace std::unique_ptr ZoneLoaderFactory::CreateLoaderForHeader(ZoneHeader& header, std::string& fileName) const { - bool isSecure; - bool isOfficial; - bool isEncrypted; + bool isBigEndian, isSecure, isOfficial, isEncrypted; // Check if this file is a supported T6 zone. - if (!CanLoad(header, &isSecure, &isOfficial, &isEncrypted)) + if (!CanLoad(header, isBigEndian, isSecure, isOfficial, isEncrypted)) return nullptr; // Create new zone @@ -196,26 +211,37 @@ std::unique_ptr ZoneLoaderFactory::CreateLoaderForHeader(ZoneHeader& ISignatureProvider* signatureProvider = AddAuthHeaderSteps(isSecure, *zoneLoader, fileName); // Setup loading XChunks from the zone from this point on. - ICapturedDataProvider* signatureDataProvider = AddXChunkProcessor(isEncrypted, *zoneLoader, fileName); + ICapturedDataProvider* signatureDataProvider = AddXChunkProcessor(isBigEndian, isEncrypted, *zoneLoader, fileName); - // Start of the XFile struct - zoneLoader->AddLoadingStep(step::CreateStepLoadZoneSizes()); - zoneLoader->AddLoadingStep(step::CreateStepAllocXBlocks()); - - // Start of the zone content - zoneLoader->AddLoadingStep(step::CreateStepLoadZoneContent( - [zonePtr](ZoneInputStream& stream) - { - return std::make_unique(*zonePtr, stream); - }, - 32u, - ZoneConstants::OFFSET_BLOCK_BIT_COUNT, - ZoneConstants::INSERT_BLOCK, - zonePtr->Memory())); - - if (isSecure) + if (!isBigEndian) { - zoneLoader->AddLoadingStep(step::CreateStepVerifySignature(std::move(rsa), signatureProvider, signatureDataProvider)); + // Start of the XFile struct + zoneLoader->AddLoadingStep(step::CreateStepLoadZoneSizes()); + zoneLoader->AddLoadingStep(step::CreateStepAllocXBlocks()); + + // Start of the zone content + zoneLoader->AddLoadingStep(step::CreateStepLoadZoneContent( + [zonePtr](ZoneInputStream& stream) + { + return std::make_unique(*zonePtr, stream); + }, + 32u, + ZoneConstants::OFFSET_BLOCK_BIT_COUNT, + ZoneConstants::INSERT_BLOCK, + zonePtr->Memory())); + + if (isSecure) + { + zoneLoader->AddLoadingStep(step::CreateStepVerifySignature(std::move(rsa), signatureProvider, signatureDataProvider)); + } + } + else + { + fs::path dumpFileNamePath = fs::path(fileName).filename(); + dumpFileNamePath.replace_extension(".dat"); + std::string dumpFileName = dumpFileNamePath.string(); + std::cerr << std::format("Dumping xbox assets is not supported, making a full fastfile data dump to {}\n", dumpFileName); + zoneLoader->AddLoadingStep(step::CreateStepDumpData(dumpFileName, 0xFFFFFFFF)); } return zoneLoader; diff --git a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp index 6d73eafc..cb57c662 100644 --- a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp +++ b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp @@ -1,11 +1,14 @@ #include "ProcessorXChunks.h" #include "Loading/Exception/InvalidChunkSizeException.h" +#include "Utils/Endianness.h" #include "Zone/ZoneTypes.h" #include #include #include +#include +#include #include #include #include @@ -125,8 +128,9 @@ namespace class ProcessorXChunks final : public processor::IProcessorXChunks { public: - ProcessorXChunks(const int numStreams, const size_t xChunkSize, const std::optional vanillaBufferSize) + ProcessorXChunks(const int numStreams, const size_t xChunkSize, const GameEndianness endianness, const std::optional vanillaBufferSize) : m_chunk_size(xChunkSize), + m_endianness(endianness), m_vanilla_buffer_size(vanillaBufferSize), m_initialized_streams(false), m_current_stream(0), @@ -227,6 +231,11 @@ namespace return; } + if (m_endianness == GameEndianness::LITTLE_ENDIAN) + chunkSize = endianness::FromLittleEndian(chunkSize); + else + chunkSize = endianness::FromBigEndian(chunkSize); + if (chunkSize > m_chunk_size) { throw InvalidChunkSizeException(chunkSize, m_chunk_size); @@ -280,6 +289,7 @@ namespace std::vector> m_streams; size_t m_chunk_size; + GameEndianness m_endianness; std::optional m_vanilla_buffer_size; std::vector> m_chunk_processors; @@ -297,13 +307,14 @@ namespace namespace processor { - std::unique_ptr CreateProcessorXChunks(int numStreams, const size_t xChunkSize) + std::unique_ptr CreateProcessorXChunks(unsigned numStreams, const size_t xChunkSize, const GameEndianness endianness) { - return std::make_unique(numStreams, xChunkSize, std::nullopt); + return std::make_unique(numStreams, xChunkSize, endianness, std::nullopt); } - std::unique_ptr CreateProcessorXChunks(int numStreams, const size_t xChunkSize, const size_t vanillaBufferSize) + std::unique_ptr + CreateProcessorXChunks(unsigned numStreams, const size_t xChunkSize, GameEndianness endianness, const size_t vanillaBufferSize) { - return std::make_unique(numStreams, xChunkSize, vanillaBufferSize); + return std::make_unique(numStreams, xChunkSize, endianness, vanillaBufferSize); } } // namespace processor diff --git a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.h b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.h index dec161af..6a876231 100644 --- a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.h +++ b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.h @@ -1,4 +1,6 @@ #pragma once + +#include "Game/IGame.h" #include "Loading/StreamProcessor.h" #include "Zone/XChunk/IXChunkProcessor.h" @@ -12,6 +14,6 @@ namespace processor virtual void AddChunkProcessor(std::unique_ptr chunkProcessor) = 0; }; - std::unique_ptr CreateProcessorXChunks(int numStreams, size_t xChunkSize); - std::unique_ptr CreateProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize); + std::unique_ptr CreateProcessorXChunks(unsigned numStreams, size_t xChunkSize, GameEndianness endianness); + std::unique_ptr CreateProcessorXChunks(unsigned numStreams, size_t xChunkSize, GameEndianness endianness, size_t vanillaBufferSize); } // namespace processor diff --git a/src/ZoneLoading/Loading/Steps/StepDumpData.cpp b/src/ZoneLoading/Loading/Steps/StepDumpData.cpp index 3999dc4d..906fce4b 100644 --- a/src/ZoneLoading/Loading/Steps/StepDumpData.cpp +++ b/src/ZoneLoading/Loading/Steps/StepDumpData.cpp @@ -7,17 +7,18 @@ namespace class StepDumpData final : public ILoadingStep { public: - explicit StepDumpData(const size_t dumpCount) - : m_dump_count(dumpCount) + StepDumpData(std::string fileName, const size_t dumpCount) + : m_file_name(std::move(fileName)), + m_dump_count(dumpCount) { } void PerformStep(ZoneLoader& zoneLoader, ILoadingStream& stream) override { - uint8_t tempBuffer[128]; + uint8_t tempBuffer[0x1000]; auto dumpedBytes = 0uz; - std::ofstream tempFile("dump.dat", std::fstream::out | std::fstream::binary); + std::ofstream tempFile(m_file_name, std::fstream::out | std::fstream::binary); while (dumpedBytes < m_dump_count) { @@ -45,14 +46,15 @@ namespace } private: + std::string m_file_name; size_t m_dump_count; }; } // namespace namespace step { - std::unique_ptr CreateStepDumpData(size_t dumpCount) + std::unique_ptr CreateStepDumpData(std::string fileName, size_t dumpCount) { - return std::make_unique(dumpCount); + return std::make_unique(std::move(fileName), dumpCount); } } // namespace step diff --git a/src/ZoneLoading/Loading/Steps/StepDumpData.h b/src/ZoneLoading/Loading/Steps/StepDumpData.h index 07beac1a..a174a3a3 100644 --- a/src/ZoneLoading/Loading/Steps/StepDumpData.h +++ b/src/ZoneLoading/Loading/Steps/StepDumpData.h @@ -3,8 +3,9 @@ #include "Loading/ILoadingStep.h" #include +#include namespace step { - std::unique_ptr CreateStepDumpData(size_t dumpCount); + std::unique_ptr CreateStepDumpData(std::string fileName, size_t dumpCount); } diff --git a/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp b/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp index 3d0d1d70..5edf9dcf 100644 --- a/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp +++ b/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp @@ -44,14 +44,14 @@ namespace ZoneHeader CreateHeaderForParams(const bool isSecure, const bool isOfficial, const bool isEncrypted) { ZoneHeader header{}; - header.m_version = ZoneConstants::ZONE_VERSION; + header.m_version = ZoneConstants::ZONE_VERSION_PC; if (isSecure) { if (isOfficial) - memcpy(header.m_magic, ZoneConstants::MAGIC_SIGNED_TREYARCH, sizeof(ZoneHeader::m_magic)); + memcpy(header.m_magic, ZoneConstants::MAGIC_SIGNED_PC_TREYARCH, sizeof(ZoneHeader::m_magic)); else - memcpy(header.m_magic, ZoneConstants::MAGIC_SIGNED_OAT, sizeof(ZoneHeader::m_magic)); + memcpy(header.m_magic, ZoneConstants::MAGIC_SIGNED_PC_OAT, sizeof(ZoneHeader::m_magic)); } else { @@ -82,7 +82,7 @@ namespace { // If zone is encrypted, the decryption is applied before the decompression. T6 Zones always use Salsa20. auto chunkProcessorSalsa20 = std::make_unique( - ZoneConstants::STREAM_COUNT, zone.m_name, ZoneConstants::SALSA20_KEY_TREYARCH, sizeof(ZoneConstants::SALSA20_KEY_TREYARCH)); + ZoneConstants::STREAM_COUNT, zone.m_name, ZoneConstants::SALSA20_KEY_TREYARCH_PC, sizeof(ZoneConstants::SALSA20_KEY_TREYARCH_PC)); // If there is encryption, the signed data of the zone is the final hash blocks provided by the Salsa20 IV adaption algorithm if (dataToSignProviderPtr) From 763d28080549d07c82df84655a82404deaca55dd Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Wed, 20 Aug 2025 22:13:01 +0100 Subject: [PATCH 2/4] chore: recognize when xenon fastfile use lzx compression --- src/Common/Game/IGame.h | 8 +++- src/ZoneCommon/Game/T6/ZoneConstantsT6.h | 12 ++--- .../Zone/XChunk/AbstractSalsa20Processor.cpp | 3 +- .../Zone/XChunk/AbstractSalsa20Processor.h | 2 +- .../XChunk/XChunkProcessorLzxDecompress.cpp | 12 +++++ .../XChunk/XChunkProcessorLzxDecompress.h | 8 ++++ .../Game/T6/ZoneLoaderFactoryT6.cpp | 47 +++++++++++++------ .../Loading/Processor/ProcessorXChunks.cpp | 2 +- .../Game/T6/ZoneWriterFactoryT6.cpp | 4 +- 9 files changed, 71 insertions(+), 27 deletions(-) create mode 100644 src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp create mode 100644 src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h diff --git a/src/Common/Game/IGame.h b/src/Common/Game/IGame.h index d9868724..d8021ee2 100644 --- a/src/Common/Game/IGame.h +++ b/src/Common/Game/IGame.h @@ -16,10 +16,14 @@ enum class GameId COUNT }; +// The full uppercase names are macros in the standard lib +// So unfortunately not usable as values in the enum enum class GameEndianness { - LITTLE_ENDIAN, - BIG_ENDIAN + /* Little endian */ + LE, + /* Big endian */ + BE }; enum class GameWordSize diff --git a/src/ZoneCommon/Game/T6/ZoneConstantsT6.h b/src/ZoneCommon/Game/T6/ZoneConstantsT6.h index 785c3742..37ea4224 100644 --- a/src/ZoneCommon/Game/T6/ZoneConstantsT6.h +++ b/src/ZoneCommon/Game/T6/ZoneConstantsT6.h @@ -12,15 +12,15 @@ namespace T6 ZoneConstants() = default; public: - static constexpr const char* MAGIC_SIGNED_PC_TREYARCH = "TAff0100"; - static constexpr const char* MAGIC_SIGNED_XENON_TREYARCH = "TAffx100"; - static constexpr const char* MAGIC_SIGNED_PC_OAT = "ABff0100"; + static constexpr const char* MAGIC_SIGNED_TREYARCH = "TAff0100"; + static constexpr const char* MAGIC_SIGNED_LZX_TREYARCH = "TAffx100"; + static constexpr const char* MAGIC_SIGNED_OAT = "ABff0100"; static constexpr const char* MAGIC_UNSIGNED = "TAffu100"; static constexpr const char* MAGIC_UNSIGNED_SERVER = "TAsvu100"; - static_assert(std::char_traits::length(MAGIC_SIGNED_PC_TREYARCH) == sizeof(ZoneHeader::m_magic)); - static_assert(std::char_traits::length(MAGIC_SIGNED_XENON_TREYARCH) == sizeof(ZoneHeader::m_magic)); - static_assert(std::char_traits::length(MAGIC_SIGNED_PC_OAT) == sizeof(ZoneHeader::m_magic)); + static_assert(std::char_traits::length(MAGIC_SIGNED_TREYARCH) == sizeof(ZoneHeader::m_magic)); + static_assert(std::char_traits::length(MAGIC_SIGNED_LZX_TREYARCH) == sizeof(ZoneHeader::m_magic)); + static_assert(std::char_traits::length(MAGIC_SIGNED_OAT) == sizeof(ZoneHeader::m_magic)); static_assert(std::char_traits::length(MAGIC_UNSIGNED) == sizeof(ZoneHeader::m_magic)); static_assert(std::char_traits::length(MAGIC_UNSIGNED_SERVER) == sizeof(ZoneHeader::m_magic)); diff --git a/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.cpp b/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.cpp index 63018e4c..01968187 100644 --- a/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.cpp +++ b/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.cpp @@ -1,8 +1,9 @@ #include "AbstractSalsa20Processor.h" #include +#include -AbstractSalsa20Processor::AbstractSalsa20Processor(const unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, const unsigned keySize) +AbstractSalsa20Processor::AbstractSalsa20Processor(const unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize) : m_stream_count(streamCount), m_stream_contexts(streamCount), m_block_hashes(BLOCK_HASHES_COUNT * streamCount * SHA1_HASH_SIZE), diff --git a/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.h b/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.h index b5045c54..d4b3775d 100644 --- a/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.h +++ b/src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.h @@ -31,7 +31,7 @@ protected: static constexpr auto SHA1_HASH_SIZE = 20u; static constexpr auto SALSA20_IV_SIZE = 8u; - AbstractSalsa20Processor(unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, unsigned keySize); + AbstractSalsa20Processor(unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); void InitStreams(const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); [[nodiscard]] uint8_t* GetHashBlock(unsigned streamNumber); diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp new file mode 100644 index 00000000..512a2a53 --- /dev/null +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp @@ -0,0 +1,12 @@ +#include "XChunkProcessorLzxDecompress.h" + +#include + +size_t + XChunkProcessorLzxDecompress::Process(unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) +{ + // TODO: Implement + + memcpy(output, input, inputLength); + return inputLength; +} diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h new file mode 100644 index 00000000..7e79c367 --- /dev/null +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h @@ -0,0 +1,8 @@ +#pragma once +#include "IXChunkProcessor.h" + +class XChunkProcessorLzxDecompress final : public IXChunkProcessor +{ +public: + size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; +}; diff --git a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp index 7c0a9718..caa9b1c2 100644 --- a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp +++ b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp @@ -19,6 +19,7 @@ #include "Loading/Steps/StepVerifySignature.h" #include "Utils/Endianness.h" #include "Zone/XChunk/XChunkProcessorInflate.h" +#include "Zone/XChunk/XChunkProcessorLzxDecompress.h" #include "Zone/XChunk/XChunkProcessorSalsa20Decryption.h" #include @@ -49,12 +50,13 @@ namespace return GameLanguage::LANGUAGE_NONE; } - bool CanLoad(const ZoneHeader& header, bool& isBigEndian, bool& isSecure, bool& isOfficial, bool& isEncrypted) + bool CanLoad(const ZoneHeader& header, bool& isBigEndian, bool& isSecure, bool& isOfficial, bool& isEncrypted, bool& isLzxCompressed) { if (endianness::FromLittleEndian(header.m_version) == ZoneConstants::ZONE_VERSION_PC) { isBigEndian = false; - if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_PC_TREYARCH, 8)) + isLzxCompressed = false; + if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_TREYARCH, 8)) { isSecure = true; isOfficial = true; @@ -62,7 +64,7 @@ namespace return true; } - if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_PC_OAT, 8)) + if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_OAT, 8)) { isSecure = true; isOfficial = false; @@ -89,11 +91,20 @@ namespace else if (endianness::FromBigEndian(header.m_version) == ZoneConstants::ZONE_VERSION_XENON) { isBigEndian = true; - if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_XENON_TREYARCH, 8)) + if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_TREYARCH, 8)) { isSecure = true; isOfficial = true; isEncrypted = true; + isLzxCompressed = false; + return true; + } + if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_LZX_TREYARCH, 8)) + { + isSecure = true; + isOfficial = true; + isEncrypted = true; + isLzxCompressed = true; return true; } } @@ -157,13 +168,12 @@ namespace return signatureLoadStepPtr; } - ICapturedDataProvider* AddXChunkProcessor(const bool isBigEndian, const bool isEncrypted, ZoneLoader& zoneLoader, std::string& fileName) + ICapturedDataProvider* + AddXChunkProcessor(const bool isBigEndian, const bool isEncrypted, const bool isLzxCompressed, ZoneLoader& zoneLoader, std::string& fileName) { ICapturedDataProvider* result = nullptr; - auto xChunkProcessor = processor::CreateProcessorXChunks(ZoneConstants::STREAM_COUNT, - ZoneConstants::XCHUNK_SIZE, - isBigEndian ? GameEndianness::BIG_ENDIAN : GameEndianness::LITTLE_ENDIAN, - ZoneConstants::VANILLA_BUFFER_SIZE); + auto xChunkProcessor = processor::CreateProcessorXChunks( + ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, isBigEndian ? GameEndianness::BE : GameEndianness::LE, ZoneConstants::VANILLA_BUFFER_SIZE); const uint8_t (&salsa20Key)[32] = isBigEndian ? ZoneConstants::SALSA20_KEY_TREYARCH_XENON : ZoneConstants::SALSA20_KEY_TREYARCH_PC; @@ -176,8 +186,17 @@ namespace xChunkProcessor->AddChunkProcessor(std::move(chunkProcessorSalsa20)); } - // Decompress the chunks using zlib - xChunkProcessor->AddChunkProcessor(std::make_unique()); + if (isLzxCompressed) + { + // Decompress the chunks using lzx + xChunkProcessor->AddChunkProcessor(std::make_unique()); + } + else + { + // Decompress the chunks using zlib + xChunkProcessor->AddChunkProcessor(std::make_unique()); + } + zoneLoader.AddLoadingStep(step::CreateStepAddProcessor(std::move(xChunkProcessor))); // If there is encryption, the signed data of the zone is the final hash blocks provided by the Salsa20 IV adaption algorithm @@ -187,10 +206,10 @@ namespace std::unique_ptr ZoneLoaderFactory::CreateLoaderForHeader(ZoneHeader& header, std::string& fileName) const { - bool isBigEndian, isSecure, isOfficial, isEncrypted; + bool isBigEndian, isSecure, isOfficial, isEncrypted, isLzxCompressed; // Check if this file is a supported T6 zone. - if (!CanLoad(header, isBigEndian, isSecure, isOfficial, isEncrypted)) + if (!CanLoad(header, isBigEndian, isSecure, isOfficial, isEncrypted, isLzxCompressed)) return nullptr; // Create new zone @@ -211,7 +230,7 @@ std::unique_ptr ZoneLoaderFactory::CreateLoaderForHeader(ZoneHeader& ISignatureProvider* signatureProvider = AddAuthHeaderSteps(isSecure, *zoneLoader, fileName); // Setup loading XChunks from the zone from this point on. - ICapturedDataProvider* signatureDataProvider = AddXChunkProcessor(isBigEndian, isEncrypted, *zoneLoader, fileName); + ICapturedDataProvider* signatureDataProvider = AddXChunkProcessor(isBigEndian, isEncrypted, isLzxCompressed, *zoneLoader, fileName); if (!isBigEndian) { diff --git a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp index cb57c662..af4f2c51 100644 --- a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp +++ b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp @@ -231,7 +231,7 @@ namespace return; } - if (m_endianness == GameEndianness::LITTLE_ENDIAN) + if (m_endianness == GameEndianness::LE) chunkSize = endianness::FromLittleEndian(chunkSize); else chunkSize = endianness::FromBigEndian(chunkSize); diff --git a/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp b/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp index 5edf9dcf..536e08b6 100644 --- a/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp +++ b/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp @@ -49,9 +49,9 @@ namespace if (isSecure) { if (isOfficial) - memcpy(header.m_magic, ZoneConstants::MAGIC_SIGNED_PC_TREYARCH, sizeof(ZoneHeader::m_magic)); + memcpy(header.m_magic, ZoneConstants::MAGIC_SIGNED_TREYARCH, sizeof(ZoneHeader::m_magic)); else - memcpy(header.m_magic, ZoneConstants::MAGIC_SIGNED_PC_OAT, sizeof(ZoneHeader::m_magic)); + memcpy(header.m_magic, ZoneConstants::MAGIC_SIGNED_OAT, sizeof(ZoneHeader::m_magic)); } else { From 75d579b5e317afb06b382b7a1796f0c82b69bab8 Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Thu, 28 Aug 2025 21:14:12 +0200 Subject: [PATCH 3/4] feat: dump t6 xbox fastfile data --- premake5.lua | 2 + src/ZoneCommon.lua | 2 + .../XChunk/XChunkProcessorLzxDecompress.cpp | 130 ++- .../XChunk/XChunkProcessorLzxDecompress.h | 8 + src/ZoneLoading.lua | 2 + .../Game/T6/ZoneLoaderFactoryT6.cpp | 16 +- .../Loading/Processor/ProcessorXChunks.cpp | 11 +- thirdparty/lzx.lua | 48 ++ thirdparty/lzx/lzx.c | 808 ++++++++++++++++++ thirdparty/lzx/lzx.h | 61 ++ 10 files changed, 1074 insertions(+), 14 deletions(-) create mode 100644 thirdparty/lzx.lua create mode 100644 thirdparty/lzx/lzx.c create mode 100644 thirdparty/lzx/lzx.h diff --git a/premake5.lua b/premake5.lua index 4a0f4d8a..092121c6 100644 --- a/premake5.lua +++ b/premake5.lua @@ -95,6 +95,7 @@ include "thirdparty/catch2.lua" include "thirdparty/eigen.lua" include "thirdparty/libtomcrypt.lua" include "thirdparty/libtommath.lua" +include "thirdparty/lzx.lua" include "thirdparty/json.lua" include "thirdparty/minilzo.lua" include "thirdparty/minizip.lua" @@ -108,6 +109,7 @@ group "ThirdParty" eigen:project() libtommath:project() libtomcrypt:project() + lzx:project() json:project() minilzo:project() minizip:project() diff --git a/src/ZoneCommon.lua b/src/ZoneCommon.lua index 3f005f49..bd887778 100644 --- a/src/ZoneCommon.lua +++ b/src/ZoneCommon.lua @@ -20,6 +20,7 @@ function ZoneCommon:link(links) links:linkto(ObjCommon) links:linkto(Parser) links:linkto(Utils) + links:linkto(lzx) end function ZoneCommon:use() @@ -45,5 +46,6 @@ function ZoneCommon:project() path.join(folder, "ZoneCommon/**.cpp") } + lzx:include(includes) self:include(includes) end diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp index 512a2a53..657f9143 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp @@ -1,12 +1,132 @@ #include "XChunkProcessorLzxDecompress.h" #include +#include +#include +#include -size_t - XChunkProcessorLzxDecompress::Process(unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) +namespace { - // TODO: Implement + uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize) + { + const auto value = input[offset]; + offset++; + remainingSize--; + return value; + } - memcpy(output, input, inputLength); - return inputLength; + uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte) + { + return static_cast(static_cast(static_cast(highByte) << 8u) | static_cast(lowByte)); + } + + void LogErrorHeaderSpace(size_t remainingInputSize) + { + std::cerr << std::format("XMemCompress: Not enough data for header: {}\n", remainingInputSize); + } +} // namespace + +XChunkProcessorLzxDecompress::XChunkProcessorLzxDecompress(const unsigned streamCount) + : m_lzx_states(streamCount) +{ + // T6 uses 17 for window bits + for (auto& lzxState : m_lzx_states) + lzxState = lzx_init(17); +} + +XChunkProcessorLzxDecompress::~XChunkProcessorLzxDecompress() +{ + for (auto* lzxState : m_lzx_states) + lzx_teardown(static_cast(lzxState)); +} + +size_t XChunkProcessorLzxDecompress::Process( + const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) +{ + auto* state = static_cast(m_lzx_states[streamNumber]); + + // lzx state is reset before each chunk + lzx_reset(state); + + size_t curInputOffset = 0uz; + size_t curInputSize = inputLength; + + size_t curOutputOffset = 0uz; + size_t curOutputSize = outputBufferSize; + + uint8_t lowByte; + uint16_t dstSize, srcSize; + + while (curInputSize > 0) + { + uint8_t highByte = NextByte(input, curInputOffset, curInputSize); + + uint8_t suffixSize; + if (highByte == 0xFF) // magic number: output is smaller than 0x8000 + { + if (curInputSize < 4) + { + LogErrorHeaderSpace(curInputSize); + return curOutputOffset; + } + + highByte = NextByte(input, curInputOffset, curInputSize); + lowByte = NextByte(input, curInputOffset, curInputSize); + dstSize = CombineHighLow(highByte, lowByte); + + highByte = NextByte(input, curInputOffset, curInputSize); + lowByte = NextByte(input, curInputOffset, curInputSize); + srcSize = CombineHighLow(highByte, lowByte); + + // The game seems to skip a 5 byte suffix after these blocks, not sure why. + suffixSize = 5u; + } + else + { + if (curInputSize < 1) + { + LogErrorHeaderSpace(curInputSize); + return curOutputOffset; + } + + dstSize = 0x8000u; + lowByte = NextByte(input, curInputOffset, curInputSize); + srcSize = CombineHighLow(highByte, lowByte); + + suffixSize = 0u; + } + + if (srcSize == 0 || dstSize == 0) + { + // Other implementations do not handle this as a failure, game code suggests otherwise though + std::cerr << std::format("XMemCompress: EOF: {} {}, {}\n", srcSize, dstSize, curInputSize); + return curOutputOffset; + } + + if (static_cast(srcSize) + suffixSize > curInputSize) + { + std::cerr << std::format("XMemCompress: block size bigger than remaining data: {} > {}\n", srcSize, curInputSize); + return curOutputOffset; + } + + if (dstSize > curOutputSize) + { + std::cerr << std::format("XMemCompress: output size bigger than remaining data: {} > {}\n", dstSize, curOutputSize); + return curOutputOffset; + } + + auto ret = lzx_decompress(state, &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize); + curInputOffset += srcSize + suffixSize; + curInputSize -= (srcSize + suffixSize); + curOutputOffset += dstSize; + curOutputSize -= srcSize; + + if (ret != DECR_OK) + { + std::cerr << std::format("XMemCompress: lzx decompression failed: {}\n", ret); + return curOutputOffset; + } + } + + return curOutputOffset; } diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h index 7e79c367..e32ee92a 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h @@ -1,8 +1,16 @@ #pragma once + #include "IXChunkProcessor.h" +#include + class XChunkProcessorLzxDecompress final : public IXChunkProcessor { public: + explicit XChunkProcessorLzxDecompress(unsigned streamCount); + ~XChunkProcessorLzxDecompress(); size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; + +private: + std::vector m_lzx_states; }; diff --git a/src/ZoneLoading.lua b/src/ZoneLoading.lua index 0d555bca..3819a0f6 100644 --- a/src/ZoneLoading.lua +++ b/src/ZoneLoading.lua @@ -14,6 +14,7 @@ function ZoneLoading:link(links) links:linkto(Cryptography) links:linkto(Utils) links:linkto(ZoneCommon) + links:linkto(lzx) links:linkto(zlib) if os.host() == "linux" then @@ -55,6 +56,7 @@ function ZoneLoading:project() self:include(includes) Cryptography:include(includes) Utils:include(includes) + lzx:include(includes) zlib:include(includes) ZoneCode:include(includes) diff --git a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp index caa9b1c2..e0eaf669 100644 --- a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp +++ b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp @@ -172,8 +172,18 @@ namespace AddXChunkProcessor(const bool isBigEndian, const bool isEncrypted, const bool isLzxCompressed, ZoneLoader& zoneLoader, std::string& fileName) { ICapturedDataProvider* result = nullptr; - auto xChunkProcessor = processor::CreateProcessorXChunks( - ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, isBigEndian ? GameEndianness::BE : GameEndianness::LE, ZoneConstants::VANILLA_BUFFER_SIZE); + std::unique_ptr xChunkProcessor; + + if (isBigEndian) + { + xChunkProcessor = processor::CreateProcessorXChunks( + ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, GameEndianness::BE, ZoneConstants::VANILLA_BUFFER_SIZE); + } + else + { + xChunkProcessor = processor::CreateProcessorXChunks( + ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, GameEndianness::LE, ZoneConstants::VANILLA_BUFFER_SIZE); + } const uint8_t (&salsa20Key)[32] = isBigEndian ? ZoneConstants::SALSA20_KEY_TREYARCH_XENON : ZoneConstants::SALSA20_KEY_TREYARCH_PC; @@ -189,7 +199,7 @@ namespace if (isLzxCompressed) { // Decompress the chunks using lzx - xChunkProcessor->AddChunkProcessor(std::make_unique()); + xChunkProcessor->AddChunkProcessor(std::make_unique(ZoneConstants::STREAM_COUNT)); } else { diff --git a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp index af4f2c51..0e7bf18b 100644 --- a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp +++ b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp @@ -223,19 +223,18 @@ namespace } const size_t readSize = m_base_stream->Load(&chunkSize, sizeof(chunkSize)); + if (m_endianness == GameEndianness::LE) + chunkSize = endianness::FromLittleEndian(chunkSize); + else + chunkSize = endianness::FromBigEndian(chunkSize); - if (readSize == 0) + if (readSize < sizeof(chunkSize) || chunkSize == 0) { m_eof_reached = true; m_eof_stream = streamNum; return; } - if (m_endianness == GameEndianness::LE) - chunkSize = endianness::FromLittleEndian(chunkSize); - else - chunkSize = endianness::FromBigEndian(chunkSize); - if (chunkSize > m_chunk_size) { throw InvalidChunkSizeException(chunkSize, m_chunk_size); diff --git a/thirdparty/lzx.lua b/thirdparty/lzx.lua new file mode 100644 index 00000000..1674eb4d --- /dev/null +++ b/thirdparty/lzx.lua @@ -0,0 +1,48 @@ +lzx = {} + +function lzx:include(includes) + if includes:handle(self:name()) then + includedirs { + path.join(ThirdPartyFolder(), "lzx") + } + end +end + +function lzx:link(links) + links:add(self:name()) +end + +function lzx:use() + +end + +function lzx:name() + return "lzx" +end + +function lzx:project() + local folder = ThirdPartyFolder() + local includes = Includes:create() + + project(self:name()) + targetdir(TargetDirectoryLib) + location "%{wks.location}/thirdparty/%{prj.name}" + kind "StaticLib" + language "C" + + files { + path.join(folder, "lzx/*.h"), + path.join(folder, "lzx/*.c") + } + + defines { + "_CRT_SECURE_NO_WARNINGS", + "_CRT_NONSTDC_NO_DEPRECATE" + } + + + self:include(includes) + + -- Disable warnings. They do not have any value to us since it is not our code. + warnings "off" +end diff --git a/thirdparty/lzx/lzx.c b/thirdparty/lzx/lzx.c new file mode 100644 index 00000000..c1199a72 --- /dev/null +++ b/thirdparty/lzx/lzx.c @@ -0,0 +1,808 @@ +// Modified version of the code from Wine: +// https://gitlab.winehq.org/wine/wine/-/blob/fcc40a07909dc7626b6d1e2ec73f823d828a47e8/dlls/itss/lzx.c + +/*************************************************************************** + * lzx.c - LZX decompression routines * + * ------------------- * + * * + * maintainer: Jed Wing * + * source: modified lzx.c from cabextract v0.5 * + * notes: This file was taken from cabextract v0.5, which was, * + * itself, a modified version of the lzx decompression code * + * from unlzx. * + * * + * platforms: In its current incarnation, this file has been tested on * + * two different Linux platforms (one, redhat-based, with a * + * 2.1.2 glibc and gcc 2.95.x, and the other, Debian, with * + * 2.2.4 glibc and both gcc 2.95.4 and gcc 3.0.2). Both were * + * Intel x86 compatible machines. * + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. Note that an exemption to this * + * license has been granted by Stuart Caie for the purposes of * + * distribution with chmlib. This does not, to the best of my * + * knowledge, constitute a change in the license of this (the LZX) code * + * in general. * + * * + ***************************************************************************/ + +#include "lzx.h" +#include +#include +#include +#include + +/* some constants defined by the LZX specification */ +#define LZX_MIN_MATCH 2 +/* #define LZX_MAX_MATCH 257 */ +#define LZX_NUM_CHARS 256 +#define LZX_BLOCKTYPE_INVALID 0 /* also blocktypes 4-7 invalid */ +#define LZX_BLOCKTYPE_VERBATIM 1 +#define LZX_BLOCKTYPE_ALIGNED 2 +#define LZX_BLOCKTYPE_UNCOMPRESSED 3 +#define LZX_PRETREE_NUM_ELEMENTS 20 +#define LZX_ALIGNED_NUM_ELEMENTS 8 /* aligned offset tree #elements */ +#define LZX_NUM_PRIMARY_LENGTHS 7 /* this one missing from spec! */ +#define LZX_NUM_SECONDARY_LENGTHS 249 /* length tree #elements */ + +/* LZX huffman defines: tweak tablebits as desired */ +#define LZX_PRETREE_MAXSYMBOLS LZX_PRETREE_NUM_ELEMENTS +#define LZX_PRETREE_TABLEBITS 6 +#define LZX_MAINTREE_MAXSYMBOLS (LZX_NUM_CHARS + 50 * 8) +#define LZX_MAINTREE_TABLEBITS 12 +#define LZX_LENGTH_MAXSYMBOLS (LZX_NUM_SECONDARY_LENGTHS + 1) +#define LZX_LENGTH_TABLEBITS 12 +#define LZX_ALIGNED_MAXSYMBOLS LZX_ALIGNED_NUM_ELEMENTS +#define LZX_ALIGNED_TABLEBITS 7 + +#define LZX_LENTABLE_SAFETY 64 /* we allow length table decoding overruns */ + +#define LZX_DECLARE_TABLE(tbl) \ + uint16_t tbl## _table[(1 << LZX_## tbl## _TABLEBITS) + (LZX_## tbl## _MAXSYMBOLS << 1)]; \ + uint8_t tbl## _len[LZX_## tbl## _MAXSYMBOLS + LZX_LENTABLE_SAFETY] + +struct lzx_state { + uint8_t* window; /* the actual decoding window */ + uint32_t window_size; /* window size (32Kb through 2Mb) */ + uint32_t actual_size; /* window size when it was first allocated */ + uint32_t window_posn; /* current offset within the window */ + uint32_t R0, R1, R2; /* for the LRU offset system */ + uint16_t main_elements; /* number of main tree elements */ + int header_read; /* have we started decoding at all yet? */ + uint16_t block_type; /* type of this block */ + uint32_t block_length; /* uncompressed length of this block */ + uint32_t block_remaining; /* uncompressed bytes still left to decode */ + uint32_t frames_read; /* the number of CFDATA blocks processed */ + int32_t intel_filesize; /* magic header value used for transform */ + int32_t intel_curpos; /* current offset in transform space */ + int intel_started; /* have we seen any translatable data yet? */ + + LZX_DECLARE_TABLE(PRETREE); + LZX_DECLARE_TABLE(MAINTREE); + LZX_DECLARE_TABLE(LENGTH); + LZX_DECLARE_TABLE(ALIGNED); +}; + +/* LZX decruncher */ + +/* Microsoft's LZX document and their implementation of the + * com.ms.util.cab Java package do not concur. + * + * In the LZX document, there is a table showing the correlation between + * window size and the number of position slots. It states that the 1MB + * window = 40 slots and the 2MB window = 42 slots. In the implementation, + * 1MB = 42 slots, 2MB = 50 slots. The actual calculation is 'find the + * first slot whose position base is equal to or more than the required + * window size'. This would explain why other tables in the document refer + * to 50 slots rather than 42. + * + * The constant NUM_PRIMARY_LENGTHS used in the decompression pseudocode + * is not defined in the specification. + * + * The LZX document does not state the uncompressed block has an + * uncompressed length field. Where does this length field come from, so + * we can know how large the block is? The implementation has it as the 24 + * bits following after the 3 blocktype bits, before the alignment + * padding. + * + * The LZX document states that aligned offset blocks have their aligned + * offset huffman tree AFTER the main and length trees. The implementation + * suggests that the aligned offset tree is BEFORE the main and length + * trees. + * + * The LZX document decoding algorithm states that, in an aligned offset + * block, if an extra_bits value is 1, 2 or 3, then that number of bits + * should be read and the result added to the match offset. This is + * correct for 1 and 2, but not 3, where just a huffman symbol (using the + * aligned tree) should be read. + * + * Regarding the E8 preprocessing, the LZX document states 'No translation + * may be performed on the last 6 bytes of the input block'. This is + * correct. However, the pseudocode provided checks for the *E8 leader* + * up to the last 6 bytes. If the leader appears between -10 and -7 bytes + * from the end, this would cause the next four bytes to be modified, at + * least one of which would be in the last 6 bytes, which is not allowed + * according to the spec. + * + * The specification states that the huffman trees must always contain at + * least one element. However, many CAB files contain blocks where the + * length tree is completely empty (because there are no matches), and + * this is expected to succeed. + */ + +/* LZX uses what it calls 'position slots' to represent match offsets. + * What this means is that a small 'position slot' number and a small + * offset from that slot are encoded instead of one large offset for + * every match. + * - position_base is an index to the position slot bases + * - extra_bits states how many bits of offset-from-base data is needed. + */ +static const uint8_t extra_bits[51] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, + 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, + 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17}; + +static const uint32_t position_base[51] = { + 0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, + 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, + 2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536, + 98304, 131072, 196608, 262144, 393216, 524288, 655360, 786432, 917504, 1048576, 1179648, + 1310720, 1441792, 1572864, 1703936, 1835008, 1966080, 2097152}; + +struct lzx_state* lzx_init(int window) { + struct lzx_state* pState = NULL; + uint32_t wndsize = 1 << window; + int i, posn_slots; + + /* LZX supports window sizes of 2^15 (32Kb) through 2^21 (2Mb) */ + /* if a previously allocated window is big enough, keep it */ + if (window < 15 || window > 21) + return NULL; + + /* allocate state and associated window */ + pState = (struct lzx_state*)malloc(sizeof(struct lzx_state)); + if (!pState || !(pState->window = (uint8_t*)malloc(wndsize))) { + free(pState); + return NULL; + } + pState->actual_size = wndsize; + pState->window_size = wndsize; + + /* calculate required position slots */ + if (window == 20) + posn_slots = 42; + else if (window == 21) + posn_slots = 50; + else + posn_slots = window << 1; + + /** alternatively **/ + /* posn_slots=i=0; while (i < wndsize) i += 1 << extra_bits[posn_slots++]; */ + + /* initialize other state */ + pState->R0 = pState->R1 = pState->R2 = 1; + pState->main_elements = LZX_NUM_CHARS + (posn_slots << 3); + pState->header_read = 0; + pState->frames_read = 0; + pState->block_remaining = 0; + pState->block_type = LZX_BLOCKTYPE_INVALID; + pState->intel_curpos = 0; + pState->intel_started = 0; + pState->window_posn = 0; + + /* initialise tables to 0 (because deltas will be applied to them) */ + for (i = 0; i < LZX_MAINTREE_MAXSYMBOLS; i++) + pState->MAINTREE_len[i] = 0; + for (i = 0; i < LZX_LENGTH_MAXSYMBOLS; i++) + pState->LENGTH_len[i] = 0; + + return pState; +} + +void lzx_teardown(struct lzx_state* pState) { + if (pState) { + if (pState->window) + free(pState->window); + free(pState); + } +} + +void lzx_reset(struct lzx_state* pState) { + int i; + + pState->R0 = pState->R1 = pState->R2 = 1; + pState->header_read = 0; + pState->frames_read = 0; + pState->block_remaining = 0; + pState->block_type = LZX_BLOCKTYPE_INVALID; + pState->intel_curpos = 0; + pState->intel_started = 0; + pState->window_posn = 0; + + for (i = 0; i < LZX_MAINTREE_MAXSYMBOLS + LZX_LENTABLE_SAFETY; i++) { + pState->MAINTREE_len[i] = 0; + } + + for (i = 0; i < LZX_LENGTH_MAXSYMBOLS + LZX_LENTABLE_SAFETY; i++) { + pState->LENGTH_len[i] = 0; + } +} + +/* Bitstream reading macros: + * + * INIT_BITSTREAM should be used first to set up the system + * READ_BITS(var,n) takes N bits from the buffer and puts them in var + * + * ENSURE_BITS(n) ensures there are at least N bits in the bit buffer + * PEEK_BITS(n) extracts (without removing) N bits from the bit buffer + * REMOVE_BITS(n) removes N bits from the bit buffer + * + * These bit access routines work by using the area beyond the MSB and the + * LSB as a free source of zeroes. This avoids having to mask any bits. + * So we have to know the bit width of the bitbuffer variable. This is + * sizeof(uint32_t) * 8, also defined as uint32_t_BITS + */ + +/* number of bits in uint32_t. Note: This must be at multiple of 16, and at + * least 32 for the bitbuffer code to work (ie, it must be able to ensure + * up to 17 bits - that's adding 16 bits when there's one bit left, or + * adding 32 bits when there are no bits left. The code should work fine + * for machines where uint32_t >= 32 bits. + */ +#define uint32_t_BITS (sizeof(uint32_t) << 3) + +#define INIT_BITSTREAM \ + do { \ + bitsleft = 0; \ + bitbuf = 0; \ + } while (0) + +#define ENSURE_BITS(n) \ + while (bitsleft < (n)) { \ + bitbuf |= ((inpos[1] << 8) | inpos[0]) << (uint32_t_BITS - 16 - bitsleft); \ + bitsleft += 16; \ + inpos += 2; \ + } + +#define PEEK_BITS(n) (bitbuf >> (uint32_t_BITS - (n))) +#define REMOVE_BITS(n) ((bitbuf <<= (n)), (bitsleft -= (n))) + +#define READ_BITS(v, n) \ + do { \ + ENSURE_BITS(n); \ + (v) = PEEK_BITS(n); \ + REMOVE_BITS(n); \ + } while (0) + +/* Huffman macros */ + +#define TABLEBITS(tbl) (LZX_## tbl## _TABLEBITS) +#define MAXSYMBOLS(tbl) (LZX_## tbl## _MAXSYMBOLS) +#define SYMTABLE(tbl) (pState->tbl## _table) +#define LENTABLE(tbl) (pState->tbl## _len) + +/* BUILD_TABLE(tablename) builds a huffman lookup table from code lengths. + * In reality, it just calls make_decode_table() with the appropriate + * values - they're all fixed by some #defines anyway, so there's no point + * writing each call out in full by hand. + */ +#define BUILD_TABLE(tbl) \ + if (make_decode_table(MAXSYMBOLS(tbl), TABLEBITS(tbl), LENTABLE(tbl), SYMTABLE(tbl))) { \ + return DECR_ILLEGALDATA; \ + } + +/* READ_HUFFSYM(tablename, var) decodes one huffman symbol from the + * bitstream using the stated table and puts it in var. + */ +#define READ_HUFFSYM(tbl, var) \ + do { \ + ENSURE_BITS(16); \ + hufftbl = SYMTABLE(tbl); \ + if ((i = hufftbl[PEEK_BITS(TABLEBITS(tbl))]) >= MAXSYMBOLS(tbl)) { \ + j = 1 << (uint32_t_BITS - TABLEBITS(tbl)); \ + do { \ + j >>= 1; \ + i <<= 1; \ + i |= (bitbuf & j) ? 1 : 0; \ + if (!j) { \ + return DECR_ILLEGALDATA; \ + } \ + } while ((i = hufftbl[i]) >= MAXSYMBOLS(tbl)); \ + } \ + j = LENTABLE(tbl)[(var) = i]; \ + REMOVE_BITS(j); \ + } while (0) + +/* READ_LENGTHS(tablename, first, last) reads in code lengths for symbols + * first to last in the given table. The code lengths are stored in their + * own special LZX way. + */ +#define READ_LENGTHS(tbl, first, last) \ + do { \ + lb.bb = bitbuf; \ + lb.bl = bitsleft; \ + lb.ip = inpos; \ + if (lzx_read_lens(pState, LENTABLE(tbl), (first), (last), &lb)) { \ + return DECR_ILLEGALDATA; \ + } \ + bitbuf = lb.bb; \ + bitsleft = lb.bl; \ + inpos = lb.ip; \ + } while (0) + +/* make_decode_table(nsyms, nbits, length[], table[]) + * + * This function was coded by David Tritscher. It builds a fast huffman + * decoding table out of just a canonical huffman code lengths table. + * + * nsyms = total number of symbols in this huffman tree. + * nbits = any symbols with a code length of nbits or less can be decoded + * in one lookup of the table. + * length = A table to get code lengths from [0 to syms-1] + * table = The table to fill up with decoded symbols and pointers. + * + * Returns 0 for OK or 1 for error + */ +static int make_decode_table(uint32_t nsyms, uint32_t nbits, uint8_t* length, uint16_t* table) { + uint16_t sym; + uint32_t leaf; + uint8_t bit_num = 1; + uint32_t fill; + uint32_t pos = 0; /* the current position in the decode table */ + uint32_t table_mask = 1 << nbits; + uint32_t bit_mask = table_mask >> 1; /* don't do 0 length codes */ + uint32_t next_symbol = bit_mask; /* base of allocation for long codes */ + + /* fill entries for codes short enough for a direct mapping */ + while (bit_num <= nbits) { + for (sym = 0; sym < nsyms; sym++) { + if (length[sym] != bit_num) { + continue; + } + leaf = pos; + + if ((pos += bit_mask) > table_mask) + return 1; /* table overrun */ + + /* fill all possible lookups of this symbol with the symbol itself */ + fill = bit_mask; + while (fill-- > 0) { + table[leaf++] = sym; + } + } + bit_mask >>= 1; + bit_num++; + } + + /* if there are any codes longer than nbits */ + if (pos != table_mask) { + /* clear the remainder of the table */ + for (sym = pos; sym < table_mask; sym++) { + table[sym] = 0; + } + + /* give ourselves room for codes to grow by up to 16 more bits */ + pos <<= 16; + table_mask <<= 16; + bit_mask = 1 << 15; + + while (bit_num <= 16) { + for (sym = 0; sym < nsyms; sym++) { + if (length[sym] != bit_num) { + continue; + } + leaf = pos >> 16; + for (fill = 0; fill < bit_num - nbits; fill++) { + /* if this path hasn't been taken yet, 'allocate' two entries */ + if (table[leaf] == 0) { + table[(next_symbol << 1)] = 0; + table[(next_symbol << 1) + 1] = 0; + table[leaf] = next_symbol++; + } + /* follow the path and select either left or right for next bit */ + leaf = table[leaf] << 1; + if ((pos >> (15 - fill)) & 1) + leaf++; + } + table[leaf] = sym; + + if ((pos += bit_mask) > table_mask) + return 1; /* table overflow */ + } + bit_mask >>= 1; + bit_num++; + } + } + + /* full table? */ + if (pos == table_mask) + return 0; + + /* either erroneous table, or all elements are 0 - let's find out. */ + for (sym = 0; sym < nsyms; sym++) { + if (length[sym]) + return 1; + } + return 0; +} + +struct lzx_bits { + uint32_t bb; + int bl; + uint8_t* ip; +}; + +static int lzx_read_lens(struct lzx_state* pState, uint8_t* lens, uint32_t first, uint32_t last, + struct lzx_bits* lb) { + uint32_t i, j, x, y; + int z; + + uint32_t bitbuf = lb->bb; + int bitsleft = lb->bl; + uint8_t* inpos = lb->ip; + uint16_t* hufftbl; + + for (x = 0; x < 20; x++) { + READ_BITS(y, 4); + LENTABLE(PRETREE)[x] = y; + } + BUILD_TABLE(PRETREE); + + for (x = first; x < last;) { + READ_HUFFSYM(PRETREE, z); + if (z == 17) { + READ_BITS(y, 4); + y += 4; + while (y--) + lens[x++] = 0; + } else if (z == 18) { + READ_BITS(y, 5); + y += 20; + while (y--) + lens[x++] = 0; + } else if (z == 19) { + READ_BITS(y, 1); + y += 4; + READ_HUFFSYM(PRETREE, z); + z = lens[x] - z; + if (z < 0) + z += 17; + while (y--) + lens[x++] = z; + } else { + z = lens[x] - z; + if (z < 0) + z += 17; + lens[x++] = z; + } + } + + lb->bb = bitbuf; + lb->bl = bitsleft; + lb->ip = inpos; + return 0; +} + +int lzx_decompress(struct lzx_state* pState, const unsigned char* inpos, unsigned char* outpos, int inlen, + int outlen) { + const uint8_t* endinp = inpos + inlen; + uint8_t* window = pState->window; + uint8_t* runsrc, *rundest; + uint16_t* hufftbl; /* used in READ_HUFFSYM macro as chosen decoding table */ + + uint32_t window_posn = pState->window_posn; + uint32_t window_size = pState->window_size; + uint32_t R0 = pState->R0; + uint32_t R1 = pState->R1; + uint32_t R2 = pState->R2; + + uint32_t bitbuf; + int bitsleft; + uint32_t match_offset, i, j, k; /* ijk used in READ_HUFFSYM macro */ + struct lzx_bits lb; /* used in READ_LENGTHS macro */ + + int togo = outlen, this_run, main_element, aligned_bits; + int match_length, length_footer, extra, verbatim_bits; + + INIT_BITSTREAM; + + /* read header if necessary */ + if (!pState->header_read) { + i = j = 0; + READ_BITS(k, 1); + if (k) { + READ_BITS(i, 16); + READ_BITS(j, 16); + } + pState->intel_filesize = (i << 16) | j; /* or 0 if not encoded */ + pState->header_read = 1; + } + + /* main decoding loop */ + while (togo > 0) { + /* last block finished, new block expected */ + if (pState->block_remaining == 0) { + if (pState->block_type == LZX_BLOCKTYPE_UNCOMPRESSED) { + if (pState->block_length & 1) + inpos++; /* realign bitstream to word */ + INIT_BITSTREAM; + } + + READ_BITS(pState->block_type, 3); + READ_BITS(i, 16); + READ_BITS(j, 8); + pState->block_remaining = pState->block_length = (i << 8) | j; + + switch (pState->block_type) { + case LZX_BLOCKTYPE_ALIGNED: + for (i = 0; i < 8; i++) { + READ_BITS(j, 3); + LENTABLE(ALIGNED)[i] = j; + } + BUILD_TABLE(ALIGNED); + /* rest of aligned header is same as verbatim */ + + case LZX_BLOCKTYPE_VERBATIM: + READ_LENGTHS(MAINTREE, 0, 256); + READ_LENGTHS(MAINTREE, 256, pState->main_elements); + BUILD_TABLE(MAINTREE); + if (LENTABLE(MAINTREE)[0xE8] != 0) + pState->intel_started = 1; + + READ_LENGTHS(LENGTH, 0, LZX_NUM_SECONDARY_LENGTHS); + BUILD_TABLE(LENGTH); + break; + + case LZX_BLOCKTYPE_UNCOMPRESSED: + pState->intel_started = 1; /* because we can't assume otherwise */ + ENSURE_BITS(16); /* get up to 16 pad bits into the buffer */ + if (bitsleft > 16) + inpos -= 2; /* and align the bitstream! */ + R0 = inpos[0] | (inpos[1] << 8) | (inpos[2] << 16) | (inpos[3] << 24); + inpos += 4; + R1 = inpos[0] | (inpos[1] << 8) | (inpos[2] << 16) | (inpos[3] << 24); + inpos += 4; + R2 = inpos[0] | (inpos[1] << 8) | (inpos[2] << 16) | (inpos[3] << 24); + inpos += 4; + break; + + default: + return DECR_ILLEGALDATA; + } + } + + /* buffer exhaustion check */ + if (inpos > endinp) { + /* it's possible to have a file where the next run is less than + * 16 bits in size. In this case, the READ_HUFFSYM() macro used + * in building the tables will exhaust the buffer, so we should + * allow for this, but not allow those accidentally read bits to + * be used (so we check that there are at least 16 bits + * remaining - in this boundary case they aren't really part of + * the compressed data) + */ + if (inpos > (endinp + 2) || bitsleft < 16) + return DECR_ILLEGALDATA; + } + + while ((this_run = pState->block_remaining) > 0 && togo > 0) { + if (this_run > togo) + this_run = togo; + togo -= this_run; + pState->block_remaining -= this_run; + + /* apply 2^x-1 mask */ + window_posn &= window_size - 1; + /* runs can't straddle the window wraparound */ + if ((window_posn + this_run) > window_size) + return DECR_DATAFORMAT; + + switch (pState->block_type) { + case LZX_BLOCKTYPE_VERBATIM: + while (this_run > 0) { + READ_HUFFSYM(MAINTREE, main_element); + + if (main_element < LZX_NUM_CHARS) { + /* literal: 0 to LZX_NUM_CHARS-1 */ + window[window_posn++] = main_element; + this_run--; + } else { + /* match: LZX_NUM_CHARS + ((slot<<3) | length_header (3 bits)) */ + main_element -= LZX_NUM_CHARS; + + match_length = main_element & LZX_NUM_PRIMARY_LENGTHS; + if (match_length == LZX_NUM_PRIMARY_LENGTHS) { + READ_HUFFSYM(LENGTH, length_footer); + match_length += length_footer; + } + match_length += LZX_MIN_MATCH; + + match_offset = main_element >> 3; + + if (match_offset > 2) { + /* not repeated offset */ + if (match_offset != 3) { + extra = extra_bits[match_offset]; + READ_BITS(verbatim_bits, extra); + match_offset = position_base[match_offset] - 2 + verbatim_bits; + } else { + match_offset = 1; + } + + /* update repeated offset LRU queue */ + R2 = R1; + R1 = R0; + R0 = match_offset; + } else if (match_offset == 0) { + match_offset = R0; + } else if (match_offset == 1) { + match_offset = R1; + R1 = R0; + R0 = match_offset; + } else /* match_offset == 2 */ { + match_offset = R2; + R2 = R0; + R0 = match_offset; + } + + rundest = window + window_posn; + runsrc = rundest - match_offset; + window_posn += match_length; + if (window_posn > window_size) + return DECR_ILLEGALDATA; + this_run -= match_length; + + /* copy any wrapped around source data */ + while ((runsrc < window) && (match_length-- > 0)) { + *rundest++ = *(runsrc + window_size); + runsrc++; + } + /* copy match data - no worries about destination wraps */ + while (match_length-- > 0) + *rundest++ = *runsrc++; + } + } + break; + + case LZX_BLOCKTYPE_ALIGNED: + while (this_run > 0) { + READ_HUFFSYM(MAINTREE, main_element); + + if (main_element < LZX_NUM_CHARS) { + /* literal: 0 to LZX_NUM_CHARS-1 */ + window[window_posn++] = main_element; + this_run--; + } else { + /* match: LZX_NUM_CHARS + ((slot<<3) | length_header (3 bits)) */ + main_element -= LZX_NUM_CHARS; + + match_length = main_element & LZX_NUM_PRIMARY_LENGTHS; + if (match_length == LZX_NUM_PRIMARY_LENGTHS) { + READ_HUFFSYM(LENGTH, length_footer); + match_length += length_footer; + } + match_length += LZX_MIN_MATCH; + + match_offset = main_element >> 3; + + if (match_offset > 2) { + /* not repeated offset */ + extra = extra_bits[match_offset]; + match_offset = position_base[match_offset] - 2; + if (extra > 3) { + /* verbatim and aligned bits */ + extra -= 3; + READ_BITS(verbatim_bits, extra); + match_offset += (verbatim_bits << 3); + READ_HUFFSYM(ALIGNED, aligned_bits); + match_offset += aligned_bits; + } else if (extra == 3) { + /* aligned bits only */ + READ_HUFFSYM(ALIGNED, aligned_bits); + match_offset += aligned_bits; + } else if (extra > 0) { /* extra==1, extra==2 */ + /* verbatim bits only */ + READ_BITS(verbatim_bits, extra); + match_offset += (uint32_t)verbatim_bits; + } else /* extra == 0 */ { + /* ??? */ + match_offset = 1; + } + + /* update repeated offset LRU queue */ + R2 = R1; + R1 = R0; + R0 = match_offset; + } else if (match_offset == 0) { + match_offset = R0; + } else if (match_offset == 1) { + match_offset = R1; + R1 = R0; + R0 = match_offset; + } else /* match_offset == 2 */ { + match_offset = R2; + R2 = R0; + R0 = match_offset; + } + + rundest = window + window_posn; + runsrc = rundest - match_offset; + window_posn += (uint32_t)match_length; + if (window_posn > window_size) + return DECR_ILLEGALDATA; + this_run -= match_length; + + /* copy any wrapped around source data */ + while ((runsrc < window) && (match_length-- > 0)) { + *rundest++ = *(runsrc + window_size); + runsrc++; + } + /* copy match data - no worries about destination wraps */ + while (match_length-- > 0) + *rundest++ = *runsrc++; + } + } + break; + + case LZX_BLOCKTYPE_UNCOMPRESSED: + if ((inpos + this_run) > endinp) + return DECR_ILLEGALDATA; + memcpy(window + window_posn, inpos, (size_t)this_run); + inpos += this_run; + window_posn += (uint32_t)this_run; + break; + + default: + return DECR_ILLEGALDATA; /* might as well */ + } + } + } + + if (togo != 0) + return DECR_ILLEGALDATA; + memcpy(outpos, window + ((!window_posn) ? window_size : window_posn) - outlen, (size_t)outlen); + + pState->window_posn = window_posn; + pState->R0 = R0; + pState->R1 = R1; + pState->R2 = R2; + + /* intel E8 decoding */ + if ((pState->frames_read++ < 32768) && pState->intel_filesize != 0) { + if (outlen <= 6 || !pState->intel_started) { + pState->intel_curpos += outlen; + } else { + uint8_t* data = outpos; + uint8_t* dataend = data + outlen - 10; + int32_t curpos = pState->intel_curpos; + int32_t filesize = pState->intel_filesize; + int32_t abs_off, rel_off; + + pState->intel_curpos = curpos + outlen; + + while (data < dataend) { + if (*data++ != 0xE8) { + curpos++; + continue; + } + abs_off = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24); + if ((abs_off >= -curpos) && (abs_off < filesize)) { + rel_off = (abs_off >= 0) ? abs_off - curpos : abs_off + filesize; + data[0] = (uint8_t)rel_off; + data[1] = (uint8_t)(rel_off >> 8); + data[2] = (uint8_t)(rel_off >> 16); + data[3] = (uint8_t)(rel_off >> 24); + } + data += 4; + curpos += 5; + } + } + } + return DECR_OK; +} + diff --git a/thirdparty/lzx/lzx.h b/thirdparty/lzx/lzx.h new file mode 100644 index 00000000..ce9dd5a4 --- /dev/null +++ b/thirdparty/lzx/lzx.h @@ -0,0 +1,61 @@ +// Modified version of the code from Wine: +// https://gitlab.winehq.org/wine/wine/-/blob/fcc40a07909dc7626b6d1e2ec73f823d828a47e8/dlls/itss/lzx.h + +/*************************************************************************** +* lzx.h - LZX decompression routines * +* ------------------- * +* * +* maintainer: Jed Wing * +* source: modified lzx.c from cabextract v0.5 * +* notes: This file was taken from cabextract v0.5, which was, * +* itself, a modified version of the lzx decompression code * +* from unlzx. * +***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. Note that an exemption to this * + * license has been granted by Stuart Caie for the purposes of * + * distribution with chmlib. This does not, to the best of my * + * knowledge, constitute a change in the license of this (the LZX) code * + * in general. * + * * + ***************************************************************************/ + +#ifndef INCLUDED_LZX_H +#define INCLUDED_LZX_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* return codes */ +#define DECR_OK (0) +#define DECR_DATAFORMAT (1) +#define DECR_ILLEGALDATA (2) +#define DECR_NOMEMORY (3) + +/* opaque state structure */ +struct lzx_state; + +/* create an lzx state object */ +struct lzx_state* lzx_init(int window); + +/* destroy an lzx state object */ +void lzx_teardown(struct lzx_state* pState); + +/* reset an lzx stream */ +void lzx_reset(struct lzx_state* pState); + +/* decompress an LZX compressed block */ +int lzx_decompress(struct lzx_state* pState, const unsigned char* inpos, unsigned char* outpos, int inlen, + int outlen); + +#ifdef __cplusplus +} +#endif + +#endif /* INCLUDED_LZX_H */ From 07189ed58a8402b64c73c8654206ce90c5e9c78b Mon Sep 17 00:00:00 2001 From: Jan Laupetin Date: Sat, 30 Aug 2025 13:19:54 +0200 Subject: [PATCH 4/4] chore: add define for not loading xchunks async --- .../Loading/Processor/ProcessorXChunks.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp index 0e7bf18b..71aed486 100644 --- a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp +++ b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp @@ -15,6 +15,8 @@ #include #include +#define XCHUNK_ASYNC 1 + namespace { class DbLoadStream @@ -44,17 +46,24 @@ namespace { if (inputSize > 0) { +#ifdef XCHUNK_ASYNC std::unique_lock lock(m_load_mutex); if (m_is_loading) { m_loading_finished.wait(lock); } +#endif m_input_size = inputSize; m_is_loading = true; + +#ifdef XCHUNK_ASYNC m_load_thread = std::thread(&DbLoadStream::Load, this); m_load_thread.detach(); +#else + Load(); +#endif } else { @@ -80,7 +89,9 @@ namespace private: void Load() { +#ifdef XCHUNK_ASYNC std::lock_guard lock(m_load_mutex); +#endif bool firstProcessor = true; @@ -102,7 +113,10 @@ namespace } m_is_loading = false; + +#ifdef XCHUNK_ASYNC m_loading_finished.notify_all(); +#endif } int m_index;