zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-09-06 00:37:26 +00:00
Code Activity

Merge pull request #500 from Laupetin/dump-console-ff-data

Browse Source 
feat: dump t6 xbox fastfile data
This commit is contained in:
Jan
2025-09-01 23:48:07 +02:00
committed by GitHub
parent facb95f1cb 07189ed58a
commit 30367ad721
27 changed files with 1341 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
premake5.lua
View File

@@ -95,6 +95,7 @@ include "thirdparty/catch2.lua"
include "thirdparty/eigen.lua" include "thirdparty/eigen.lua"
include "thirdparty/libtomcrypt.lua" include "thirdparty/libtomcrypt.lua"
include "thirdparty/libtommath.lua" include "thirdparty/libtommath.lua"
include "thirdparty/lzx.lua"
include "thirdparty/json.lua" include "thirdparty/json.lua"
include "thirdparty/minilzo.lua" include "thirdparty/minilzo.lua"
include "thirdparty/minizip.lua" include "thirdparty/minizip.lua"
@@ -108,6 +109,7 @@ group "ThirdParty"
eigen:project() eigen:project()
libtommath:project() libtommath:project()
libtomcrypt:project() libtomcrypt:project()
lzx:project()
json:project() json:project()
minilzo:project() minilzo:project()
minizip:project() minizip:project()

16
src/Common/Game/IGame.h
View File

@@ -16,6 +16,22 @@ enum class GameId
COUNT COUNT
}; };
// The full uppercase names are macros in the standard lib
// So unfortunately not usable as values in the enum
enum class GameEndianness
{
/* Little endian */
LE,
/* Big endian */
BE
};
enum class GameWordSize
{
ARCH_32,
ARCH_64
};
static constexpr const char* GameId_Names[]{ static constexpr const char* GameId_Names[]{
"IW3", "IW3",
"IW4", "IW4",

2
src/ZoneCommon.lua
View File

@@ -20,6 +20,7 @@ function ZoneCommon:link(links)
links:linkto(ObjCommon) links:linkto(ObjCommon)
links:linkto(Parser) links:linkto(Parser)
links:linkto(Utils) links:linkto(Utils)
links:linkto(lzx)
end end
function ZoneCommon:use() function ZoneCommon:use()
@@ -45,5 +46,6 @@ function ZoneCommon:project()
path.join(folder, "ZoneCommon/**.cpp") path.join(folder, "ZoneCommon/**.cpp")
} }
lzx:include(includes)
self:include(includes) self:include(includes)
end end

20
src/ZoneCommon/Game/T6/ZoneConstantsT6.h
View File

@@ -13,20 +13,24 @@ namespace T6
public: public:
static constexpr const char* MAGIC_SIGNED_TREYARCH = "TAff0100"; static constexpr const char* MAGIC_SIGNED_TREYARCH = "TAff0100";
static constexpr const char* MAGIC_SIGNED_LZX_TREYARCH = "TAffx100";
static constexpr const char* MAGIC_SIGNED_OAT = "ABff0100"; static constexpr const char* MAGIC_SIGNED_OAT = "ABff0100";
static constexpr const char* MAGIC_UNSIGNED = "TAffu100"; static constexpr const char* MAGIC_UNSIGNED = "TAffu100";
static constexpr const char* MAGIC_UNSIGNED_SERVER = "TAsvu100"; static constexpr const char* MAGIC_UNSIGNED_SERVER = "TAsvu100";
static_assert(std::char_traits<char>::length(MAGIC_SIGNED_TREYARCH) == sizeof(ZoneHeader::m_magic)); static_assert(std::char_traits<char>::length(MAGIC_SIGNED_TREYARCH) == sizeof(ZoneHeader::m_magic));
static_assert(std::char_traits<char>::length(MAGIC_SIGNED_LZX_TREYARCH) == sizeof(ZoneHeader::m_magic));
static_assert(std::char_traits<char>::length(MAGIC_SIGNED_OAT) == sizeof(ZoneHeader::m_magic)); static_assert(std::char_traits<char>::length(MAGIC_SIGNED_OAT) == sizeof(ZoneHeader::m_magic));
static_assert(std::char_traits<char>::length(MAGIC_UNSIGNED) == sizeof(ZoneHeader::m_magic)); static_assert(std::char_traits<char>::length(MAGIC_UNSIGNED) == sizeof(ZoneHeader::m_magic));
static_assert(std::char_traits<char>::length(MAGIC_UNSIGNED_SERVER) == sizeof(ZoneHeader::m_magic)); static_assert(std::char_traits<char>::length(MAGIC_UNSIGNED_SERVER) == sizeof(ZoneHeader::m_magic));
static constexpr int ZONE_VERSION = 147; static constexpr unsigned ZONE_VERSION_PC = 147;
static constexpr int STREAM_COUNT = 4; static constexpr unsigned ZONE_VERSION_XENON = 146;
static constexpr int XCHUNK_SIZE = 0x8000;
static constexpr int XCHUNK_MAX_WRITE_SIZE = XCHUNK_SIZE - 0x40; static constexpr unsigned STREAM_COUNT = 4;
static constexpr int VANILLA_BUFFER_SIZE = 0x80000; static constexpr unsigned XCHUNK_SIZE = 0x8000;
static constexpr unsigned XCHUNK_MAX_WRITE_SIZE = XCHUNK_SIZE - 0x40;
static constexpr unsigned VANILLA_BUFFER_SIZE = 0x80000;
static constexpr unsigned OFFSET_BLOCK_BIT_COUNT = 3u; static constexpr unsigned OFFSET_BLOCK_BIT_COUNT = 3u;
static constexpr block_t INSERT_BLOCK = XFILE_BLOCK_VIRTUAL; static constexpr block_t INSERT_BLOCK = XFILE_BLOCK_VIRTUAL;
@@ -34,10 +38,14 @@ namespace T6
static constexpr size_t FILE_SUFFIX_ZERO_ALIGN = 0x40; static constexpr size_t FILE_SUFFIX_ZERO_ALIGN = 0x40;
static constexpr const char* MAGIC_AUTH_HEADER = "PHEEBs71"; static constexpr const char* MAGIC_AUTH_HEADER = "PHEEBs71";
inline static const uint8_t SALSA20_KEY_TREYARCH[]{ inline static const uint8_t SALSA20_KEY_TREYARCH_PC[]{
0x64, 0x1D, 0x8A, 0x2F, 0xE3, 0x1D, 0x3A, 0xA6, 0x36, 0x22, 0xBB, 0xC9, 0xCE, 0x85, 0x87, 0x22, 0x64, 0x1D, 0x8A, 0x2F, 0xE3, 0x1D, 0x3A, 0xA6, 0x36, 0x22, 0xBB, 0xC9, 0xCE, 0x85, 0x87, 0x22,
0x9D, 0x42, 0xB0, 0xF8, 0xED, 0x9B, 0x92, 0x41, 0x30, 0xBF, 0x88, 0xB6, 0x5E, 0xDC, 0x50, 0xBE, 0x9D, 0x42, 0xB0, 0xF8, 0xED, 0x9B, 0x92, 0x41, 0x30, 0xBF, 0x88, 0xB6, 0x5E, 0xDC, 0x50, 0xBE,
}; };
inline static const uint8_t SALSA20_KEY_TREYARCH_XENON[]{
0x0E, 0x50, 0xF4, 0x9F, 0x41, 0x23, 0x17, 0x09, 0x60, 0x38, 0x66, 0x56, 0x22, 0xDD, 0x09, 0x13,
0x32, 0xA2, 0x09, 0xBA, 0x0A, 0x05, 0xA0, 0x0E, 0x13, 0x77, 0xCE, 0xDB, 0x0A, 0x3C, 0xB1, 0xD3,
};
inline static const uint8_t RSA_PUBLIC_KEY_TREYARCH[]{ inline static const uint8_t RSA_PUBLIC_KEY_TREYARCH[]{
0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, 0x00, 0xc7, 0x9d, 0x33, 0xe0, 0x75, 0xaf, 0xef, 0x08, 0x08, 0x2b, 0x89, 0xd9, 0x3b, 0xf3, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, 0x00, 0xc7, 0x9d, 0x33, 0xe0, 0x75, 0xaf, 0xef, 0x08, 0x08, 0x2b, 0x89, 0xd9, 0x3b, 0xf3,

35
src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.cpp
View File

@@ -1,25 +1,18 @@
#include "AbstractSalsa20Processor.h" #include "AbstractSalsa20Processor.h"
#include <cassert> #include <cassert>
#include <cstring>
AbstractSalsa20Processor::AbstractSalsa20Processor(const int streamCount, const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize) AbstractSalsa20Processor::AbstractSalsa20Processor(const unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize)
: m_stream_count(streamCount), : m_stream_count(streamCount),
m_stream_contexts(std::make_unique<StreamContext[]>(streamCount)), m_stream_contexts(streamCount),
m_stream_block_indices(std::make_unique<unsigned int[]>(streamCount)) m_block_hashes(BLOCK_HASHES_COUNT * streamCount * SHA1_HASH_SIZE),
m_stream_block_indices(streamCount)
{ {
m_block_hashes = std::make_unique<uint8_t[]>(BLOCK_HASHES_COUNT * streamCount * SHA1_HASH_SIZE);
InitStreams(zoneName, salsa20Key, keySize); InitStreams(zoneName, salsa20Key, keySize);
} }
uint8_t* AbstractSalsa20Processor::GetHashBlock(const int streamNumber) const void AbstractSalsa20Processor::InitStreams(const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize)
{
const auto blockIndexOffset = m_stream_block_indices[streamNumber] * m_stream_count * SHA1_HASH_SIZE;
const auto streamOffset = static_cast<size_t>(streamNumber) * SHA1_HASH_SIZE;
return &m_block_hashes[blockIndexOffset + streamOffset];
}
void AbstractSalsa20Processor::InitStreams(const std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize) const
{ {
// Original buffer must have been 32 bytes because the zoneName can at most be 31 characters be long before being cut off // Original buffer must have been 32 bytes because the zoneName can at most be 31 characters be long before being cut off
const auto zoneNameLength = std::min(zoneName.length(), 31uz); const auto zoneNameLength = std::min(zoneName.length(), 31uz);
@@ -29,14 +22,14 @@ void AbstractSalsa20Processor::InitStreams(const std::string& zoneName, const ui
assert(blockHashBufferSize % 4 == 0); assert(blockHashBufferSize % 4 == 0);
size_t zoneNameOffset = 0; size_t zoneNameOffset = 0;
for (size_t i = 0; i < blockHashBufferSize; i += 4) for (auto i = 0uz; i < blockHashBufferSize; i += 4)
{ {
*reinterpret_cast<uint32_t*>(&m_block_hashes[i]) = 0x1010101 * zoneName[zoneNameOffset++]; memset(&m_block_hashes[i], zoneName[zoneNameOffset++], 4u);
zoneNameOffset %= zoneNameLength; zoneNameOffset %= zoneNameLength;
} }
for (auto stream = 0; stream < m_stream_count; stream++) for (auto stream = 0u; stream < m_stream_count; stream++)
{ {
m_stream_block_indices[stream] = 0; m_stream_block_indices[stream] = 0;
@@ -45,11 +38,19 @@ void AbstractSalsa20Processor::InitStreams(const std::string& zoneName, const ui
} }
} }
uint8_t* AbstractSalsa20Processor::GetHashBlock(const unsigned streamNumber)
{
const auto blockIndexOffset = m_stream_block_indices[streamNumber] * m_stream_count * SHA1_HASH_SIZE;
const auto streamOffset = static_cast<size_t>(streamNumber) * SHA1_HASH_SIZE;
return &m_block_hashes[blockIndexOffset + streamOffset];
}
void AbstractSalsa20Processor::GetCapturedData(const uint8_t** pCapturedData, size_t* pSize) void AbstractSalsa20Processor::GetCapturedData(const uint8_t** pCapturedData, size_t* pSize)
{ {
assert(pCapturedData != nullptr); assert(pCapturedData != nullptr);
assert(pSize != nullptr); assert(pSize != nullptr);
*pCapturedData = m_block_hashes.get(); *pCapturedData = m_block_hashes.data();
*pSize = BLOCK_HASHES_COUNT * m_stream_count * SHA1_HASH_SIZE; *pSize = BLOCK_HASHES_COUNT * m_stream_count * SHA1_HASH_SIZE;
} }

51
src/ZoneCommon/Zone/XChunk/AbstractSalsa20Processor.h
View File

@@ -1,40 +1,22 @@
#pragma once #pragma once
#include "Cryptography.h" #include "Cryptography.h"
#include "Utils/ClassUtils.h"
#include "Utils/ICapturedDataProvider.h" #include "Utils/ICapturedDataProvider.h"
#include <cstdint> #include <cstdint>
#include <memory> #include <memory>
#include <string> #include <string>
#include <vector>
class Salsa20StreamContext
{
public:
std::unique_ptr<cryptography::IStreamCipher> m_salsa20;
std::unique_ptr<cryptography::IHashFunction> m_sha1;
};
class AbstractSalsa20Processor : public ICapturedDataProvider class AbstractSalsa20Processor : public ICapturedDataProvider
{ {
protected:
static constexpr int BLOCK_HASHES_COUNT = 200;
static constexpr int SHA1_HASH_SIZE = 20;
static constexpr int SALSA20_IV_SIZE = 8;
class StreamContext
{
public:
std::unique_ptr<cryptography::IStreamCipher> m_salsa20;
std::unique_ptr<cryptography::IHashFunction> m_sha1;
};
int m_stream_count;
std::unique_ptr<StreamContext[]> m_stream_contexts;
// m_block_hashes[BLOCK_HASHES_COUNT][numStreams][HASH_SIZE]
std::unique_ptr<uint8_t[]> m_block_hashes;
std::unique_ptr<unsigned int[]> m_stream_block_indices;
AbstractSalsa20Processor(int streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize);
_NODISCARD uint8_t* GetHashBlock(int streamNumber) const;
void InitStreams(const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize) const;
public: public:
virtual ~AbstractSalsa20Processor() = default; virtual ~AbstractSalsa20Processor() = default;
AbstractSalsa20Processor(const AbstractSalsa20Processor& other) = delete; AbstractSalsa20Processor(const AbstractSalsa20Processor& other) = delete;
@@ -43,4 +25,21 @@ public:
AbstractSalsa20Processor& operator=(AbstractSalsa20Processor&& other) noexcept = default; AbstractSalsa20Processor& operator=(AbstractSalsa20Processor&& other) noexcept = default;
void GetCapturedData(const uint8_t** pCapturedData, size_t* pSize) override; void GetCapturedData(const uint8_t** pCapturedData, size_t* pSize) override;
protected:
static constexpr auto BLOCK_HASHES_COUNT = 200u;
static constexpr auto SHA1_HASH_SIZE = 20u;
static constexpr auto SALSA20_IV_SIZE = 8u;
AbstractSalsa20Processor(unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize);
void InitStreams(const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize);
[[nodiscard]] uint8_t* GetHashBlock(unsigned streamNumber);
unsigned m_stream_count;
std::vector<Salsa20StreamContext> m_stream_contexts;
// m_block_hashes[BLOCK_HASHES_COUNT][numStreams][SHA1_HASH_SIZE]
std::vector<uint8_t> m_block_hashes;
std::vector<unsigned int> m_stream_block_indices;
}; };

8
src/ZoneCommon/Zone/XChunk/IXChunkProcessor.h
View File

@@ -6,6 +6,12 @@
class IXChunkProcessor class IXChunkProcessor
{ {
public: public:
IXChunkProcessor() = default;
virtual ~IXChunkProcessor() = default; virtual ~IXChunkProcessor() = default;
virtual size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) = 0; IXChunkProcessor(const IXChunkProcessor& other) = default;
IXChunkProcessor(IXChunkProcessor&& other) noexcept = default;
IXChunkProcessor& operator=(const IXChunkProcessor& other) = default;
IXChunkProcessor& operator=(IXChunkProcessor&& other) noexcept = default;
virtual size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) = 0;
}; };

2
src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp
View File

@@ -6,7 +6,7 @@
#include <zlib.h> #include <zlib.h>
#include <zutil.h> #include <zutil.h>
size_t XChunkProcessorDeflate::Process(int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) size_t XChunkProcessorDeflate::Process(unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
{ {
z_stream stream{}; z_stream stream{};
stream.zalloc = Z_NULL; stream.zalloc = Z_NULL;

2
src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.h
View File

@@ -4,5 +4,5 @@
class XChunkProcessorDeflate final : public IXChunkProcessor class XChunkProcessorDeflate final : public IXChunkProcessor
{ {
public: public:
size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
}; };

9
src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.cpp
View File

@@ -2,10 +2,12 @@
#include "XChunkException.h" #include "XChunkException.h"
#include <format>
#include <iostream>
#include <zlib.h> #include <zlib.h>
#include <zutil.h> #include <zutil.h>
size_t XChunkProcessorInflate::Process(int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) size_t XChunkProcessorInflate::Process(unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
{ {
z_stream stream{}; z_stream stream{};
stream.zalloc = Z_NULL; stream.zalloc = Z_NULL;
@@ -23,7 +25,10 @@ size_t XChunkProcessorInflate::Process(int streamNumber, const uint8_t* input, c
ret = inflate(&stream, Z_FULL_FLUSH); ret = inflate(&stream, Z_FULL_FLUSH);
if (ret != Z_STREAM_END) if (ret != Z_STREAM_END)
throw XChunkException("Zone has invalid or unsupported compression. Inflate failed"); {
std::cerr << std::format("inflate of stream failed with error code {}: {}\n", streamNumber, ret, stream.msg);
throw XChunkException(std::format("Zone has invalid or unsupported compression: {}", stream.msg));
}
const size_t outputSize = stream.total_out; const size_t outputSize = stream.total_out;

2
src/ZoneCommon/Zone/XChunk/XChunkProcessorInflate.h
View File

@@ -4,5 +4,5 @@
class XChunkProcessorInflate final : public IXChunkProcessor class XChunkProcessorInflate final : public IXChunkProcessor
{ {
public: public:
size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
}; };

132
src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp Normal file
View File

@@ -0,0 +1,132 @@
#include "XChunkProcessorLzxDecompress.h"
#include <cstring>
#include <format>
#include <iostream>
#include <lzx.h>
namespace
{
uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize)
{
const auto value = input[offset];
offset++;
remainingSize--;
return value;
}
uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte)
{
return static_cast<uint16_t>(static_cast<uint16_t>(static_cast<uint16_t>(highByte) << 8u) | static_cast<uint16_t>(lowByte));
}
void LogErrorHeaderSpace(size_t remainingInputSize)
{
std::cerr << std::format("XMemCompress: Not enough data for header: {}\n", remainingInputSize);
}
} // namespace
XChunkProcessorLzxDecompress::XChunkProcessorLzxDecompress(const unsigned streamCount)
: m_lzx_states(streamCount)
{
// T6 uses 17 for window bits
for (auto& lzxState : m_lzx_states)
lzxState = lzx_init(17);
}
XChunkProcessorLzxDecompress::~XChunkProcessorLzxDecompress()
{
for (auto* lzxState : m_lzx_states)
lzx_teardown(static_cast<lzx_state*>(lzxState));
}
size_t XChunkProcessorLzxDecompress::Process(
const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
{
auto* state = static_cast<lzx_state*>(m_lzx_states[streamNumber]);
// lzx state is reset before each chunk
lzx_reset(state);
size_t curInputOffset = 0uz;
size_t curInputSize = inputLength;
size_t curOutputOffset = 0uz;
size_t curOutputSize = outputBufferSize;
uint8_t lowByte;
uint16_t dstSize, srcSize;
while (curInputSize > 0)
{
uint8_t highByte = NextByte(input, curInputOffset, curInputSize);
uint8_t suffixSize;
if (highByte == 0xFF) // magic number: output is smaller than 0x8000
{
if (curInputSize < 4)
{
LogErrorHeaderSpace(curInputSize);
return curOutputOffset;
}
highByte = NextByte(input, curInputOffset, curInputSize);
lowByte = NextByte(input, curInputOffset, curInputSize);
dstSize = CombineHighLow(highByte, lowByte);
highByte = NextByte(input, curInputOffset, curInputSize);
lowByte = NextByte(input, curInputOffset, curInputSize);
srcSize = CombineHighLow(highByte, lowByte);
// The game seems to skip a 5 byte suffix after these blocks, not sure why.
suffixSize = 5u;
}
else
{
if (curInputSize < 1)
{
LogErrorHeaderSpace(curInputSize);
return curOutputOffset;
}
dstSize = 0x8000u;
lowByte = NextByte(input, curInputOffset, curInputSize);
srcSize = CombineHighLow(highByte, lowByte);
suffixSize = 0u;
}
if (srcSize == 0 || dstSize == 0)
{
// Other implementations do not handle this as a failure, game code suggests otherwise though
std::cerr << std::format("XMemCompress: EOF: {} {}, {}\n", srcSize, dstSize, curInputSize);
return curOutputOffset;
}
if (static_cast<size_t>(srcSize) + suffixSize > curInputSize)
{
std::cerr << std::format("XMemCompress: block size bigger than remaining data: {} > {}\n", srcSize, curInputSize);
return curOutputOffset;
}
if (dstSize > curOutputSize)
{
std::cerr << std::format("XMemCompress: output size bigger than remaining data: {} > {}\n", dstSize, curOutputSize);
return curOutputOffset;
}
auto ret = lzx_decompress(state, &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize);
curInputOffset += srcSize + suffixSize;
curInputSize -= (srcSize + suffixSize);
curOutputOffset += dstSize;
curOutputSize -= srcSize;
if (ret != DECR_OK)
{
std::cerr << std::format("XMemCompress: lzx decompression failed: {}\n", ret);
return curOutputOffset;
}
}
return curOutputOffset;
}

16
src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h Normal file
View File

@@ -0,0 +1,16 @@
#pragma once
#include "IXChunkProcessor.h"
#include <vector>
class XChunkProcessorLzxDecompress final : public IXChunkProcessor
{
public:
explicit XChunkProcessorLzxDecompress(unsigned streamCount);
~XChunkProcessorLzxDecompress();
size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
private:
std::vector<void*> m_lzx_states;
};

10
src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp
View File

@@ -4,9 +4,11 @@
#include "Cryptography.h" #include "Cryptography.h"
#include <cassert> #include <cassert>
#include <format>
#include <iostream>
XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const int streamCount, XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const unsigned streamCount,
std::string& zoneName, const std::string& zoneName,
const uint8_t* salsa20Key, const uint8_t* salsa20Key,
const size_t keySize) const size_t keySize)
: AbstractSalsa20Processor(streamCount, zoneName, salsa20Key, keySize) : AbstractSalsa20Processor(streamCount, zoneName, salsa20Key, keySize)
@@ -14,9 +16,9 @@ XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const int str
} }
size_t XChunkProcessorSalsa20Decryption::Process( size_t XChunkProcessorSalsa20Decryption::Process(
const int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
{ {
assert(streamNumber >= 0 && streamNumber < m_stream_count); assert(streamNumber < m_stream_count);
assert(input != nullptr); assert(input != nullptr);
assert(output != nullptr); assert(output != nullptr);
assert(inputLength <= outputBufferSize); assert(inputLength <= outputBufferSize);

4
src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.h
View File

@@ -7,7 +7,7 @@
class XChunkProcessorSalsa20Decryption final : public IXChunkProcessor, public AbstractSalsa20Processor class XChunkProcessorSalsa20Decryption final : public IXChunkProcessor, public AbstractSalsa20Processor
{ {
public: public:
XChunkProcessorSalsa20Decryption(int streamCount, std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); XChunkProcessorSalsa20Decryption(unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize);
size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
}; };

8
src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.cpp
View File

@@ -2,7 +2,7 @@
#include <cassert> #include <cassert>
XChunkProcessorSalsa20Encryption::XChunkProcessorSalsa20Encryption(const int streamCount, XChunkProcessorSalsa20Encryption::XChunkProcessorSalsa20Encryption(const unsigned streamCount,
const std::string& zoneName, const std::string& zoneName,
const uint8_t* salsa20Key, const uint8_t* salsa20Key,
const size_t keySize) const size_t keySize)
@@ -11,14 +11,14 @@ XChunkProcessorSalsa20Encryption::XChunkProcessorSalsa20Encryption(const int str
} }
size_t XChunkProcessorSalsa20Encryption::Process( size_t XChunkProcessorSalsa20Encryption::Process(
const int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
{ {
assert(streamNumber >= 0 && streamNumber < m_stream_count); assert(streamNumber < m_stream_count);
assert(input != nullptr); assert(input != nullptr);
assert(output != nullptr); assert(output != nullptr);
assert(inputLength <= outputBufferSize); assert(inputLength <= outputBufferSize);
auto& streamContext = m_stream_contexts[streamNumber]; const auto& streamContext = m_stream_contexts[streamNumber];
// Hash not yet encrypted XChunk // Hash not yet encrypted XChunk
uint8_t blockSha1Hash[SHA1_HASH_SIZE]; uint8_t blockSha1Hash[SHA1_HASH_SIZE];

4
src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Encryption.h
View File

@@ -8,7 +8,7 @@
class XChunkProcessorSalsa20Encryption final : public IXChunkProcessor, public AbstractSalsa20Processor class XChunkProcessorSalsa20Encryption final : public IXChunkProcessor, public AbstractSalsa20Processor
{ {
public: public:
XChunkProcessorSalsa20Encryption(int streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize); XChunkProcessorSalsa20Encryption(unsigned streamCount, const std::string& zoneName, const uint8_t* salsa20Key, size_t keySize);
size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
}; };

2
src/ZoneLoading.lua
View File

@@ -14,6 +14,7 @@ function ZoneLoading:link(links)
links:linkto(Cryptography) links:linkto(Cryptography)
links:linkto(Utils) links:linkto(Utils)
links:linkto(ZoneCommon) links:linkto(ZoneCommon)
links:linkto(lzx)
links:linkto(zlib) links:linkto(zlib)
if os.host() == "linux" then if os.host() == "linux" then
@@ -55,6 +56,7 @@ function ZoneLoading:project()
self:include(includes) self:include(includes)
Cryptography:include(includes) Cryptography:include(includes)
Utils:include(includes) Utils:include(includes)
lzx:include(includes)
zlib:include(includes) zlib:include(includes)
ZoneCode:include(includes) ZoneCode:include(includes)

185
src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp
View File

@@ -9,6 +9,7 @@
#include "Loading/Processor/ProcessorXChunks.h" #include "Loading/Processor/ProcessorXChunks.h"
#include "Loading/Steps/StepAddProcessor.h" #include "Loading/Steps/StepAddProcessor.h"
#include "Loading/Steps/StepAllocXBlocks.h" #include "Loading/Steps/StepAllocXBlocks.h"
#include "Loading/Steps/StepDumpData.h"
#include "Loading/Steps/StepLoadSignature.h" #include "Loading/Steps/StepLoadSignature.h"
#include "Loading/Steps/StepLoadZoneContent.h" #include "Loading/Steps/StepLoadZoneContent.h"
#include "Loading/Steps/StepLoadZoneSizes.h" #include "Loading/Steps/StepLoadZoneSizes.h"
@@ -16,16 +17,21 @@
#include "Loading/Steps/StepVerifyFileName.h" #include "Loading/Steps/StepVerifyFileName.h"
#include "Loading/Steps/StepVerifyMagic.h" #include "Loading/Steps/StepVerifyMagic.h"
#include "Loading/Steps/StepVerifySignature.h" #include "Loading/Steps/StepVerifySignature.h"
#include "Utils/ClassUtils.h" #include "Utils/Endianness.h"
#include "Zone/XChunk/XChunkProcessorInflate.h" #include "Zone/XChunk/XChunkProcessorInflate.h"
#include "Zone/XChunk/XChunkProcessorLzxDecompress.h"
#include "Zone/XChunk/XChunkProcessorSalsa20Decryption.h" #include "Zone/XChunk/XChunkProcessorSalsa20Decryption.h"
#include <cassert> #include <cassert>
#include <cstdio>
#include <cstring> #include <cstring>
#include <filesystem>
#include <format>
#include <iostream> #include <iostream>
#include <memory> #include <memory>
using namespace T6; using namespace T6;
namespace fs = std::filesystem;
namespace namespace
{ {
@@ -44,46 +50,63 @@ namespace
return GameLanguage::LANGUAGE_NONE; return GameLanguage::LANGUAGE_NONE;
} }
bool CanLoad(const ZoneHeader& header, bool* isSecure, bool* isOfficial, bool* isEncrypted) bool CanLoad(const ZoneHeader& header, bool& isBigEndian, bool& isSecure, bool& isOfficial, bool& isEncrypted, bool& isLzxCompressed)
{ {
assert(isSecure != nullptr); if (endianness::FromLittleEndian(header.m_version) == ZoneConstants::ZONE_VERSION_PC)
assert(isOfficial != nullptr);
if (header.m_version != ZoneConstants::ZONE_VERSION)
{ {
return false; isBigEndian = false;
isLzxCompressed = false;
if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_TREYARCH, 8))
{
isSecure = true;
isOfficial = true;
isEncrypted = true;
return true;
}
if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_OAT, 8))
{
isSecure = true;
isOfficial = false;
isEncrypted = true;
return true;
}
if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED, 8))
{
isSecure = false;
isOfficial = true;
isEncrypted = true;
return true;
}
if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED_SERVER, 8))
{
isSecure = false;
isOfficial = true;
isEncrypted = false;
return true;
}
} }
else if (endianness::FromBigEndian(header.m_version) == ZoneConstants::ZONE_VERSION_XENON)
if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_TREYARCH, 8))
{ {
*isSecure = true; isBigEndian = true;
*isOfficial = true; if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_TREYARCH, 8))
*isEncrypted = true; {
return true; isSecure = true;
} isOfficial = true;
isEncrypted = true;
if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_OAT, 8)) isLzxCompressed = false;
{ return true;
*isSecure = true; }
*isOfficial = false; if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_LZX_TREYARCH, 8))
*isEncrypted = true; {
return true; isSecure = true;
} isOfficial = true;
isEncrypted = true;
if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED, 8)) isLzxCompressed = true;
{ return true;
*isSecure = false; }
*isOfficial = true;
*isEncrypted = true;
return true;
}
if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED_SERVER, 8))
{
*isSecure = false;
*isOfficial = true;
*isEncrypted = false;
return true;
} }
return false; return false;
@@ -145,22 +168,45 @@ namespace
return signatureLoadStepPtr; return signatureLoadStepPtr;
} }
ICapturedDataProvider* AddXChunkProcessor(const bool isEncrypted, ZoneLoader& zoneLoader, std::string& fileName) ICapturedDataProvider*
AddXChunkProcessor(const bool isBigEndian, const bool isEncrypted, const bool isLzxCompressed, ZoneLoader& zoneLoader, std::string& fileName)
{ {
ICapturedDataProvider* result = nullptr; ICapturedDataProvider* result = nullptr;
auto xChunkProcessor = processor::CreateProcessorXChunks(ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, ZoneConstants::VANILLA_BUFFER_SIZE); std::unique_ptr<processor::IProcessorXChunks> xChunkProcessor;
if (isBigEndian)
{
xChunkProcessor = processor::CreateProcessorXChunks(
ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, GameEndianness::BE, ZoneConstants::VANILLA_BUFFER_SIZE);
}
else
{
xChunkProcessor = processor::CreateProcessorXChunks(
ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, GameEndianness::LE, ZoneConstants::VANILLA_BUFFER_SIZE);
}
const uint8_t (&salsa20Key)[32] = isBigEndian ? ZoneConstants::SALSA20_KEY_TREYARCH_XENON : ZoneConstants::SALSA20_KEY_TREYARCH_PC;
if (isEncrypted) if (isEncrypted)
{ {
// If zone is encrypted, the decryption is applied before the decompression. T6 Zones always use Salsa20. // If zone is encrypted, the decryption is applied before the decompression. T6 Zones always use Salsa20.
auto chunkProcessorSalsa20 = std::make_unique<XChunkProcessorSalsa20Decryption>( auto chunkProcessorSalsa20 =
ZoneConstants::STREAM_COUNT, fileName, ZoneConstants::SALSA20_KEY_TREYARCH, sizeof(ZoneConstants::SALSA20_KEY_TREYARCH)); std::make_unique<XChunkProcessorSalsa20Decryption>(ZoneConstants::STREAM_COUNT, fileName, salsa20Key, sizeof(salsa20Key));
result = chunkProcessorSalsa20.get(); result = chunkProcessorSalsa20.get();
xChunkProcessor->AddChunkProcessor(std::move(chunkProcessorSalsa20)); xChunkProcessor->AddChunkProcessor(std::move(chunkProcessorSalsa20));
} }
// Decompress the chunks using zlib if (isLzxCompressed)
xChunkProcessor->AddChunkProcessor(std::make_unique<XChunkProcessorInflate>()); {
// Decompress the chunks using lzx
xChunkProcessor->AddChunkProcessor(std::make_unique<XChunkProcessorLzxDecompress>(ZoneConstants::STREAM_COUNT));
}
else
{
// Decompress the chunks using zlib
xChunkProcessor->AddChunkProcessor(std::make_unique<XChunkProcessorInflate>());
}
zoneLoader.AddLoadingStep(step::CreateStepAddProcessor(std::move(xChunkProcessor))); zoneLoader.AddLoadingStep(step::CreateStepAddProcessor(std::move(xChunkProcessor)));
// If there is encryption, the signed data of the zone is the final hash blocks provided by the Salsa20 IV adaption algorithm // If there is encryption, the signed data of the zone is the final hash blocks provided by the Salsa20 IV adaption algorithm
@@ -170,12 +216,10 @@ namespace
std::unique_ptr<ZoneLoader> ZoneLoaderFactory::CreateLoaderForHeader(ZoneHeader& header, std::string& fileName) const std::unique_ptr<ZoneLoader> ZoneLoaderFactory::CreateLoaderForHeader(ZoneHeader& header, std::string& fileName) const
{ {
bool isSecure; bool isBigEndian, isSecure, isOfficial, isEncrypted, isLzxCompressed;
bool isOfficial;
bool isEncrypted;
// Check if this file is a supported T6 zone. // Check if this file is a supported T6 zone.
if (!CanLoad(header, &isSecure, &isOfficial, &isEncrypted)) if (!CanLoad(header, isBigEndian, isSecure, isOfficial, isEncrypted, isLzxCompressed))
return nullptr; return nullptr;
// Create new zone // Create new zone
@@ -196,26 +240,37 @@ std::unique_ptr<ZoneLoader> ZoneLoaderFactory::CreateLoaderForHeader(ZoneHeader&
ISignatureProvider* signatureProvider = AddAuthHeaderSteps(isSecure, *zoneLoader, fileName); ISignatureProvider* signatureProvider = AddAuthHeaderSteps(isSecure, *zoneLoader, fileName);
// Setup loading XChunks from the zone from this point on. // Setup loading XChunks from the zone from this point on.
ICapturedDataProvider* signatureDataProvider = AddXChunkProcessor(isEncrypted, *zoneLoader, fileName); ICapturedDataProvider* signatureDataProvider = AddXChunkProcessor(isBigEndian, isEncrypted, isLzxCompressed, *zoneLoader, fileName);
// Start of the XFile struct if (!isBigEndian)
zoneLoader->AddLoadingStep(step::CreateStepLoadZoneSizes());
zoneLoader->AddLoadingStep(step::CreateStepAllocXBlocks());
// Start of the zone content
zoneLoader->AddLoadingStep(step::CreateStepLoadZoneContent(
[zonePtr](ZoneInputStream& stream)
{
return std::make_unique<ContentLoader>(*zonePtr, stream);
},
32u,
ZoneConstants::OFFSET_BLOCK_BIT_COUNT,
ZoneConstants::INSERT_BLOCK,
zonePtr->Memory()));
if (isSecure)
{ {
zoneLoader->AddLoadingStep(step::CreateStepVerifySignature(std::move(rsa), signatureProvider, signatureDataProvider)); // Start of the XFile struct
zoneLoader->AddLoadingStep(step::CreateStepLoadZoneSizes());
zoneLoader->AddLoadingStep(step::CreateStepAllocXBlocks());
// Start of the zone content
zoneLoader->AddLoadingStep(step::CreateStepLoadZoneContent(
[zonePtr](ZoneInputStream& stream)
{
return std::make_unique<ContentLoader>(*zonePtr, stream);
},
32u,
ZoneConstants::OFFSET_BLOCK_BIT_COUNT,
ZoneConstants::INSERT_BLOCK,
zonePtr->Memory()));
if (isSecure)
{
zoneLoader->AddLoadingStep(step::CreateStepVerifySignature(std::move(rsa), signatureProvider, signatureDataProvider));
}
}
else
{
fs::path dumpFileNamePath = fs::path(fileName).filename();
dumpFileNamePath.replace_extension(".dat");
std::string dumpFileName = dumpFileNamePath.string();
std::cerr << std::format("Dumping xbox assets is not supported, making a full fastfile data dump to {}\n", dumpFileName);
zoneLoader->AddLoadingStep(step::CreateStepDumpData(dumpFileName, 0xFFFFFFFF));
} }
return zoneLoader; return zoneLoader;

36
src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp
View File

@@ -1,17 +1,22 @@
#include "ProcessorXChunks.h" #include "ProcessorXChunks.h"
#include "Loading/Exception/InvalidChunkSizeException.h" #include "Loading/Exception/InvalidChunkSizeException.h"
#include "Utils/Endianness.h"
#include "Zone/ZoneTypes.h" #include "Zone/ZoneTypes.h"
#include <cassert> #include <cassert>
#include <condition_variable> #include <condition_variable>
#include <cstring> #include <cstring>
#include <format>
#include <iostream>
#include <memory> #include <memory>
#include <mutex> #include <mutex>
#include <optional> #include <optional>
#include <thread> #include <thread>
#include <vector> #include <vector>
#define XCHUNK_ASYNC 1
namespace namespace
{ {
class DbLoadStream class DbLoadStream
@@ -41,17 +46,24 @@ namespace
{ {
if (inputSize > 0) if (inputSize > 0)
{ {
#ifdef XCHUNK_ASYNC
std::unique_lock lock(m_load_mutex); std::unique_lock lock(m_load_mutex);
if (m_is_loading) if (m_is_loading)
{ {
m_loading_finished.wait(lock); m_loading_finished.wait(lock);
} }
#endif
m_input_size = inputSize; m_input_size = inputSize;
m_is_loading = true; m_is_loading = true;
#ifdef XCHUNK_ASYNC
m_load_thread = std::thread(&DbLoadStream::Load, this); m_load_thread = std::thread(&DbLoadStream::Load, this);
m_load_thread.detach(); m_load_thread.detach();
#else
Load();
#endif
} }
else else
{ {
@@ -77,7 +89,9 @@ namespace
private: private:
void Load() void Load()
{ {
#ifdef XCHUNK_ASYNC
std::lock_guard lock(m_load_mutex); std::lock_guard lock(m_load_mutex);
#endif
bool firstProcessor = true; bool firstProcessor = true;
@@ -99,7 +113,10 @@ namespace
} }
m_is_loading = false; m_is_loading = false;
#ifdef XCHUNK_ASYNC
m_loading_finished.notify_all(); m_loading_finished.notify_all();
#endif
} }
int m_index; int m_index;
@@ -125,8 +142,9 @@ namespace
class ProcessorXChunks final : public processor::IProcessorXChunks class ProcessorXChunks final : public processor::IProcessorXChunks
{ {
public: public:
ProcessorXChunks(const int numStreams, const size_t xChunkSize, const std::optional<size_t> vanillaBufferSize) ProcessorXChunks(const int numStreams, const size_t xChunkSize, const GameEndianness endianness, const std::optional<size_t> vanillaBufferSize)
: m_chunk_size(xChunkSize), : m_chunk_size(xChunkSize),
m_endianness(endianness),
m_vanilla_buffer_size(vanillaBufferSize), m_vanilla_buffer_size(vanillaBufferSize),
m_initialized_streams(false), m_initialized_streams(false),
m_current_stream(0), m_current_stream(0),
@@ -219,8 +237,12 @@ namespace
} }
const size_t readSize = m_base_stream->Load(&chunkSize, sizeof(chunkSize)); const size_t readSize = m_base_stream->Load(&chunkSize, sizeof(chunkSize));
if (m_endianness == GameEndianness::LE)
chunkSize = endianness::FromLittleEndian(chunkSize);
else
chunkSize = endianness::FromBigEndian(chunkSize);
if (readSize == 0) if (readSize < sizeof(chunkSize) || chunkSize == 0)
{ {
m_eof_reached = true; m_eof_reached = true;
m_eof_stream = streamNum; m_eof_stream = streamNum;
@@ -280,6 +302,7 @@ namespace
std::vector<std::unique_ptr<DbLoadStream>> m_streams; std::vector<std::unique_ptr<DbLoadStream>> m_streams;
size_t m_chunk_size; size_t m_chunk_size;
GameEndianness m_endianness;
std::optional<size_t> m_vanilla_buffer_size; std::optional<size_t> m_vanilla_buffer_size;
std::vector<std::unique_ptr<IXChunkProcessor>> m_chunk_processors; std::vector<std::unique_ptr<IXChunkProcessor>> m_chunk_processors;
@@ -297,13 +320,14 @@ namespace
namespace processor namespace processor
{ {
std::unique_ptr<IProcessorXChunks> CreateProcessorXChunks(int numStreams, const size_t xChunkSize) std::unique_ptr<IProcessorXChunks> CreateProcessorXChunks(unsigned numStreams, const size_t xChunkSize, const GameEndianness endianness)
{ {
return std::make_unique<ProcessorXChunks>(numStreams, xChunkSize, std::nullopt); return std::make_unique<ProcessorXChunks>(numStreams, xChunkSize, endianness, std::nullopt);
} }
std::unique_ptr<IProcessorXChunks> CreateProcessorXChunks(int numStreams, const size_t xChunkSize, const size_t vanillaBufferSize) std::unique_ptr<IProcessorXChunks>
CreateProcessorXChunks(unsigned numStreams, const size_t xChunkSize, GameEndianness endianness, const size_t vanillaBufferSize)
{ {
return std::make_unique<ProcessorXChunks>(numStreams, xChunkSize, vanillaBufferSize); return std::make_unique<ProcessorXChunks>(numStreams, xChunkSize, endianness, vanillaBufferSize);
} }
} // namespace processor } // namespace processor

6
src/ZoneLoading/Loading/Processor/ProcessorXChunks.h
View File

@@ -1,4 +1,6 @@
#pragma once #pragma once
#include "Game/IGame.h"
#include "Loading/StreamProcessor.h" #include "Loading/StreamProcessor.h"
#include "Zone/XChunk/IXChunkProcessor.h" #include "Zone/XChunk/IXChunkProcessor.h"
@@ -12,6 +14,6 @@ namespace processor
virtual void AddChunkProcessor(std::unique_ptr<IXChunkProcessor> chunkProcessor) = 0; virtual void AddChunkProcessor(std::unique_ptr<IXChunkProcessor> chunkProcessor) = 0;
}; };
std::unique_ptr<IProcessorXChunks> CreateProcessorXChunks(int numStreams, size_t xChunkSize); std::unique_ptr<IProcessorXChunks> CreateProcessorXChunks(unsigned numStreams, size_t xChunkSize, GameEndianness endianness);
std::unique_ptr<IProcessorXChunks> CreateProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize); std::unique_ptr<IProcessorXChunks> CreateProcessorXChunks(unsigned numStreams, size_t xChunkSize, GameEndianness endianness, size_t vanillaBufferSize);
} // namespace processor } // namespace processor

14
src/ZoneLoading/Loading/Steps/StepDumpData.cpp
View File

@@ -7,17 +7,18 @@ namespace
class StepDumpData final : public ILoadingStep class StepDumpData final : public ILoadingStep
{ {
public: public:
explicit StepDumpData(const size_t dumpCount) StepDumpData(std::string fileName, const size_t dumpCount)
: m_dump_count(dumpCount) : m_file_name(std::move(fileName)),
m_dump_count(dumpCount)
{ {
} }
void PerformStep(ZoneLoader& zoneLoader, ILoadingStream& stream) override void PerformStep(ZoneLoader& zoneLoader, ILoadingStream& stream) override
{ {
uint8_t tempBuffer[128]; uint8_t tempBuffer[0x1000];
auto dumpedBytes = 0uz; auto dumpedBytes = 0uz;
std::ofstream tempFile("dump.dat", std::fstream::out | std::fstream::binary); std::ofstream tempFile(m_file_name, std::fstream::out | std::fstream::binary);
while (dumpedBytes < m_dump_count) while (dumpedBytes < m_dump_count)
{ {
@@ -45,14 +46,15 @@ namespace
} }
private: private:
std::string m_file_name;
size_t m_dump_count; size_t m_dump_count;
}; };
} // namespace } // namespace
namespace step namespace step
{ {
std::unique_ptr<ILoadingStep> CreateStepDumpData(size_t dumpCount) std::unique_ptr<ILoadingStep> CreateStepDumpData(std::string fileName, size_t dumpCount)
{ {
return std::make_unique<StepDumpData>(dumpCount); return std::make_unique<StepDumpData>(std::move(fileName), dumpCount);
} }
} // namespace step } // namespace step

3
src/ZoneLoading/Loading/Steps/StepDumpData.h
View File

@@ -3,8 +3,9 @@
#include "Loading/ILoadingStep.h" #include "Loading/ILoadingStep.h"
#include <memory> #include <memory>
#include <string>
namespace step namespace step
{ {
std::unique_ptr<ILoadingStep> CreateStepDumpData(size_t dumpCount); std::unique_ptr<ILoadingStep> CreateStepDumpData(std::string fileName, size_t dumpCount);
} }

4
src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp
View File

@@ -44,7 +44,7 @@ namespace
ZoneHeader CreateHeaderForParams(const bool isSecure, const bool isOfficial, const bool isEncrypted) ZoneHeader CreateHeaderForParams(const bool isSecure, const bool isOfficial, const bool isEncrypted)
{ {
ZoneHeader header{}; ZoneHeader header{};
header.m_version = ZoneConstants::ZONE_VERSION; header.m_version = ZoneConstants::ZONE_VERSION_PC;
if (isSecure) if (isSecure)
{ {
@@ -82,7 +82,7 @@ namespace
{ {
// If zone is encrypted, the decryption is applied before the decompression. T6 Zones always use Salsa20. // If zone is encrypted, the decryption is applied before the decompression. T6 Zones always use Salsa20.
auto chunkProcessorSalsa20 = std::make_unique<XChunkProcessorSalsa20Encryption>( auto chunkProcessorSalsa20 = std::make_unique<XChunkProcessorSalsa20Encryption>(
ZoneConstants::STREAM_COUNT, zone.m_name, ZoneConstants::SALSA20_KEY_TREYARCH, sizeof(ZoneConstants::SALSA20_KEY_TREYARCH)); ZoneConstants::STREAM_COUNT, zone.m_name, ZoneConstants::SALSA20_KEY_TREYARCH_PC, sizeof(ZoneConstants::SALSA20_KEY_TREYARCH_PC));
// If there is encryption, the signed data of the zone is the final hash blocks provided by the Salsa20 IV adaption algorithm // If there is encryption, the signed data of the zone is the final hash blocks provided by the Salsa20 IV adaption algorithm
if (dataToSignProviderPtr) if (dataToSignProviderPtr)

48
thirdparty/lzx.lua vendored Normal file
View File

@@ -0,0 +1,48 @@
lzx = {}
function lzx:include(includes)
if includes:handle(self:name()) then
includedirs {
path.join(ThirdPartyFolder(), "lzx")
}
end
end
function lzx:link(links)
links:add(self:name())
end
function lzx:use()
end
function lzx:name()
return "lzx"
end
function lzx:project()
local folder = ThirdPartyFolder()
local includes = Includes:create()
project(self:name())
targetdir(TargetDirectoryLib)
location "%{wks.location}/thirdparty/%{prj.name}"
kind "StaticLib"
language "C"
files {
path.join(folder, "lzx/*.h"),
path.join(folder, "lzx/*.c")
}
defines {
"_CRT_SECURE_NO_WARNINGS",
"_CRT_NONSTDC_NO_DEPRECATE"
}
self:include(includes)
-- Disable warnings. They do not have any value to us since it is not our code.
warnings "off"
end

808
thirdparty/lzx/lzx.c vendored Normal file
View File

@@ -0,0 +1,808 @@
// Modified version of the code from Wine:
// https://gitlab.winehq.org/wine/wine/-/blob/fcc40a07909dc7626b6d1e2ec73f823d828a47e8/dlls/itss/lzx.c
/***************************************************************************
* lzx.c - LZX decompression routines *
* ------------------- *
* *
* maintainer: Jed Wing <jedwin@ugcs.caltech.edu> *
* source: modified lzx.c from cabextract v0.5 *
* notes: This file was taken from cabextract v0.5, which was, *
* itself, a modified version of the lzx decompression code *
* from unlzx. *
* *
* platforms: In its current incarnation, this file has been tested on *
* two different Linux platforms (one, redhat-based, with a *
* 2.1.2 glibc and gcc 2.95.x, and the other, Debian, with *
* 2.2.4 glibc and both gcc 2.95.4 and gcc 3.0.2). Both were *
* Intel x86 compatible machines. *
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. Note that an exemption to this *
* license has been granted by Stuart Caie for the purposes of *
* distribution with chmlib. This does not, to the best of my *
* knowledge, constitute a change in the license of this (the LZX) code *
* in general. *
* *
***************************************************************************/
#include "lzx.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
/* some constants defined by the LZX specification */
#define LZX_MIN_MATCH 2
/* #define LZX_MAX_MATCH 257 */
#define LZX_NUM_CHARS 256
#define LZX_BLOCKTYPE_INVALID 0 /* also blocktypes 4-7 invalid */
#define LZX_BLOCKTYPE_VERBATIM 1
#define LZX_BLOCKTYPE_ALIGNED 2
#define LZX_BLOCKTYPE_UNCOMPRESSED 3
#define LZX_PRETREE_NUM_ELEMENTS 20
#define LZX_ALIGNED_NUM_ELEMENTS 8 /* aligned offset tree #elements */
#define LZX_NUM_PRIMARY_LENGTHS 7 /* this one missing from spec! */
#define LZX_NUM_SECONDARY_LENGTHS 249 /* length tree #elements */
/* LZX huffman defines: tweak tablebits as desired */
#define LZX_PRETREE_MAXSYMBOLS LZX_PRETREE_NUM_ELEMENTS
#define LZX_PRETREE_TABLEBITS 6
#define LZX_MAINTREE_MAXSYMBOLS (LZX_NUM_CHARS + 50 * 8)
#define LZX_MAINTREE_TABLEBITS 12
#define LZX_LENGTH_MAXSYMBOLS (LZX_NUM_SECONDARY_LENGTHS + 1)
#define LZX_LENGTH_TABLEBITS 12
#define LZX_ALIGNED_MAXSYMBOLS LZX_ALIGNED_NUM_ELEMENTS
#define LZX_ALIGNED_TABLEBITS 7
#define LZX_LENTABLE_SAFETY 64 /* we allow length table decoding overruns */
#define LZX_DECLARE_TABLE(tbl) \
uint16_t tbl## _table[(1 << LZX_## tbl## _TABLEBITS) + (LZX_## tbl## _MAXSYMBOLS << 1)]; \
uint8_t tbl## _len[LZX_## tbl## _MAXSYMBOLS + LZX_LENTABLE_SAFETY]
struct lzx_state {
uint8_t* window; /* the actual decoding window */
uint32_t window_size; /* window size (32Kb through 2Mb) */
uint32_t actual_size; /* window size when it was first allocated */
uint32_t window_posn; /* current offset within the window */
uint32_t R0, R1, R2; /* for the LRU offset system */
uint16_t main_elements; /* number of main tree elements */
int header_read; /* have we started decoding at all yet? */
uint16_t block_type; /* type of this block */
uint32_t block_length; /* uncompressed length of this block */
uint32_t block_remaining; /* uncompressed bytes still left to decode */
uint32_t frames_read; /* the number of CFDATA blocks processed */
int32_t intel_filesize; /* magic header value used for transform */
int32_t intel_curpos; /* current offset in transform space */
int intel_started; /* have we seen any translatable data yet? */
LZX_DECLARE_TABLE(PRETREE);
LZX_DECLARE_TABLE(MAINTREE);
LZX_DECLARE_TABLE(LENGTH);
LZX_DECLARE_TABLE(ALIGNED);
};
/* LZX decruncher */
/* Microsoft's LZX document and their implementation of the
* com.ms.util.cab Java package do not concur.
*
* In the LZX document, there is a table showing the correlation between
* window size and the number of position slots. It states that the 1MB
* window = 40 slots and the 2MB window = 42 slots. In the implementation,
* 1MB = 42 slots, 2MB = 50 slots. The actual calculation is 'find the
* first slot whose position base is equal to or more than the required
* window size'. This would explain why other tables in the document refer
* to 50 slots rather than 42.
*
* The constant NUM_PRIMARY_LENGTHS used in the decompression pseudocode
* is not defined in the specification.
*
* The LZX document does not state the uncompressed block has an
* uncompressed length field. Where does this length field come from, so
* we can know how large the block is? The implementation has it as the 24
* bits following after the 3 blocktype bits, before the alignment
* padding.
*
* The LZX document states that aligned offset blocks have their aligned
* offset huffman tree AFTER the main and length trees. The implementation
* suggests that the aligned offset tree is BEFORE the main and length
* trees.
*
* The LZX document decoding algorithm states that, in an aligned offset
* block, if an extra_bits value is 1, 2 or 3, then that number of bits
* should be read and the result added to the match offset. This is
* correct for 1 and 2, but not 3, where just a huffman symbol (using the
* aligned tree) should be read.
*
* Regarding the E8 preprocessing, the LZX document states 'No translation
* may be performed on the last 6 bytes of the input block'. This is
* correct. However, the pseudocode provided checks for the *E8 leader*
* up to the last 6 bytes. If the leader appears between -10 and -7 bytes
* from the end, this would cause the next four bytes to be modified, at
* least one of which would be in the last 6 bytes, which is not allowed
* according to the spec.
*
* The specification states that the huffman trees must always contain at
* least one element. However, many CAB files contain blocks where the
* length tree is completely empty (because there are no matches), and
* this is expected to succeed.
*/
/* LZX uses what it calls 'position slots' to represent match offsets.
* What this means is that a small 'position slot' number and a small
* offset from that slot are encoded instead of one large offset for
* every match.
* - position_base is an index to the position slot bases
* - extra_bits states how many bits of offset-from-base data is needed.
*/
static const uint8_t extra_bits[51] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5,
5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11,
12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17};
static const uint32_t position_base[51] = {
0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32,
48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536,
2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536,
98304, 131072, 196608, 262144, 393216, 524288, 655360, 786432, 917504, 1048576, 1179648,
1310720, 1441792, 1572864, 1703936, 1835008, 1966080, 2097152};
struct lzx_state* lzx_init(int window) {
struct lzx_state* pState = NULL;
uint32_t wndsize = 1 << window;
int i, posn_slots;
/* LZX supports window sizes of 2^15 (32Kb) through 2^21 (2Mb) */
/* if a previously allocated window is big enough, keep it */
if (window < 15 || window > 21)
return NULL;
/* allocate state and associated window */
pState = (struct lzx_state*)malloc(sizeof(struct lzx_state));
if (!pState || !(pState->window = (uint8_t*)malloc(wndsize))) {
free(pState);
return NULL;
}
pState->actual_size = wndsize;
pState->window_size = wndsize;
/* calculate required position slots */
if (window == 20)
posn_slots = 42;
else if (window == 21)
posn_slots = 50;
else
posn_slots = window << 1;
/** alternatively **/
/* posn_slots=i=0; while (i < wndsize) i += 1 << extra_bits[posn_slots++]; */
/* initialize other state */
pState->R0 = pState->R1 = pState->R2 = 1;
pState->main_elements = LZX_NUM_CHARS + (posn_slots << 3);
pState->header_read = 0;
pState->frames_read = 0;
pState->block_remaining = 0;
pState->block_type = LZX_BLOCKTYPE_INVALID;
pState->intel_curpos = 0;
pState->intel_started = 0;
pState->window_posn = 0;
/* initialise tables to 0 (because deltas will be applied to them) */
for (i = 0; i < LZX_MAINTREE_MAXSYMBOLS; i++)
pState->MAINTREE_len[i] = 0;
for (i = 0; i < LZX_LENGTH_MAXSYMBOLS; i++)
pState->LENGTH_len[i] = 0;
return pState;
}
void lzx_teardown(struct lzx_state* pState) {
if (pState) {
if (pState->window)
free(pState->window);
free(pState);
}
}
void lzx_reset(struct lzx_state* pState) {
int i;
pState->R0 = pState->R1 = pState->R2 = 1;
pState->header_read = 0;
pState->frames_read = 0;
pState->block_remaining = 0;
pState->block_type = LZX_BLOCKTYPE_INVALID;
pState->intel_curpos = 0;
pState->intel_started = 0;
pState->window_posn = 0;
for (i = 0; i < LZX_MAINTREE_MAXSYMBOLS + LZX_LENTABLE_SAFETY; i++) {
pState->MAINTREE_len[i] = 0;
}
for (i = 0; i < LZX_LENGTH_MAXSYMBOLS + LZX_LENTABLE_SAFETY; i++) {
pState->LENGTH_len[i] = 0;
}
}
/* Bitstream reading macros:
*
* INIT_BITSTREAM should be used first to set up the system
* READ_BITS(var,n) takes N bits from the buffer and puts them in var
*
* ENSURE_BITS(n) ensures there are at least N bits in the bit buffer
* PEEK_BITS(n) extracts (without removing) N bits from the bit buffer
* REMOVE_BITS(n) removes N bits from the bit buffer
*
* These bit access routines work by using the area beyond the MSB and the
* LSB as a free source of zeroes. This avoids having to mask any bits.
* So we have to know the bit width of the bitbuffer variable. This is
* sizeof(uint32_t) * 8, also defined as uint32_t_BITS
*/
/* number of bits in uint32_t. Note: This must be at multiple of 16, and at
* least 32 for the bitbuffer code to work (ie, it must be able to ensure
* up to 17 bits - that's adding 16 bits when there's one bit left, or
* adding 32 bits when there are no bits left. The code should work fine
* for machines where uint32_t >= 32 bits.
*/
#define uint32_t_BITS (sizeof(uint32_t) << 3)
#define INIT_BITSTREAM \
do { \
bitsleft = 0; \
bitbuf = 0; \
} while (0)
#define ENSURE_BITS(n) \
while (bitsleft < (n)) { \
bitbuf |= ((inpos[1] << 8) | inpos[0]) << (uint32_t_BITS - 16 - bitsleft); \
bitsleft += 16; \
inpos += 2; \
}
#define PEEK_BITS(n) (bitbuf >> (uint32_t_BITS - (n)))
#define REMOVE_BITS(n) ((bitbuf <<= (n)), (bitsleft -= (n)))
#define READ_BITS(v, n) \
do { \
ENSURE_BITS(n); \
(v) = PEEK_BITS(n); \
REMOVE_BITS(n); \
} while (0)
/* Huffman macros */
#define TABLEBITS(tbl) (LZX_## tbl## _TABLEBITS)
#define MAXSYMBOLS(tbl) (LZX_## tbl## _MAXSYMBOLS)
#define SYMTABLE(tbl) (pState->tbl## _table)
#define LENTABLE(tbl) (pState->tbl## _len)
/* BUILD_TABLE(tablename) builds a huffman lookup table from code lengths.
* In reality, it just calls make_decode_table() with the appropriate
* values - they're all fixed by some #defines anyway, so there's no point
* writing each call out in full by hand.
*/
#define BUILD_TABLE(tbl) \
if (make_decode_table(MAXSYMBOLS(tbl), TABLEBITS(tbl), LENTABLE(tbl), SYMTABLE(tbl))) { \
return DECR_ILLEGALDATA; \
}
/* READ_HUFFSYM(tablename, var) decodes one huffman symbol from the
* bitstream using the stated table and puts it in var.
*/
#define READ_HUFFSYM(tbl, var) \
do { \
ENSURE_BITS(16); \
hufftbl = SYMTABLE(tbl); \
if ((i = hufftbl[PEEK_BITS(TABLEBITS(tbl))]) >= MAXSYMBOLS(tbl)) { \
j = 1 << (uint32_t_BITS - TABLEBITS(tbl)); \
do { \
j >>= 1; \
i <<= 1; \
i |= (bitbuf & j) ? 1 : 0; \
if (!j) { \
return DECR_ILLEGALDATA; \
} \
} while ((i = hufftbl[i]) >= MAXSYMBOLS(tbl)); \
} \
j = LENTABLE(tbl)[(var) = i]; \
REMOVE_BITS(j); \
} while (0)
/* READ_LENGTHS(tablename, first, last) reads in code lengths for symbols
* first to last in the given table. The code lengths are stored in their
* own special LZX way.
*/
#define READ_LENGTHS(tbl, first, last) \
do { \
lb.bb = bitbuf; \
lb.bl = bitsleft; \
lb.ip = inpos; \
if (lzx_read_lens(pState, LENTABLE(tbl), (first), (last), &lb)) { \
return DECR_ILLEGALDATA; \
} \
bitbuf = lb.bb; \
bitsleft = lb.bl; \
inpos = lb.ip; \
} while (0)
/* make_decode_table(nsyms, nbits, length[], table[])
*
* This function was coded by David Tritscher. It builds a fast huffman
* decoding table out of just a canonical huffman code lengths table.
*
* nsyms = total number of symbols in this huffman tree.
* nbits = any symbols with a code length of nbits or less can be decoded
* in one lookup of the table.
* length = A table to get code lengths from [0 to syms-1]
* table = The table to fill up with decoded symbols and pointers.
*
* Returns 0 for OK or 1 for error
*/
static int make_decode_table(uint32_t nsyms, uint32_t nbits, uint8_t* length, uint16_t* table) {
uint16_t sym;
uint32_t leaf;
uint8_t bit_num = 1;
uint32_t fill;
uint32_t pos = 0; /* the current position in the decode table */
uint32_t table_mask = 1 << nbits;
uint32_t bit_mask = table_mask >> 1; /* don't do 0 length codes */
uint32_t next_symbol = bit_mask; /* base of allocation for long codes */
/* fill entries for codes short enough for a direct mapping */
while (bit_num <= nbits) {
for (sym = 0; sym < nsyms; sym++) {
if (length[sym] != bit_num) {
continue;
}
leaf = pos;
if ((pos += bit_mask) > table_mask)
return 1; /* table overrun */
/* fill all possible lookups of this symbol with the symbol itself */
fill = bit_mask;
while (fill-- > 0) {
table[leaf++] = sym;
}
}
bit_mask >>= 1;
bit_num++;
}
/* if there are any codes longer than nbits */
if (pos != table_mask) {
/* clear the remainder of the table */
for (sym = pos; sym < table_mask; sym++) {
table[sym] = 0;
}
/* give ourselves room for codes to grow by up to 16 more bits */
pos <<= 16;
table_mask <<= 16;
bit_mask = 1 << 15;
while (bit_num <= 16) {
for (sym = 0; sym < nsyms; sym++) {
if (length[sym] != bit_num) {
continue;
}
leaf = pos >> 16;
for (fill = 0; fill < bit_num - nbits; fill++) {
/* if this path hasn't been taken yet, 'allocate' two entries */
if (table[leaf] == 0) {
table[(next_symbol << 1)] = 0;
table[(next_symbol << 1) + 1] = 0;
table[leaf] = next_symbol++;
}
/* follow the path and select either left or right for next bit */
leaf = table[leaf] << 1;
if ((pos >> (15 - fill)) & 1)
leaf++;
}
table[leaf] = sym;
if ((pos += bit_mask) > table_mask)
return 1; /* table overflow */
}
bit_mask >>= 1;
bit_num++;
}
}
/* full table? */
if (pos == table_mask)
return 0;
/* either erroneous table, or all elements are 0 - let's find out. */
for (sym = 0; sym < nsyms; sym++) {
if (length[sym])
return 1;
}
return 0;
}
struct lzx_bits {
uint32_t bb;
int bl;
uint8_t* ip;
};
static int lzx_read_lens(struct lzx_state* pState, uint8_t* lens, uint32_t first, uint32_t last,
struct lzx_bits* lb) {
uint32_t i, j, x, y;
int z;
uint32_t bitbuf = lb->bb;
int bitsleft = lb->bl;
uint8_t* inpos = lb->ip;
uint16_t* hufftbl;
for (x = 0; x < 20; x++) {
READ_BITS(y, 4);
LENTABLE(PRETREE)[x] = y;
}
BUILD_TABLE(PRETREE);
for (x = first; x < last;) {
READ_HUFFSYM(PRETREE, z);
if (z == 17) {
READ_BITS(y, 4);
y += 4;
while (y--)
lens[x++] = 0;
} else if (z == 18) {
READ_BITS(y, 5);
y += 20;
while (y--)
lens[x++] = 0;
} else if (z == 19) {
READ_BITS(y, 1);
y += 4;
READ_HUFFSYM(PRETREE, z);
z = lens[x] - z;
if (z < 0)
z += 17;
while (y--)
lens[x++] = z;
} else {
z = lens[x] - z;
if (z < 0)
z += 17;
lens[x++] = z;
}
}
lb->bb = bitbuf;
lb->bl = bitsleft;
lb->ip = inpos;
return 0;
}
int lzx_decompress(struct lzx_state* pState, const unsigned char* inpos, unsigned char* outpos, int inlen,
int outlen) {
const uint8_t* endinp = inpos + inlen;
uint8_t* window = pState->window;
uint8_t* runsrc, *rundest;
uint16_t* hufftbl; /* used in READ_HUFFSYM macro as chosen decoding table */
uint32_t window_posn = pState->window_posn;
uint32_t window_size = pState->window_size;
uint32_t R0 = pState->R0;
uint32_t R1 = pState->R1;
uint32_t R2 = pState->R2;
uint32_t bitbuf;
int bitsleft;
uint32_t match_offset, i, j, k; /* ijk used in READ_HUFFSYM macro */
struct lzx_bits lb; /* used in READ_LENGTHS macro */
int togo = outlen, this_run, main_element, aligned_bits;
int match_length, length_footer, extra, verbatim_bits;
INIT_BITSTREAM;
/* read header if necessary */
if (!pState->header_read) {
i = j = 0;
READ_BITS(k, 1);
if (k) {
READ_BITS(i, 16);
READ_BITS(j, 16);
}
pState->intel_filesize = (i << 16) | j; /* or 0 if not encoded */
pState->header_read = 1;
}
/* main decoding loop */
while (togo > 0) {
/* last block finished, new block expected */
if (pState->block_remaining == 0) {
if (pState->block_type == LZX_BLOCKTYPE_UNCOMPRESSED) {
if (pState->block_length & 1)
inpos++; /* realign bitstream to word */
INIT_BITSTREAM;
}
READ_BITS(pState->block_type, 3);
READ_BITS(i, 16);
READ_BITS(j, 8);
pState->block_remaining = pState->block_length = (i << 8) | j;
switch (pState->block_type) {
case LZX_BLOCKTYPE_ALIGNED:
for (i = 0; i < 8; i++) {
READ_BITS(j, 3);
LENTABLE(ALIGNED)[i] = j;
}
BUILD_TABLE(ALIGNED);
/* rest of aligned header is same as verbatim */
case LZX_BLOCKTYPE_VERBATIM:
READ_LENGTHS(MAINTREE, 0, 256);
READ_LENGTHS(MAINTREE, 256, pState->main_elements);
BUILD_TABLE(MAINTREE);
if (LENTABLE(MAINTREE)[0xE8] != 0)
pState->intel_started = 1;
READ_LENGTHS(LENGTH, 0, LZX_NUM_SECONDARY_LENGTHS);
BUILD_TABLE(LENGTH);
break;
case LZX_BLOCKTYPE_UNCOMPRESSED:
pState->intel_started = 1; /* because we can't assume otherwise */
ENSURE_BITS(16); /* get up to 16 pad bits into the buffer */
if (bitsleft > 16)
inpos -= 2; /* and align the bitstream! */
R0 = inpos[0] | (inpos[1] << 8) | (inpos[2] << 16) | (inpos[3] << 24);
inpos += 4;
R1 = inpos[0] | (inpos[1] << 8) | (inpos[2] << 16) | (inpos[3] << 24);
inpos += 4;
R2 = inpos[0] | (inpos[1] << 8) | (inpos[2] << 16) | (inpos[3] << 24);
inpos += 4;
break;
default:
return DECR_ILLEGALDATA;
}
}
/* buffer exhaustion check */
if (inpos > endinp) {
/* it's possible to have a file where the next run is less than
* 16 bits in size. In this case, the READ_HUFFSYM() macro used
* in building the tables will exhaust the buffer, so we should
* allow for this, but not allow those accidentally read bits to
* be used (so we check that there are at least 16 bits
* remaining - in this boundary case they aren't really part of
* the compressed data)
*/
if (inpos > (endinp + 2) || bitsleft < 16)
return DECR_ILLEGALDATA;
}
while ((this_run = pState->block_remaining) > 0 && togo > 0) {
if (this_run > togo)
this_run = togo;
togo -= this_run;
pState->block_remaining -= this_run;
/* apply 2^x-1 mask */
window_posn &= window_size - 1;
/* runs can't straddle the window wraparound */
if ((window_posn + this_run) > window_size)
return DECR_DATAFORMAT;
switch (pState->block_type) {
case LZX_BLOCKTYPE_VERBATIM:
while (this_run > 0) {
READ_HUFFSYM(MAINTREE, main_element);
if (main_element < LZX_NUM_CHARS) {
/* literal: 0 to LZX_NUM_CHARS-1 */
window[window_posn++] = main_element;
this_run--;
} else {
/* match: LZX_NUM_CHARS + ((slot<<3) | length_header (3 bits)) */
main_element -= LZX_NUM_CHARS;
match_length = main_element & LZX_NUM_PRIMARY_LENGTHS;
if (match_length == LZX_NUM_PRIMARY_LENGTHS) {
READ_HUFFSYM(LENGTH, length_footer);
match_length += length_footer;
}
match_length += LZX_MIN_MATCH;
match_offset = main_element >> 3;
if (match_offset > 2) {
/* not repeated offset */
if (match_offset != 3) {
extra = extra_bits[match_offset];
READ_BITS(verbatim_bits, extra);
match_offset = position_base[match_offset] - 2 + verbatim_bits;
} else {
match_offset = 1;
}
/* update repeated offset LRU queue */
R2 = R1;
R1 = R0;
R0 = match_offset;
} else if (match_offset == 0) {
match_offset = R0;
} else if (match_offset == 1) {
match_offset = R1;
R1 = R0;
R0 = match_offset;
} else /* match_offset == 2 */ {
match_offset = R2;
R2 = R0;
R0 = match_offset;
}
rundest = window + window_posn;
runsrc = rundest - match_offset;
window_posn += match_length;
if (window_posn > window_size)
return DECR_ILLEGALDATA;
this_run -= match_length;
/* copy any wrapped around source data */
while ((runsrc < window) && (match_length-- > 0)) {
*rundest++ = *(runsrc + window_size);
runsrc++;
}
/* copy match data - no worries about destination wraps */
while (match_length-- > 0)
*rundest++ = *runsrc++;
}
}
break;
case LZX_BLOCKTYPE_ALIGNED:
while (this_run > 0) {
READ_HUFFSYM(MAINTREE, main_element);
if (main_element < LZX_NUM_CHARS) {
/* literal: 0 to LZX_NUM_CHARS-1 */
window[window_posn++] = main_element;
this_run--;
} else {
/* match: LZX_NUM_CHARS + ((slot<<3) | length_header (3 bits)) */
main_element -= LZX_NUM_CHARS;
match_length = main_element & LZX_NUM_PRIMARY_LENGTHS;
if (match_length == LZX_NUM_PRIMARY_LENGTHS) {
READ_HUFFSYM(LENGTH, length_footer);
match_length += length_footer;
}
match_length += LZX_MIN_MATCH;
match_offset = main_element >> 3;
if (match_offset > 2) {
/* not repeated offset */
extra = extra_bits[match_offset];
match_offset = position_base[match_offset] - 2;
if (extra > 3) {
/* verbatim and aligned bits */
extra -= 3;
READ_BITS(verbatim_bits, extra);
match_offset += (verbatim_bits << 3);
READ_HUFFSYM(ALIGNED, aligned_bits);
match_offset += aligned_bits;
} else if (extra == 3) {
/* aligned bits only */
READ_HUFFSYM(ALIGNED, aligned_bits);
match_offset += aligned_bits;
} else if (extra > 0) { /* extra==1, extra==2 */
/* verbatim bits only */
READ_BITS(verbatim_bits, extra);
match_offset += (uint32_t)verbatim_bits;
} else /* extra == 0 */ {
/* ??? */
match_offset = 1;
}
/* update repeated offset LRU queue */
R2 = R1;
R1 = R0;
R0 = match_offset;
} else if (match_offset == 0) {
match_offset = R0;
} else if (match_offset == 1) {
match_offset = R1;
R1 = R0;
R0 = match_offset;
} else /* match_offset == 2 */ {
match_offset = R2;
R2 = R0;
R0 = match_offset;
}
rundest = window + window_posn;
runsrc = rundest - match_offset;
window_posn += (uint32_t)match_length;
if (window_posn > window_size)
return DECR_ILLEGALDATA;
this_run -= match_length;
/* copy any wrapped around source data */
while ((runsrc < window) && (match_length-- > 0)) {
*rundest++ = *(runsrc + window_size);
runsrc++;
}
/* copy match data - no worries about destination wraps */
while (match_length-- > 0)
*rundest++ = *runsrc++;
}
}
break;
case LZX_BLOCKTYPE_UNCOMPRESSED:
if ((inpos + this_run) > endinp)
return DECR_ILLEGALDATA;
memcpy(window + window_posn, inpos, (size_t)this_run);
inpos += this_run;
window_posn += (uint32_t)this_run;
break;
default:
return DECR_ILLEGALDATA; /* might as well */
}
}
}
if (togo != 0)
return DECR_ILLEGALDATA;
memcpy(outpos, window + ((!window_posn) ? window_size : window_posn) - outlen, (size_t)outlen);
pState->window_posn = window_posn;
pState->R0 = R0;
pState->R1 = R1;
pState->R2 = R2;
/* intel E8 decoding */
if ((pState->frames_read++ < 32768) && pState->intel_filesize != 0) {
if (outlen <= 6 || !pState->intel_started) {
pState->intel_curpos += outlen;
} else {
uint8_t* data = outpos;
uint8_t* dataend = data + outlen - 10;
int32_t curpos = pState->intel_curpos;
int32_t filesize = pState->intel_filesize;
int32_t abs_off, rel_off;
pState->intel_curpos = curpos + outlen;
while (data < dataend) {
if (*data++ != 0xE8) {
curpos++;
continue;
}
abs_off = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24);
if ((abs_off >= -curpos) && (abs_off < filesize)) {
rel_off = (abs_off >= 0) ? abs_off - curpos : abs_off + filesize;
data[0] = (uint8_t)rel_off;
data[1] = (uint8_t)(rel_off >> 8);
data[2] = (uint8_t)(rel_off >> 16);
data[3] = (uint8_t)(rel_off >> 24);
}
data += 4;
curpos += 5;
}
}
}
return DECR_OK;
}

61
thirdparty/lzx/lzx.h vendored Normal file
View File

@@ -0,0 +1,61 @@
// Modified version of the code from Wine:
// https://gitlab.winehq.org/wine/wine/-/blob/fcc40a07909dc7626b6d1e2ec73f823d828a47e8/dlls/itss/lzx.h
/***************************************************************************
* lzx.h - LZX decompression routines *
* ------------------- *
* *
* maintainer: Jed Wing <jedwin@ugcs.caltech.edu> *
* source: modified lzx.c from cabextract v0.5 *
* notes: This file was taken from cabextract v0.5, which was, *
* itself, a modified version of the lzx decompression code *
* from unlzx. *
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. Note that an exemption to this *
* license has been granted by Stuart Caie for the purposes of *
* distribution with chmlib. This does not, to the best of my *
* knowledge, constitute a change in the license of this (the LZX) code *
* in general. *
* *
***************************************************************************/
#ifndef INCLUDED_LZX_H
#define INCLUDED_LZX_H
#ifdef __cplusplus
extern "C" {
#endif
/* return codes */
#define DECR_OK (0)
#define DECR_DATAFORMAT (1)
#define DECR_ILLEGALDATA (2)
#define DECR_NOMEMORY (3)
/* opaque state structure */
struct lzx_state;
/* create an lzx state object */
struct lzx_state* lzx_init(int window);
/* destroy an lzx state object */
void lzx_teardown(struct lzx_state* pState);
/* reset an lzx stream */
void lzx_reset(struct lzx_state* pState);
/* decompress an LZX compressed block */
int lzx_decompress(struct lzx_state* pState, const unsigned char* inpos, unsigned char* outpos, int inlen,
int outlen);
#ifdef __cplusplus
}
#endif
#endif /* INCLUDED_LZX_H */