zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-09-02 15:07:26 +00:00
Code Activity

feat: dump t6 xbox fastfile data

Browse Source
This commit is contained in:
Jan Laupetin
2025-08-28 21:14:12 +02:00
parent 763d280805
commit 75d579b5e3
10 changed files with 1074 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
premake5.lua
View File

@@ -95,6 +95,7 @@ include "thirdparty/catch2.lua"
include "thirdparty/eigen.lua" include "thirdparty/eigen.lua"
include "thirdparty/libtomcrypt.lua" include "thirdparty/libtomcrypt.lua"
include "thirdparty/libtommath.lua" include "thirdparty/libtommath.lua"
include "thirdparty/lzx.lua"
include "thirdparty/json.lua" include "thirdparty/json.lua"
include "thirdparty/minilzo.lua" include "thirdparty/minilzo.lua"
include "thirdparty/minizip.lua" include "thirdparty/minizip.lua"
@@ -108,6 +109,7 @@ group "ThirdParty"
eigen:project() eigen:project()
libtommath:project() libtommath:project()
libtomcrypt:project() libtomcrypt:project()
lzx:project()
json:project() json:project()
minilzo:project() minilzo:project()
minizip:project() minizip:project()

2
src/ZoneCommon.lua
View File

@@ -20,6 +20,7 @@ function ZoneCommon:link(links)
links:linkto(ObjCommon) links:linkto(ObjCommon)
links:linkto(Parser) links:linkto(Parser)
links:linkto(Utils) links:linkto(Utils)
links:linkto(lzx)
end end
function ZoneCommon:use() function ZoneCommon:use()
@@ -45,5 +46,6 @@ function ZoneCommon:project()
path.join(folder, "ZoneCommon/**.cpp") path.join(folder, "ZoneCommon/**.cpp")
} }
lzx:include(includes)
self:include(includes) self:include(includes)
end end

130
src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp
View File

@@ -1,12 +1,132 @@
#include "XChunkProcessorLzxDecompress.h" #include "XChunkProcessorLzxDecompress.h"
#include <cstring> #include <cstring>
#include <format>
#include <iostream>
#include <lzx.h>
size_t namespace
XChunkProcessorLzxDecompress::Process(unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
{ {
// TODO: Implement uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize)
{
const auto value = input[offset];
offset++;
remainingSize--;
return value;
}
memcpy(output, input, inputLength); uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte)
return inputLength; {
return static_cast<uint16_t>(static_cast<uint16_t>(static_cast<uint16_t>(highByte) << 8u) | static_cast<uint16_t>(lowByte));
}
void LogErrorHeaderSpace(size_t remainingInputSize)
{
std::cerr << std::format("XMemCompress: Not enough data for header: {}\n", remainingInputSize);
}
} // namespace
XChunkProcessorLzxDecompress::XChunkProcessorLzxDecompress(const unsigned streamCount)
: m_lzx_states(streamCount)
{
// T6 uses 17 for window bits
for (auto& lzxState : m_lzx_states)
lzxState = lzx_init(17);
}
XChunkProcessorLzxDecompress::~XChunkProcessorLzxDecompress()
{
for (auto* lzxState : m_lzx_states)
lzx_teardown(static_cast<lzx_state*>(lzxState));
}
size_t XChunkProcessorLzxDecompress::Process(
const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
{
auto* state = static_cast<lzx_state*>(m_lzx_states[streamNumber]);
// lzx state is reset before each chunk
lzx_reset(state);
size_t curInputOffset = 0uz;
size_t curInputSize = inputLength;
size_t curOutputOffset = 0uz;
size_t curOutputSize = outputBufferSize;
uint8_t lowByte;
uint16_t dstSize, srcSize;
while (curInputSize > 0)
{
uint8_t highByte = NextByte(input, curInputOffset, curInputSize);
uint8_t suffixSize;
if (highByte == 0xFF) // magic number: output is smaller than 0x8000
{
if (curInputSize < 4)
{
LogErrorHeaderSpace(curInputSize);
return curOutputOffset;
}
highByte = NextByte(input, curInputOffset, curInputSize);
lowByte = NextByte(input, curInputOffset, curInputSize);
dstSize = CombineHighLow(highByte, lowByte);
highByte = NextByte(input, curInputOffset, curInputSize);
lowByte = NextByte(input, curInputOffset, curInputSize);
srcSize = CombineHighLow(highByte, lowByte);
// The game seems to skip a 5 byte suffix after these blocks, not sure why.
suffixSize = 5u;
}
else
{
if (curInputSize < 1)
{
LogErrorHeaderSpace(curInputSize);
return curOutputOffset;
}
dstSize = 0x8000u;
lowByte = NextByte(input, curInputOffset, curInputSize);
srcSize = CombineHighLow(highByte, lowByte);
suffixSize = 0u;
}
if (srcSize == 0 || dstSize == 0)
{
// Other implementations do not handle this as a failure, game code suggests otherwise though
std::cerr << std::format("XMemCompress: EOF: {} {}, {}\n", srcSize, dstSize, curInputSize);
return curOutputOffset;
}
if (static_cast<size_t>(srcSize) + suffixSize > curInputSize)
{
std::cerr << std::format("XMemCompress: block size bigger than remaining data: {} > {}\n", srcSize, curInputSize);
return curOutputOffset;
}
if (dstSize > curOutputSize)
{
std::cerr << std::format("XMemCompress: output size bigger than remaining data: {} > {}\n", dstSize, curOutputSize);
return curOutputOffset;
}
auto ret = lzx_decompress(state, &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize);
curInputOffset += srcSize + suffixSize;
curInputSize -= (srcSize + suffixSize);
curOutputOffset += dstSize;
curOutputSize -= srcSize;
if (ret != DECR_OK)
{
std::cerr << std::format("XMemCompress: lzx decompression failed: {}\n", ret);
return curOutputOffset;
}
}
return curOutputOffset;
} }

8
src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h
View File

@@ -1,8 +1,16 @@
#pragma once #pragma once
#include "IXChunkProcessor.h" #include "IXChunkProcessor.h"
#include <vector>
class XChunkProcessorLzxDecompress final : public IXChunkProcessor class XChunkProcessorLzxDecompress final : public IXChunkProcessor
{ {
public: public:
explicit XChunkProcessorLzxDecompress(unsigned streamCount);
~XChunkProcessorLzxDecompress();
size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override; size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
private:
std::vector<void*> m_lzx_states;
}; };

2
src/ZoneLoading.lua
View File

@@ -14,6 +14,7 @@ function ZoneLoading:link(links)
links:linkto(Cryptography) links:linkto(Cryptography)
links:linkto(Utils) links:linkto(Utils)
links:linkto(ZoneCommon) links:linkto(ZoneCommon)
links:linkto(lzx)
links:linkto(zlib) links:linkto(zlib)
if os.host() == "linux" then if os.host() == "linux" then
@@ -55,6 +56,7 @@ function ZoneLoading:project()
self:include(includes) self:include(includes)
Cryptography:include(includes) Cryptography:include(includes)
Utils:include(includes) Utils:include(includes)
lzx:include(includes)
zlib:include(includes) zlib:include(includes)
ZoneCode:include(includes) ZoneCode:include(includes)

16
src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp
View File

@@ -172,8 +172,18 @@ namespace
AddXChunkProcessor(const bool isBigEndian, const bool isEncrypted, const bool isLzxCompressed, ZoneLoader& zoneLoader, std::string& fileName) AddXChunkProcessor(const bool isBigEndian, const bool isEncrypted, const bool isLzxCompressed, ZoneLoader& zoneLoader, std::string& fileName)
{ {
ICapturedDataProvider* result = nullptr; ICapturedDataProvider* result = nullptr;
auto xChunkProcessor = processor::CreateProcessorXChunks( std::unique_ptr<processor::IProcessorXChunks> xChunkProcessor;
ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, isBigEndian ? GameEndianness::BE : GameEndianness::LE, ZoneConstants::VANILLA_BUFFER_SIZE);
if (isBigEndian)
{
xChunkProcessor = processor::CreateProcessorXChunks(
ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, GameEndianness::BE, ZoneConstants::VANILLA_BUFFER_SIZE);
}
else
{
xChunkProcessor = processor::CreateProcessorXChunks(
ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, GameEndianness::LE, ZoneConstants::VANILLA_BUFFER_SIZE);
}
const uint8_t (&salsa20Key)[32] = isBigEndian ? ZoneConstants::SALSA20_KEY_TREYARCH_XENON : ZoneConstants::SALSA20_KEY_TREYARCH_PC; const uint8_t (&salsa20Key)[32] = isBigEndian ? ZoneConstants::SALSA20_KEY_TREYARCH_XENON : ZoneConstants::SALSA20_KEY_TREYARCH_PC;
@@ -189,7 +199,7 @@ namespace
if (isLzxCompressed) if (isLzxCompressed)
{ {
// Decompress the chunks using lzx // Decompress the chunks using lzx
xChunkProcessor->AddChunkProcessor(std::make_unique<XChunkProcessorLzxDecompress>()); xChunkProcessor->AddChunkProcessor(std::make_unique<XChunkProcessorLzxDecompress>(ZoneConstants::STREAM_COUNT));
} }
else else
{ {

11
src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp
View File

@@ -223,19 +223,18 @@ namespace
} }
const size_t readSize = m_base_stream->Load(&chunkSize, sizeof(chunkSize)); const size_t readSize = m_base_stream->Load(&chunkSize, sizeof(chunkSize));
if (m_endianness == GameEndianness::LE)
chunkSize = endianness::FromLittleEndian(chunkSize);
else
chunkSize = endianness::FromBigEndian(chunkSize);
if (readSize == 0) if (readSize < sizeof(chunkSize) || chunkSize == 0)
{ {
m_eof_reached = true; m_eof_reached = true;
m_eof_stream = streamNum; m_eof_stream = streamNum;
return; return;
} }
if (m_endianness == GameEndianness::LE)
chunkSize = endianness::FromLittleEndian(chunkSize);
else
chunkSize = endianness::FromBigEndian(chunkSize);
if (chunkSize > m_chunk_size) if (chunkSize > m_chunk_size)
{ {
throw InvalidChunkSizeException(chunkSize, m_chunk_size); throw InvalidChunkSizeException(chunkSize, m_chunk_size);

48
thirdparty/lzx.lua vendored Normal file
View File

@@ -0,0 +1,48 @@
lzx = {}
function lzx:include(includes)
if includes:handle(self:name()) then
includedirs {
path.join(ThirdPartyFolder(), "lzx")
}
end
end
function lzx:link(links)
links:add(self:name())
end
function lzx:use()
end
function lzx:name()
return "lzx"
end
function lzx:project()
local folder = ThirdPartyFolder()
local includes = Includes:create()
project(self:name())
targetdir(TargetDirectoryLib)
location "%{wks.location}/thirdparty/%{prj.name}"
kind "StaticLib"
language "C"
files {
path.join(folder, "lzx/*.h"),
path.join(folder, "lzx/*.c")
}
defines {
"_CRT_SECURE_NO_WARNINGS",
"_CRT_NONSTDC_NO_DEPRECATE"
}
self:include(includes)
-- Disable warnings. They do not have any value to us since it is not our code.
warnings "off"
end

808
thirdparty/lzx/lzx.c vendored Normal file
View File

@@ -0,0 +1,808 @@
// Modified version of the code from Wine:
// https://gitlab.winehq.org/wine/wine/-/blob/fcc40a07909dc7626b6d1e2ec73f823d828a47e8/dlls/itss/lzx.c
/***************************************************************************
* lzx.c - LZX decompression routines *
* ------------------- *
* *
* maintainer: Jed Wing <jedwin@ugcs.caltech.edu> *
* source: modified lzx.c from cabextract v0.5 *
* notes: This file was taken from cabextract v0.5, which was, *
* itself, a modified version of the lzx decompression code *
* from unlzx. *
* *
* platforms: In its current incarnation, this file has been tested on *
* two different Linux platforms (one, redhat-based, with a *
* 2.1.2 glibc and gcc 2.95.x, and the other, Debian, with *
* 2.2.4 glibc and both gcc 2.95.4 and gcc 3.0.2). Both were *
* Intel x86 compatible machines. *
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. Note that an exemption to this *
* license has been granted by Stuart Caie for the purposes of *
* distribution with chmlib. This does not, to the best of my *
* knowledge, constitute a change in the license of this (the LZX) code *
* in general. *
* *
***************************************************************************/
#include "lzx.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
/* some constants defined by the LZX specification */
#define LZX_MIN_MATCH 2
/* #define LZX_MAX_MATCH 257 */
#define LZX_NUM_CHARS 256
#define LZX_BLOCKTYPE_INVALID 0 /* also blocktypes 4-7 invalid */
#define LZX_BLOCKTYPE_VERBATIM 1
#define LZX_BLOCKTYPE_ALIGNED 2
#define LZX_BLOCKTYPE_UNCOMPRESSED 3
#define LZX_PRETREE_NUM_ELEMENTS 20
#define LZX_ALIGNED_NUM_ELEMENTS 8 /* aligned offset tree #elements */
#define LZX_NUM_PRIMARY_LENGTHS 7 /* this one missing from spec! */
#define LZX_NUM_SECONDARY_LENGTHS 249 /* length tree #elements */
/* LZX huffman defines: tweak tablebits as desired */
#define LZX_PRETREE_MAXSYMBOLS LZX_PRETREE_NUM_ELEMENTS
#define LZX_PRETREE_TABLEBITS 6
#define LZX_MAINTREE_MAXSYMBOLS (LZX_NUM_CHARS + 50 * 8)
#define LZX_MAINTREE_TABLEBITS 12
#define LZX_LENGTH_MAXSYMBOLS (LZX_NUM_SECONDARY_LENGTHS + 1)
#define LZX_LENGTH_TABLEBITS 12
#define LZX_ALIGNED_MAXSYMBOLS LZX_ALIGNED_NUM_ELEMENTS
#define LZX_ALIGNED_TABLEBITS 7
#define LZX_LENTABLE_SAFETY 64 /* we allow length table decoding overruns */
#define LZX_DECLARE_TABLE(tbl) \
uint16_t tbl## _table[(1 << LZX_## tbl## _TABLEBITS) + (LZX_## tbl## _MAXSYMBOLS << 1)]; \
uint8_t tbl## _len[LZX_## tbl## _MAXSYMBOLS + LZX_LENTABLE_SAFETY]
struct lzx_state {
uint8_t* window; /* the actual decoding window */
uint32_t window_size; /* window size (32Kb through 2Mb) */
uint32_t actual_size; /* window size when it was first allocated */
uint32_t window_posn; /* current offset within the window */
uint32_t R0, R1, R2; /* for the LRU offset system */
uint16_t main_elements; /* number of main tree elements */
int header_read; /* have we started decoding at all yet? */
uint16_t block_type; /* type of this block */
uint32_t block_length; /* uncompressed length of this block */
uint32_t block_remaining; /* uncompressed bytes still left to decode */
uint32_t frames_read; /* the number of CFDATA blocks processed */
int32_t intel_filesize; /* magic header value used for transform */
int32_t intel_curpos; /* current offset in transform space */
int intel_started; /* have we seen any translatable data yet? */
LZX_DECLARE_TABLE(PRETREE);
LZX_DECLARE_TABLE(MAINTREE);
LZX_DECLARE_TABLE(LENGTH);
LZX_DECLARE_TABLE(ALIGNED);
};
/* LZX decruncher */
/* Microsoft's LZX document and their implementation of the
* com.ms.util.cab Java package do not concur.
*
* In the LZX document, there is a table showing the correlation between
* window size and the number of position slots. It states that the 1MB
* window = 40 slots and the 2MB window = 42 slots. In the implementation,
* 1MB = 42 slots, 2MB = 50 slots. The actual calculation is 'find the
* first slot whose position base is equal to or more than the required
* window size'. This would explain why other tables in the document refer
* to 50 slots rather than 42.
*
* The constant NUM_PRIMARY_LENGTHS used in the decompression pseudocode
* is not defined in the specification.
*
* The LZX document does not state the uncompressed block has an
* uncompressed length field. Where does this length field come from, so
* we can know how large the block is? The implementation has it as the 24
* bits following after the 3 blocktype bits, before the alignment
* padding.
*
* The LZX document states that aligned offset blocks have their aligned
* offset huffman tree AFTER the main and length trees. The implementation
* suggests that the aligned offset tree is BEFORE the main and length
* trees.
*
* The LZX document decoding algorithm states that, in an aligned offset
* block, if an extra_bits value is 1, 2 or 3, then that number of bits
* should be read and the result added to the match offset. This is
* correct for 1 and 2, but not 3, where just a huffman symbol (using the
* aligned tree) should be read.
*
* Regarding the E8 preprocessing, the LZX document states 'No translation
* may be performed on the last 6 bytes of the input block'. This is
* correct. However, the pseudocode provided checks for the *E8 leader*
* up to the last 6 bytes. If the leader appears between -10 and -7 bytes
* from the end, this would cause the next four bytes to be modified, at
* least one of which would be in the last 6 bytes, which is not allowed
* according to the spec.
*
* The specification states that the huffman trees must always contain at
* least one element. However, many CAB files contain blocks where the
* length tree is completely empty (because there are no matches), and
* this is expected to succeed.
*/
/* LZX uses what it calls 'position slots' to represent match offsets.
* What this means is that a small 'position slot' number and a small
* offset from that slot are encoded instead of one large offset for
* every match.
* - position_base is an index to the position slot bases
* - extra_bits states how many bits of offset-from-base data is needed.
*/
static const uint8_t extra_bits[51] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5,
5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11,
12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17};
static const uint32_t position_base[51] = {
0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32,
48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536,
2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536,
98304, 131072, 196608, 262144, 393216, 524288, 655360, 786432, 917504, 1048576, 1179648,
1310720, 1441792, 1572864, 1703936, 1835008, 1966080, 2097152};
struct lzx_state* lzx_init(int window) {
struct lzx_state* pState = NULL;
uint32_t wndsize = 1 << window;
int i, posn_slots;
/* LZX supports window sizes of 2^15 (32Kb) through 2^21 (2Mb) */
/* if a previously allocated window is big enough, keep it */
if (window < 15 || window > 21)
return NULL;
/* allocate state and associated window */
pState = (struct lzx_state*)malloc(sizeof(struct lzx_state));
if (!pState || !(pState->window = (uint8_t*)malloc(wndsize))) {
free(pState);
return NULL;
}
pState->actual_size = wndsize;
pState->window_size = wndsize;
/* calculate required position slots */
if (window == 20)
posn_slots = 42;
else if (window == 21)
posn_slots = 50;
else
posn_slots = window << 1;
/** alternatively **/
/* posn_slots=i=0; while (i < wndsize) i += 1 << extra_bits[posn_slots++]; */
/* initialize other state */
pState->R0 = pState->R1 = pState->R2 = 1;
pState->main_elements = LZX_NUM_CHARS + (posn_slots << 3);
pState->header_read = 0;
pState->frames_read = 0;
pState->block_remaining = 0;
pState->block_type = LZX_BLOCKTYPE_INVALID;
pState->intel_curpos = 0;
pState->intel_started = 0;
pState->window_posn = 0;
/* initialise tables to 0 (because deltas will be applied to them) */
for (i = 0; i < LZX_MAINTREE_MAXSYMBOLS; i++)
pState->MAINTREE_len[i] = 0;
for (i = 0; i < LZX_LENGTH_MAXSYMBOLS; i++)
pState->LENGTH_len[i] = 0;
return pState;
}
void lzx_teardown(struct lzx_state* pState) {
if (pState) {
if (pState->window)
free(pState->window);
free(pState);
}
}
void lzx_reset(struct lzx_state* pState) {
int i;
pState->R0 = pState->R1 = pState->R2 = 1;
pState->header_read = 0;
pState->frames_read = 0;
pState->block_remaining = 0;
pState->block_type = LZX_BLOCKTYPE_INVALID;
pState->intel_curpos = 0;
pState->intel_started = 0;
pState->window_posn = 0;
for (i = 0; i < LZX_MAINTREE_MAXSYMBOLS + LZX_LENTABLE_SAFETY; i++) {
pState->MAINTREE_len[i] = 0;
}
for (i = 0; i < LZX_LENGTH_MAXSYMBOLS + LZX_LENTABLE_SAFETY; i++) {
pState->LENGTH_len[i] = 0;
}
}
/* Bitstream reading macros:
*
* INIT_BITSTREAM should be used first to set up the system
* READ_BITS(var,n) takes N bits from the buffer and puts them in var
*
* ENSURE_BITS(n) ensures there are at least N bits in the bit buffer
* PEEK_BITS(n) extracts (without removing) N bits from the bit buffer
* REMOVE_BITS(n) removes N bits from the bit buffer
*
* These bit access routines work by using the area beyond the MSB and the
* LSB as a free source of zeroes. This avoids having to mask any bits.
* So we have to know the bit width of the bitbuffer variable. This is
* sizeof(uint32_t) * 8, also defined as uint32_t_BITS
*/
/* number of bits in uint32_t. Note: This must be at multiple of 16, and at
* least 32 for the bitbuffer code to work (ie, it must be able to ensure
* up to 17 bits - that's adding 16 bits when there's one bit left, or
* adding 32 bits when there are no bits left. The code should work fine
* for machines where uint32_t >= 32 bits.
*/
#define uint32_t_BITS (sizeof(uint32_t) << 3)
#define INIT_BITSTREAM \
do { \
bitsleft = 0; \
bitbuf = 0; \
} while (0)
#define ENSURE_BITS(n) \
while (bitsleft < (n)) { \
bitbuf |= ((inpos[1] << 8) | inpos[0]) << (uint32_t_BITS - 16 - bitsleft); \
bitsleft += 16; \
inpos += 2; \
}
#define PEEK_BITS(n) (bitbuf >> (uint32_t_BITS - (n)))
#define REMOVE_BITS(n) ((bitbuf <<= (n)), (bitsleft -= (n)))
#define READ_BITS(v, n) \
do { \
ENSURE_BITS(n); \
(v) = PEEK_BITS(n); \
REMOVE_BITS(n); \
} while (0)
/* Huffman macros */
#define TABLEBITS(tbl) (LZX_## tbl## _TABLEBITS)
#define MAXSYMBOLS(tbl) (LZX_## tbl## _MAXSYMBOLS)
#define SYMTABLE(tbl) (pState->tbl## _table)
#define LENTABLE(tbl) (pState->tbl## _len)
/* BUILD_TABLE(tablename) builds a huffman lookup table from code lengths.
* In reality, it just calls make_decode_table() with the appropriate
* values - they're all fixed by some #defines anyway, so there's no point
* writing each call out in full by hand.
*/
#define BUILD_TABLE(tbl) \
if (make_decode_table(MAXSYMBOLS(tbl), TABLEBITS(tbl), LENTABLE(tbl), SYMTABLE(tbl))) { \
return DECR_ILLEGALDATA; \
}
/* READ_HUFFSYM(tablename, var) decodes one huffman symbol from the
* bitstream using the stated table and puts it in var.
*/
#define READ_HUFFSYM(tbl, var) \
do { \
ENSURE_BITS(16); \
hufftbl = SYMTABLE(tbl); \
if ((i = hufftbl[PEEK_BITS(TABLEBITS(tbl))]) >= MAXSYMBOLS(tbl)) { \
j = 1 << (uint32_t_BITS - TABLEBITS(tbl)); \
do { \
j >>= 1; \
i <<= 1; \
i |= (bitbuf & j) ? 1 : 0; \
if (!j) { \
return DECR_ILLEGALDATA; \
} \
} while ((i = hufftbl[i]) >= MAXSYMBOLS(tbl)); \
} \
j = LENTABLE(tbl)[(var) = i]; \
REMOVE_BITS(j); \
} while (0)
/* READ_LENGTHS(tablename, first, last) reads in code lengths for symbols
* first to last in the given table. The code lengths are stored in their
* own special LZX way.
*/
#define READ_LENGTHS(tbl, first, last) \
do { \
lb.bb = bitbuf; \
lb.bl = bitsleft; \
lb.ip = inpos; \
if (lzx_read_lens(pState, LENTABLE(tbl), (first), (last), &lb)) { \
return DECR_ILLEGALDATA; \
} \
bitbuf = lb.bb; \
bitsleft = lb.bl; \
inpos = lb.ip; \
} while (0)
/* make_decode_table(nsyms, nbits, length[], table[])
*
* This function was coded by David Tritscher. It builds a fast huffman
* decoding table out of just a canonical huffman code lengths table.
*
* nsyms = total number of symbols in this huffman tree.
* nbits = any symbols with a code length of nbits or less can be decoded
* in one lookup of the table.
* length = A table to get code lengths from [0 to syms-1]
* table = The table to fill up with decoded symbols and pointers.
*
* Returns 0 for OK or 1 for error
*/
static int make_decode_table(uint32_t nsyms, uint32_t nbits, uint8_t* length, uint16_t* table) {
uint16_t sym;
uint32_t leaf;
uint8_t bit_num = 1;
uint32_t fill;
uint32_t pos = 0; /* the current position in the decode table */
uint32_t table_mask = 1 << nbits;
uint32_t bit_mask = table_mask >> 1; /* don't do 0 length codes */
uint32_t next_symbol = bit_mask; /* base of allocation for long codes */
/* fill entries for codes short enough for a direct mapping */
while (bit_num <= nbits) {
for (sym = 0; sym < nsyms; sym++) {
if (length[sym] != bit_num) {
continue;
}
leaf = pos;
if ((pos += bit_mask) > table_mask)
return 1; /* table overrun */
/* fill all possible lookups of this symbol with the symbol itself */
fill = bit_mask;
while (fill-- > 0) {
table[leaf++] = sym;
}
}
bit_mask >>= 1;
bit_num++;
}
/* if there are any codes longer than nbits */
if (pos != table_mask) {
/* clear the remainder of the table */
for (sym = pos; sym < table_mask; sym++) {
table[sym] = 0;
}
/* give ourselves room for codes to grow by up to 16 more bits */
pos <<= 16;
table_mask <<= 16;
bit_mask = 1 << 15;
while (bit_num <= 16) {
for (sym = 0; sym < nsyms; sym++) {
if (length[sym] != bit_num) {
continue;
}
leaf = pos >> 16;
for (fill = 0; fill < bit_num - nbits; fill++) {
/* if this path hasn't been taken yet, 'allocate' two entries */
if (table[leaf] == 0) {
table[(next_symbol << 1)] = 0;
table[(next_symbol << 1) + 1] = 0;
table[leaf] = next_symbol++;
}
/* follow the path and select either left or right for next bit */
leaf = table[leaf] << 1;
if ((pos >> (15 - fill)) & 1)
leaf++;
}
table[leaf] = sym;
if ((pos += bit_mask) > table_mask)
return 1; /* table overflow */
}
bit_mask >>= 1;
bit_num++;
}
}
/* full table? */
if (pos == table_mask)
return 0;
/* either erroneous table, or all elements are 0 - let's find out. */
for (sym = 0; sym < nsyms; sym++) {
if (length[sym])
return 1;
}
return 0;
}
struct lzx_bits {
uint32_t bb;
int bl;
uint8_t* ip;
};
static int lzx_read_lens(struct lzx_state* pState, uint8_t* lens, uint32_t first, uint32_t last,
struct lzx_bits* lb) {
uint32_t i, j, x, y;
int z;
uint32_t bitbuf = lb->bb;
int bitsleft = lb->bl;
uint8_t* inpos = lb->ip;
uint16_t* hufftbl;
for (x = 0; x < 20; x++) {
READ_BITS(y, 4);
LENTABLE(PRETREE)[x] = y;
}
BUILD_TABLE(PRETREE);
for (x = first; x < last;) {
READ_HUFFSYM(PRETREE, z);
if (z == 17) {
READ_BITS(y, 4);
y += 4;
while (y--)
lens[x++] = 0;
} else if (z == 18) {
READ_BITS(y, 5);
y += 20;
while (y--)
lens[x++] = 0;
} else if (z == 19) {
READ_BITS(y, 1);
y += 4;
READ_HUFFSYM(PRETREE, z);
z = lens[x] - z;
if (z < 0)
z += 17;
while (y--)
lens[x++] = z;
} else {
z = lens[x] - z;
if (z < 0)
z += 17;
lens[x++] = z;
}
}
lb->bb = bitbuf;
lb->bl = bitsleft;
lb->ip = inpos;
return 0;
}
int lzx_decompress(struct lzx_state* pState, const unsigned char* inpos, unsigned char* outpos, int inlen,
int outlen) {
const uint8_t* endinp = inpos + inlen;
uint8_t* window = pState->window;
uint8_t* runsrc, *rundest;
uint16_t* hufftbl; /* used in READ_HUFFSYM macro as chosen decoding table */
uint32_t window_posn = pState->window_posn;
uint32_t window_size = pState->window_size;
uint32_t R0 = pState->R0;
uint32_t R1 = pState->R1;
uint32_t R2 = pState->R2;
uint32_t bitbuf;
int bitsleft;
uint32_t match_offset, i, j, k; /* ijk used in READ_HUFFSYM macro */
struct lzx_bits lb; /* used in READ_LENGTHS macro */
int togo = outlen, this_run, main_element, aligned_bits;
int match_length, length_footer, extra, verbatim_bits;
INIT_BITSTREAM;
/* read header if necessary */
if (!pState->header_read) {
i = j = 0;
READ_BITS(k, 1);
if (k) {
READ_BITS(i, 16);
READ_BITS(j, 16);
}
pState->intel_filesize = (i << 16) | j; /* or 0 if not encoded */
pState->header_read = 1;
}
/* main decoding loop */
while (togo > 0) {
/* last block finished, new block expected */
if (pState->block_remaining == 0) {
if (pState->block_type == LZX_BLOCKTYPE_UNCOMPRESSED) {
if (pState->block_length & 1)
inpos++; /* realign bitstream to word */
INIT_BITSTREAM;
}
READ_BITS(pState->block_type, 3);
READ_BITS(i, 16);
READ_BITS(j, 8);
pState->block_remaining = pState->block_length = (i << 8) | j;
switch (pState->block_type) {
case LZX_BLOCKTYPE_ALIGNED:
for (i = 0; i < 8; i++) {
READ_BITS(j, 3);
LENTABLE(ALIGNED)[i] = j;
}
BUILD_TABLE(ALIGNED);
/* rest of aligned header is same as verbatim */
case LZX_BLOCKTYPE_VERBATIM:
READ_LENGTHS(MAINTREE, 0, 256);
READ_LENGTHS(MAINTREE, 256, pState->main_elements);
BUILD_TABLE(MAINTREE);
if (LENTABLE(MAINTREE)[0xE8] != 0)
pState->intel_started = 1;
READ_LENGTHS(LENGTH, 0, LZX_NUM_SECONDARY_LENGTHS);
BUILD_TABLE(LENGTH);
break;
case LZX_BLOCKTYPE_UNCOMPRESSED:
pState->intel_started = 1; /* because we can't assume otherwise */
ENSURE_BITS(16); /* get up to 16 pad bits into the buffer */
if (bitsleft > 16)
inpos -= 2; /* and align the bitstream! */
R0 = inpos[0] | (inpos[1] << 8) | (inpos[2] << 16) | (inpos[3] << 24);
inpos += 4;
R1 = inpos[0] | (inpos[1] << 8) | (inpos[2] << 16) | (inpos[3] << 24);
inpos += 4;
R2 = inpos[0] | (inpos[1] << 8) | (inpos[2] << 16) | (inpos[3] << 24);
inpos += 4;
break;
default:
return DECR_ILLEGALDATA;
}
}
/* buffer exhaustion check */
if (inpos > endinp) {
/* it's possible to have a file where the next run is less than
* 16 bits in size. In this case, the READ_HUFFSYM() macro used
* in building the tables will exhaust the buffer, so we should
* allow for this, but not allow those accidentally read bits to
* be used (so we check that there are at least 16 bits
* remaining - in this boundary case they aren't really part of
* the compressed data)
*/
if (inpos > (endinp + 2) || bitsleft < 16)
return DECR_ILLEGALDATA;
}
while ((this_run = pState->block_remaining) > 0 && togo > 0) {
if (this_run > togo)
this_run = togo;
togo -= this_run;
pState->block_remaining -= this_run;
/* apply 2^x-1 mask */
window_posn &= window_size - 1;
/* runs can't straddle the window wraparound */
if ((window_posn + this_run) > window_size)
return DECR_DATAFORMAT;
switch (pState->block_type) {
case LZX_BLOCKTYPE_VERBATIM:
while (this_run > 0) {
READ_HUFFSYM(MAINTREE, main_element);
if (main_element < LZX_NUM_CHARS) {
/* literal: 0 to LZX_NUM_CHARS-1 */
window[window_posn++] = main_element;
this_run--;
} else {
/* match: LZX_NUM_CHARS + ((slot<<3) | length_header (3 bits)) */
main_element -= LZX_NUM_CHARS;
match_length = main_element & LZX_NUM_PRIMARY_LENGTHS;
if (match_length == LZX_NUM_PRIMARY_LENGTHS) {
READ_HUFFSYM(LENGTH, length_footer);
match_length += length_footer;
}
match_length += LZX_MIN_MATCH;
match_offset = main_element >> 3;
if (match_offset > 2) {
/* not repeated offset */
if (match_offset != 3) {
extra = extra_bits[match_offset];
READ_BITS(verbatim_bits, extra);
match_offset = position_base[match_offset] - 2 + verbatim_bits;
} else {
match_offset = 1;
}
/* update repeated offset LRU queue */
R2 = R1;
R1 = R0;
R0 = match_offset;
} else if (match_offset == 0) {
match_offset = R0;
} else if (match_offset == 1) {
match_offset = R1;
R1 = R0;
R0 = match_offset;
} else /* match_offset == 2 */ {
match_offset = R2;
R2 = R0;
R0 = match_offset;
}
rundest = window + window_posn;
runsrc = rundest - match_offset;
window_posn += match_length;
if (window_posn > window_size)
return DECR_ILLEGALDATA;
this_run -= match_length;
/* copy any wrapped around source data */
while ((runsrc < window) && (match_length-- > 0)) {
*rundest++ = *(runsrc + window_size);
runsrc++;
}
/* copy match data - no worries about destination wraps */
while (match_length-- > 0)
*rundest++ = *runsrc++;
}
}
break;
case LZX_BLOCKTYPE_ALIGNED:
while (this_run > 0) {
READ_HUFFSYM(MAINTREE, main_element);
if (main_element < LZX_NUM_CHARS) {
/* literal: 0 to LZX_NUM_CHARS-1 */
window[window_posn++] = main_element;
this_run--;
} else {
/* match: LZX_NUM_CHARS + ((slot<<3) | length_header (3 bits)) */
main_element -= LZX_NUM_CHARS;
match_length = main_element & LZX_NUM_PRIMARY_LENGTHS;
if (match_length == LZX_NUM_PRIMARY_LENGTHS) {
READ_HUFFSYM(LENGTH, length_footer);
match_length += length_footer;
}
match_length += LZX_MIN_MATCH;
match_offset = main_element >> 3;
if (match_offset > 2) {
/* not repeated offset */
extra = extra_bits[match_offset];
match_offset = position_base[match_offset] - 2;
if (extra > 3) {
/* verbatim and aligned bits */
extra -= 3;
READ_BITS(verbatim_bits, extra);
match_offset += (verbatim_bits << 3);
READ_HUFFSYM(ALIGNED, aligned_bits);
match_offset += aligned_bits;
} else if (extra == 3) {
/* aligned bits only */
READ_HUFFSYM(ALIGNED, aligned_bits);
match_offset += aligned_bits;
} else if (extra > 0) { /* extra==1, extra==2 */
/* verbatim bits only */
READ_BITS(verbatim_bits, extra);
match_offset += (uint32_t)verbatim_bits;
} else /* extra == 0 */ {
/* ??? */
match_offset = 1;
}
/* update repeated offset LRU queue */
R2 = R1;
R1 = R0;
R0 = match_offset;
} else if (match_offset == 0) {
match_offset = R0;
} else if (match_offset == 1) {
match_offset = R1;
R1 = R0;
R0 = match_offset;
} else /* match_offset == 2 */ {
match_offset = R2;
R2 = R0;
R0 = match_offset;
}
rundest = window + window_posn;
runsrc = rundest - match_offset;
window_posn += (uint32_t)match_length;
if (window_posn > window_size)
return DECR_ILLEGALDATA;
this_run -= match_length;
/* copy any wrapped around source data */
while ((runsrc < window) && (match_length-- > 0)) {
*rundest++ = *(runsrc + window_size);
runsrc++;
}
/* copy match data - no worries about destination wraps */
while (match_length-- > 0)
*rundest++ = *runsrc++;
}
}
break;
case LZX_BLOCKTYPE_UNCOMPRESSED:
if ((inpos + this_run) > endinp)
return DECR_ILLEGALDATA;
memcpy(window + window_posn, inpos, (size_t)this_run);
inpos += this_run;
window_posn += (uint32_t)this_run;
break;
default:
return DECR_ILLEGALDATA; /* might as well */
}
}
}
if (togo != 0)
return DECR_ILLEGALDATA;
memcpy(outpos, window + ((!window_posn) ? window_size : window_posn) - outlen, (size_t)outlen);
pState->window_posn = window_posn;
pState->R0 = R0;
pState->R1 = R1;
pState->R2 = R2;
/* intel E8 decoding */
if ((pState->frames_read++ < 32768) && pState->intel_filesize != 0) {
if (outlen <= 6 || !pState->intel_started) {
pState->intel_curpos += outlen;
} else {
uint8_t* data = outpos;
uint8_t* dataend = data + outlen - 10;
int32_t curpos = pState->intel_curpos;
int32_t filesize = pState->intel_filesize;
int32_t abs_off, rel_off;
pState->intel_curpos = curpos + outlen;
while (data < dataend) {
if (*data++ != 0xE8) {
curpos++;
continue;
}
abs_off = data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24);
if ((abs_off >= -curpos) && (abs_off < filesize)) {
rel_off = (abs_off >= 0) ? abs_off - curpos : abs_off + filesize;
data[0] = (uint8_t)rel_off;
data[1] = (uint8_t)(rel_off >> 8);
data[2] = (uint8_t)(rel_off >> 16);
data[3] = (uint8_t)(rel_off >> 24);
}
data += 4;
curpos += 5;
}
}
}
return DECR_OK;
}

61
thirdparty/lzx/lzx.h vendored Normal file
View File

@@ -0,0 +1,61 @@
// Modified version of the code from Wine:
// https://gitlab.winehq.org/wine/wine/-/blob/fcc40a07909dc7626b6d1e2ec73f823d828a47e8/dlls/itss/lzx.h
/***************************************************************************
* lzx.h - LZX decompression routines *
* ------------------- *
* *
* maintainer: Jed Wing <jedwin@ugcs.caltech.edu> *
* source: modified lzx.c from cabextract v0.5 *
* notes: This file was taken from cabextract v0.5, which was, *
* itself, a modified version of the lzx decompression code *
* from unlzx. *
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. Note that an exemption to this *
* license has been granted by Stuart Caie for the purposes of *
* distribution with chmlib. This does not, to the best of my *
* knowledge, constitute a change in the license of this (the LZX) code *
* in general. *
* *
***************************************************************************/
#ifndef INCLUDED_LZX_H
#define INCLUDED_LZX_H
#ifdef __cplusplus
extern "C" {
#endif
/* return codes */
#define DECR_OK (0)
#define DECR_DATAFORMAT (1)
#define DECR_ILLEGALDATA (2)
#define DECR_NOMEMORY (3)
/* opaque state structure */
struct lzx_state;
/* create an lzx state object */
struct lzx_state* lzx_init(int window);
/* destroy an lzx state object */
void lzx_teardown(struct lzx_state* pState);
/* reset an lzx stream */
void lzx_reset(struct lzx_state* pState);
/* decompress an LZX compressed block */
int lzx_decompress(struct lzx_state* pState, const unsigned char* inpos, unsigned char* outpos, int inlen,
int outlen);
#ifdef __cplusplus
}
#endif
#endif /* INCLUDED_LZX_H */