zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2026-05-16 23:11:42 +00:00
Code Activity

Merge pull request #797 from Laupetin/feature/ipak-dumping-without-ff

Browse Source 
feat: ipak dumping without ff
This commit is contained in:
Jan
2026-05-15 14:34:54 +02:00
committed by GitHub
parent a2ab44da39 f7f4deeecc
commit c33282bf37
20 changed files with 590 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
premake5.lua
+2
View File
@@ -136,6 +136,7 @@ include "src/RawTemplater.lua"
include "src/UnlinkerCli.lua"
include "src/Unlinking.lua"
include "src/Utils.lua"
include "src/XMemCompress.lua"
include "src/ZoneCode.lua"
include "src/ZoneCodeGeneratorLib.lua"
include "src/ZoneCodeGenerator.lua"
@@ -156,6 +157,7 @@ group "Components"
Cryptography:project()
Parser:project()
Utils:project()
XMemCompress:project()
ZoneCode:project()
ZoneCodeGeneratorLib:project()
ZoneCommon:project()
src/ImageConverter.lua
+2
View File
@@ -39,12 +39,14 @@ function ImageConverter:project()
self:include(includes)
Utils:include(includes)
Common:include(includes)
ObjLoading:include(includes)
ObjImage:include(includes)
Raw:use()
links:linkto(Utils)
links:linkto(Common)
links:linkto(ObjLoading)
links:linkto(ObjImage)
links:linkall()
end
src/ImageConverter/ImageConverter.cpp
+95 -27
View File
@@ -8,12 +8,13 @@
#include "Image/IwiWriter27.h"
#include "Image/IwiWriter6.h"
#include "Image/IwiWriter8.h"
#include "Image/Texture.h"
#include "ImageConverterArgs.h"
#include "ObjContainer/IPak/IPak.h"
#include "Utils/Logging/Log.h"
#include "Utils/StringUtils.h"
#include <assert.h>
#include <cstring>
#include <filesystem>
#include <format>
#include <fstream>
@@ -27,36 +28,26 @@ namespace
{
constexpr auto EXTENSION_IWI = ".iwi";
constexpr auto EXTENSION_DDS = ".dds";
constexpr auto EXTENSION_IPAK = ".ipak";
constexpr char IWI_MAGIC[3]{'I', 'W', 'i'};
class ImageConverterImpl final : public ImageConverter
const char* DetermineExtensionForBytes(const void* bytes, const size_t byteCount)
{
if (byteCount >= sizeof(IWI_MAGIC) && !memcmp(bytes, IWI_MAGIC, sizeof(IWI_MAGIC)))
return ".iwi";
return "";
}
class ImageConverter
{
public:
ImageConverterImpl()
: m_game_to_convert_to(std::nullopt)
explicit ImageConverter(const ImageConverterArgs& args)
: m_game_to_convert_to(args.m_game_to_convert_to)
{
}
bool Start(const int argc, const char** argv) override
{
con::init();
auto shouldContinue = true;
if (!m_args.ParseArgs(argc, argv, shouldContinue))
return false;
if (!shouldContinue)
return true;
m_game_to_convert_to = m_args.m_game_to_convert_to;
for (const auto& file : m_args.m_files_to_convert)
Convert(file);
return true;
}
private:
void Convert(const std::string& file)
void HandleFile(const std::string& file)
{
const fs::path filePath(file);
auto extension = filePath.extension().string();
@@ -66,10 +57,13 @@ namespace
ConvertIwi(filePath);
else if (extension == EXTENSION_DDS)
ConvertDds(filePath);
else if (extension == EXTENSION_IPAK)
ExtractIpak(filePath);
else
con::error("Unsupported extension {}", extension);
}
private:
bool ConvertIwi(const fs::path& iwiPath)
{
std::ifstream file(iwiPath, std::ios::in | std::ios::binary);
@@ -195,6 +189,66 @@ namespace
return true;
}
bool ExtractIpak(const fs::path& ipakPath)
{
auto file = std::make_unique<std::ifstream>(ipakPath, std::ios::in | std::ios::binary);
if (!file->is_open())
{
con::error("Failed to open ipak {}", ipakPath.string());
return false;
}
auto ipak = IIPak::Create(ipakPath.string(), std::move(file));
if (!ipak->Initialize())
{
con::error("Failed to read ipak {}", ipakPath.string());
return false;
}
const auto outDir = fs::absolute(ipakPath).parent_path() / ipakPath.filename().replace_extension();
fs::create_directories(outDir);
for (const auto& indexEntry : ipak->GetIndexEntries())
{
const auto baseFileName = std::format("{:0>6x}_{:0>6x}", indexEntry.key.dataHash, indexEntry.key.nameHash);
auto entryStream = ipak->GetEntryStream(indexEntry.key.nameHash, indexEntry.key.dataHash);
if (!entryStream)
{
con::error("Failed to open entry stream for {}", baseFileName);
return false;
}
char buffer[0x2000];
entryStream->read(buffer, sizeof(buffer));
auto readCount = entryStream->gcount();
const auto extension = DetermineExtensionForBytes(buffer, static_cast<size_t>(readCount));
const auto fileNameWithExtension = std::format("{}{}", baseFileName, extension);
std::ofstream outFile(outDir / fileNameWithExtension, std::ios::out | std::ios::binary);
if (!outFile.is_open())
{
con::error("Failed to open ipak file {}", fileNameWithExtension);
return false;
}
while (readCount > 0)
{
outFile.write(buffer, readCount);
entryStream->read(buffer, sizeof(buffer));
readCount = entryStream->gcount();
}
entryStream->close();
outFile.close();
con::info("Dumped {}", fileNameWithExtension);
}
return true;
}
ImageConverterArgs m_args;
std::optional<GameId> m_game_to_convert_to;
DdsWriter m_dds_writer;
@@ -202,7 +256,21 @@ namespace
};
} // namespace
std::unique_ptr<ImageConverter> ImageConverter::Create()
bool RunImageConverter(const int argc, const char** argv)
{
return std::make_unique<ImageConverterImpl>();
con::init();
auto shouldContinue = true;
ImageConverterArgs args;
if (!args.ParseArgs(argc, argv, shouldContinue))
return false;
if (!shouldContinue)
return true;
ImageConverter imageConverter(args);
for (const auto& file : args.m_files_to_convert)
imageConverter.HandleFile(file);
return true;
}
src/ImageConverter/ImageConverter.h
+1 -22
View File
@@ -1,24 +1,3 @@
#pragma once
#include <memory>
class ImageConverter
{
public:
ImageConverter() = default;
virtual ~ImageConverter() = default;
ImageConverter(const ImageConverter& other) = delete;
ImageConverter(ImageConverter&& other) noexcept = delete;
ImageConverter& operator=(const ImageConverter& other) = delete;
ImageConverter& operator=(ImageConverter&& other) noexcept = delete;
/**
* \brief Starts the ImageConverter application logic.
* \param argc The amount of command line arguments specified.
* \param argv The command line arguments.
* \return \c true if the application was successful or \c false if an error occurred.
*/
virtual bool Start(int argc, const char** argv) = 0;
static std::unique_ptr<ImageConverter> Create();
};
bool RunImageConverter(int argc, const char** argv);
src/ImageConverter/main.cpp
+1 -3
View File
@@ -2,7 +2,5 @@
int main(const int argc, const char** argv)
{
const auto imageConverter = ImageConverter::Create();
return imageConverter->Start(argc, argv) ? 0 : 1;
return RunImageConverter(argc, argv) ? 0 : 1;
}
src/ObjCommon/ObjContainer/IPak/IPakTypes.h
+14 -3
View File
@@ -6,7 +6,8 @@
namespace ipak_consts
{
static constexpr uint32_t IPAK_MAGIC = utils::MakeMagic32('K', 'A', 'P', 'I');
static constexpr uint32_t IPAK_MAGIC_LITTLE_ENDIAN = utils::MakeMagic32('K', 'A', 'P', 'I');
static constexpr uint32_t IPAK_MAGIC_BIG_ENDIAN = utils::MakeMagic32('I', 'P', 'A', 'K');
static constexpr uint32_t IPAK_VERSION = 0x50000;
static constexpr uint32_t IPAK_INDEX_SECTION = 1;
@@ -60,20 +61,30 @@ struct IPakIndexEntry
uint32_t size;
};
struct IPakDataBlockCountAndOffset
union IPakDataBlockCountAndOffset
{
struct
{
uint32_t offset : 24;
uint32_t count : 8;
};
uint32_t raw;
};
static_assert(sizeof(IPakDataBlockCountAndOffset) == 4);
struct IPakDataBlockCommand
union IPakDataBlockCommand
{
struct
{
uint32_t size : 24;
uint32_t compressed : 8;
};
uint32_t raw;
};
static_assert(sizeof(IPakDataBlockCommand) == 4);
struct IPakDataBlockHeader
src/ObjCompiling/Image/IPak/IPakCreator.cpp
+1 -1
View File
@@ -100,7 +100,7 @@ namespace
{
GoTo(0);
const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC,
const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN,
.version = ipak_consts::IPAK_VERSION,
.size = static_cast<uint32_t>(m_total_size),
.sectionCount = SECTION_COUNT};
src/ObjLoading.lua
+2
View File
@@ -16,6 +16,7 @@ function ObjLoading:link(links)
links:linkto(Utils)
links:linkto(ObjCommon)
links:linkto(ObjImage)
links:linkto(XMemCompress)
links:linkto(ZoneCommon)
links:linkto(minilzo)
links:linkto(minizip)
@@ -58,6 +59,7 @@ function ObjLoading:project()
self:include(includes)
Cryptography:include(includes)
Utils:include(includes)
XMemCompress:include(includes)
minilzo:include(includes)
minizip:include(includes)
zlib:include(includes)
src/ObjLoading/ObjContainer/IPak/IPak.cpp
+47 -8
View File
@@ -2,8 +2,10 @@
#include "IPakStreamManager.h"
#include "ObjContainer/IPak/IPakTypes.h"
#include "Utils/Endianness.h"
#include "Utils/Logging/Log.h"
#include <concepts>
#include <filesystem>
#include <format>
#include <iostream>
@@ -37,9 +39,10 @@ namespace
: m_path(std::move(path)),
m_stream(std::move(stream)),
m_initialized(false),
m_little_endian(true),
m_index_section(nullptr),
m_data_section(nullptr),
m_stream_manager(*m_stream)
m_stream_manager(nullptr)
{
}
@@ -55,7 +58,7 @@ namespace
return true;
}
[[nodiscard]] std::unique_ptr<iobjstream> GetEntryStream(const Hash nameHash, const Hash dataHash) const override
[[nodiscard]] std::unique_ptr<iobjstream> GetEntryStream(const IPakHash nameHash, const IPakHash dataHash) const override
{
const IPakIndexEntryKey wantedKey{
{.dataHash = dataHash, .nameHash = nameHash}
@@ -65,7 +68,7 @@ namespace
{
if (entry.key.combinedKey == wantedKey.combinedKey)
{
return m_stream_manager.OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
return m_stream_manager->OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
}
else if (entry.key.combinedKey > wantedKey.combinedKey)
{
@@ -77,6 +80,11 @@ namespace
return nullptr;
}
[[nodiscard]] const std::vector<IPakIndexEntry>& GetIndexEntries() const override
{
return m_index_entries;
}
std::string GetName() override
{
return fs::path(m_path).filename().replace_extension("").string();
@@ -97,7 +105,11 @@ namespace
return false;
}
m_index_entries.push_back(indexEntry);
SwapBytesIfNecessary(indexEntry.key.combinedKey);
SwapBytesIfNecessary(indexEntry.offset);
SwapBytesIfNecessary(indexEntry.size);
m_index_entries.emplace_back(indexEntry);
}
std::ranges::sort(m_index_entries,
@@ -120,6 +132,11 @@ namespace
return false;
}
SwapBytesIfNecessary(section.type);
SwapBytesIfNecessary(section.offset);
SwapBytesIfNecessary(section.size);
SwapBytesIfNecessary(section.itemCount);
switch (section.type)
{
case ipak_consts::IPAK_INDEX_SECTION:
@@ -148,18 +165,31 @@ namespace
return false;
}
if (header.magic != ipak_consts::IPAK_MAGIC)
if (header.magic == ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN)
{
m_little_endian = true;
m_stream_manager = IPakStreamManager::Create(*m_stream, true);
}
else if (header.magic == ipak_consts::IPAK_MAGIC_BIG_ENDIAN)
{
m_little_endian = false;
m_stream_manager = IPakStreamManager::Create(*m_stream, false);
}
else
{
con::error("Invalid ipak magic '{:#x}'.", header.magic);
return false;
}
SwapBytesIfNecessary(header.version);
if (header.version != ipak_consts::IPAK_VERSION)
{
con::error("Unsupported ipak version '{}'.", header.version);
return false;
}
SwapBytesIfNecessary(header.size);
SwapBytesIfNecessary(header.sectionCount);
for (unsigned section = 0; section < header.sectionCount; section++)
{
if (!ReadSection())
@@ -184,17 +214,26 @@ namespace
return true;
}
template<std::integral T> void SwapBytesIfNecessary(T& value)
{
if (m_little_endian)
value = endianness::FromLittleEndian(value);
else
value = endianness::FromBigEndian(value);
}
std::string m_path;
std::unique_ptr<std::istream> m_stream;
bool m_initialized;
bool m_little_endian;
std::unique_ptr<IPakSection> m_index_section;
std::unique_ptr<IPakSection> m_data_section;
std::vector<IPakIndexEntry> m_index_entries;
IPakStreamManager m_stream_manager;
std::unique_ptr<IPakStreamManager> m_stream_manager;
};
} // namespace
@@ -203,12 +242,12 @@ std::unique_ptr<IIPak> IIPak::Create(std::string path, std::unique_ptr<std::istr
return std::make_unique<IPak>(std::move(path), std::move(stream));
}
IIPak::Hash IIPak::HashString(const std::string& str)
IPakHash IIPak::HashString(const std::string& str)
{
return R_HashString(str.c_str(), 0);
}
IIPak::Hash IIPak::HashData(const void* data, const size_t dataSize)
IPakHash IIPak::HashData(const void* data, const size_t dataSize)
{
return crc32(0, static_cast<const Bytef*>(data), static_cast<unsigned>(dataSize));
}
src/ObjLoading/ObjContainer/IPak/IPak.h
+7 -5
View File
@@ -1,5 +1,6 @@
#pragma once
#include "ObjContainer/IPak/IPakTypes.h"
#include "ObjContainer/ObjContainerReferenceable.h"
#include "ObjContainer/ObjContainerRepository.h"
#include "Utils/ObjStream.h"
@@ -14,19 +15,20 @@ class IIPak : public ObjContainerReferenceable
{
public:
static ObjContainerRepository<IIPak, Zone> Repository;
typedef std::uint32_t Hash;
IIPak() = default;
virtual ~IIPak() = default;
~IIPak() override = default;
IIPak(const IIPak& other) = default;
IIPak(IIPak&& other) noexcept = default;
IIPak& operator=(const IIPak& other) = default;
IIPak& operator=(IIPak&& other) noexcept = default;
virtual bool Initialize() = 0;
[[nodiscard]] virtual std::unique_ptr<iobjstream> GetEntryStream(Hash nameHash, Hash dataHash) const = 0;
[[nodiscard]] virtual std::unique_ptr<iobjstream> GetEntryStream(IPakHash nameHash, IPakHash dataHash) const = 0;
[[nodiscard]] virtual const std::vector<IPakIndexEntry>& GetIndexEntries() const = 0;
static std::unique_ptr<IIPak> Create(std::string path, std::unique_ptr<std::istream> stream);
static Hash HashString(const std::string& str);
static Hash HashData(const void* data, size_t dataSize);
static IPakHash HashString(const std::string& str);
static IPakHash HashData(const void* data, size_t dataSize);
};
src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp
+41 -8
View File
@@ -7,12 +7,18 @@
#include <cassert>
#include <cstring>
#include <minilzo.h>
#include <utility>
using namespace ipak_consts;
IPakEntryReadStream::IPakEntryReadStream(
std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, const int64_t startOffset, const size_t entrySize)
IPakEntryReadStream::IPakEntryReadStream(std::istream& stream,
const bool isLittleEndian,
IPakStreamManagerActions* streamManagerActions,
uint8_t* chunkBuffer,
const int64_t startOffset,
const size_t entrySize)
: m_chunk_buffer(chunkBuffer),
m_little_endian(isLittleEndian),
m_stream(stream),
m_stream_manager_actions(streamManagerActions),
m_file_offset(0),
@@ -134,7 +140,7 @@ bool IPakEntryReadStream::ValidateBlockHeader(const IPakDataBlockHeader* blockHe
}
// We expect the current file to be continued where we left off
if (static_cast<int64_t>(blockHeader->countAndOffset.offset) != m_file_head)
if (std::cmp_not_equal(blockHeader->countAndOffset.offset, m_file_head))
{
// A matching offset is only relevant if a command contains data
for (unsigned currentCommand = 0; currentCommand < blockHeader->countAndOffset.count; currentCommand++)
@@ -181,23 +187,33 @@ bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(const IPakDataBl
bool IPakEntryReadStream::NextBlock()
{
m_pos = AlignForward<int64_t>(m_pos, sizeof(IPakDataBlockHeader));
if (m_pos >= m_end_pos)
return false;
m_pos = AlignForward<int64_t>(m_pos, sizeof(IPakDataBlockHeader));
const auto chunkStartPos = AlignBackwards<int64_t>(m_pos, IPAK_CHUNK_SIZE);
const auto blockOffsetInChunk = static_cast<size_t>(m_pos - chunkStartPos);
auto estimatedChunksToRead = AlignForward(m_entry_size - static_cast<size_t>(m_pos - m_base_pos), IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE;
if (estimatedChunksToRead > IPAK_CHUNK_COUNT_PER_READ)
estimatedChunksToRead = IPAK_CHUNK_COUNT_PER_READ;
estimatedChunksToRead = std::min(estimatedChunksToRead, IPAK_CHUNK_COUNT_PER_READ);
if (!SetChunkBufferWindow(chunkStartPos, estimatedChunksToRead))
return false;
m_current_block = reinterpret_cast<IPakDataBlockHeader*>(&m_chunk_buffer[blockOffsetInChunk]);
SwapBytesIfNecessary(m_current_block->countAndOffset.raw);
for (auto& command : m_current_block->commands)
{
if (!m_little_endian)
command.raw &= 0xFFFFFFDF; // ? idk, the game seems to do this? halp
SwapBytesIfNecessary(command.raw);
auto size = command.size;
command.size = size;
}
if (!ValidateBlockHeader(m_current_block))
return false;
@@ -231,6 +247,23 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com
m_current_command_offset = 0;
m_file_head += static_cast<int64_t>(outputSize);
}
else if (compressed == 2)
{
m_xmemdecompress_context.Reset();
const auto maybeDecompressSize = m_xmemdecompress_context.Process(
&m_chunk_buffer[m_pos - m_buffer_start_pos], static_cast<int>(commandSize), m_decompress_buffer, sizeof(m_decompress_buffer));
if (!maybeDecompressSize.has_value())
{
con::error("Decompressing block with XMemDecompress failed!");
return false;
}
m_current_command_buffer = m_decompress_buffer;
m_current_command_length = *maybeDecompressSize;
m_current_command_offset = 0;
m_file_head += static_cast<int64_t>(*maybeDecompressSize);
}
else
{
// Do not process data but instead skip specified commandSize
@@ -324,7 +357,7 @@ std::streambuf::int_type IPakEntryReadStream::uflow()
return EOF;
}
std::streamsize IPakEntryReadStream::xsgetn(char* ptr, const std::streamsize count)
std::streamsize IPakEntryReadStream::xsgetn(char* ptr, std::streamsize count)
{
auto* destBuffer = reinterpret_cast<uint8_t*>(ptr);
std::streamsize countRead = 0;
src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h
+50 -34
View File
@@ -2,37 +2,32 @@
#include "IPakStreamManager.h"
#include "ObjContainer/IPak/IPakTypes.h"
#include "Utils/Endianness.h"
#include "Utils/ObjStream.h"
#include "XMemDecompress.h"
#include <concepts>
#include <istream>
class IPakEntryReadStream final : public objbuf
{
static constexpr size_t IPAK_DECOMPRESS_BUFFER_SIZE = 0x8000;
public:
IPakEntryReadStream(
std::istream& stream, bool isLittleEndian, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
~IPakEntryReadStream() override;
uint8_t* m_chunk_buffer;
[[nodiscard]] bool is_open() const override;
bool close() override;
std::istream& m_stream;
IPakStreamManagerActions* m_stream_manager_actions;
int64_t m_file_offset;
int64_t m_file_head;
size_t m_entry_size;
uint8_t m_decompress_buffer[IPAK_DECOMPRESS_BUFFER_SIZE];
IPakDataBlockHeader* m_current_block;
unsigned m_next_command;
uint8_t* m_current_command_buffer;
size_t m_current_command_length;
size_t m_current_command_offset;
int64_t m_pos;
int64_t m_base_pos;
int64_t m_end_pos;
int64_t m_buffer_start_pos;
int64_t m_buffer_end_pos;
protected:
std::streamsize showmanyc() override;
int_type underflow() override;
int_type uflow() override;
std::streamsize xsgetn(char* ptr, std::streamsize count) override;
pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode mode) override;
pos_type seekpos(pos_type pos, std::ios_base::openmode mode) override;
private:
template<typename T> static T AlignForward(const T num, const T alignTo)
{
return (num + alignTo - 1) / alignTo * alignTo;
@@ -43,6 +38,14 @@ class IPakEntryReadStream final : public objbuf
return num / alignTo * alignTo;
}
template<std::integral T> void SwapBytesIfNecessary(T& value)
{
if (m_little_endian)
value = endianness::FromLittleEndian(value);
else
value = endianness::FromBigEndian(value);
}
/**
* \brief Reads the specified chunks from disk.
* \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data.
@@ -95,18 +98,31 @@ class IPakEntryReadStream final : public objbuf
*/
bool AdvanceStream();
public:
IPakEntryReadStream(std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
~IPakEntryReadStream() override;
static constexpr size_t IPAK_DECOMPRESS_BUFFER_SIZE = 0x8000;
[[nodiscard]] bool is_open() const override;
bool close() override;
uint8_t* m_chunk_buffer;
protected:
std::streamsize showmanyc() override;
int_type underflow() override;
int_type uflow() override;
std::streamsize xsgetn(char* ptr, std::streamsize count) override;
pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode mode) override;
pos_type seekpos(pos_type pos, std::ios_base::openmode mode) override;
bool m_little_endian;
XMemDecompressContext m_xmemdecompress_context;
std::istream& m_stream;
IPakStreamManagerActions* m_stream_manager_actions;
int64_t m_file_offset;
int64_t m_file_head;
size_t m_entry_size;
uint8_t m_decompress_buffer[IPAK_DECOMPRESS_BUFFER_SIZE];
IPakDataBlockHeader* m_current_block;
unsigned m_next_command;
uint8_t* m_current_command_buffer;
size_t m_current_command_length;
size_t m_current_command_offset;
int64_t m_pos;
int64_t m_base_pos;
int64_t m_end_pos;
int64_t m_buffer_start_pos;
int64_t m_buffer_end_pos;
};
src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp
+23 -34
View File
@@ -8,10 +8,8 @@
using namespace ipak_consts;
class IPakStreamManager::Impl final : public IPakStreamManagerActions
namespace
{
static constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
class ChunkBuffer
{
public:
@@ -32,25 +30,19 @@ class IPakStreamManager::Impl final : public IPakStreamManagerActions
}
};
std::istream& m_stream;
std::mutex m_read_mutex;
std::mutex m_stream_mutex;
std::vector<ManagedStream> m_open_streams;
std::vector<ChunkBuffer*> m_chunk_buffers;
constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
class IPakStreamManagerImpl final : public IPakStreamManager, public IPakStreamManagerActions
{
public:
explicit Impl(std::istream& stream)
: m_stream(stream)
IPakStreamManagerImpl(std::istream& stream, const bool isLittleEndian)
: m_stream(stream),
m_little_endian(isLittleEndian)
{
m_chunk_buffers.push_back(new ChunkBuffer());
}
Impl(const Impl& other) = delete;
Impl(Impl&& other) noexcept = delete;
virtual ~Impl()
~IPakStreamManagerImpl() override
{
m_stream_mutex.lock();
@@ -63,10 +55,7 @@ public:
m_stream_mutex.unlock();
}
Impl& operator=(const Impl& other) = delete;
Impl& operator=(Impl&& other) noexcept = delete;
std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length)
std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length) override
{
m_stream_mutex.lock();
@@ -85,7 +74,7 @@ public:
else
reservedChunkBuffer = *freeChunkBuffer;
auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, this, reservedChunkBuffer->m_buffer, startPosition, length);
auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, m_little_endian, this, reservedChunkBuffer->m_buffer, startPosition, length);
reservedChunkBuffer->m_using_stream = ipakEntryStream.get();
@@ -137,20 +126,20 @@ public:
m_stream_mutex.unlock();
}
private:
std::istream& m_stream;
bool m_little_endian;
std::mutex m_read_mutex;
std::mutex m_stream_mutex;
std::vector<ManagedStream> m_open_streams;
std::vector<ChunkBuffer*> m_chunk_buffers;
};
} // namespace
IPakStreamManager::IPakStreamManager(std::istream& stream)
: m_impl(new Impl(stream))
std::unique_ptr<IPakStreamManager> IPakStreamManager::Create(std::istream& stream, const bool isLittleEndian)
{
}
IPakStreamManager::~IPakStreamManager()
{
delete m_impl;
m_impl = nullptr;
}
std::unique_ptr<iobjstream> IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) const
{
return m_impl->OpenStream(startPosition, length);
return std::make_unique<IPakStreamManagerImpl>(stream, isLittleEndian);
}
src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h
+5 -10
View File
@@ -4,6 +4,7 @@
#include <cstdint>
#include <istream>
#include <memory>
#include <mutex>
class IPakStreamManagerActions
@@ -17,17 +18,11 @@ public:
class IPakStreamManager
{
class Impl;
Impl* m_impl;
public:
explicit IPakStreamManager(std::istream& stream);
IPakStreamManager(const IPakStreamManager& other) = delete;
IPakStreamManager(IPakStreamManager&& other) noexcept = delete;
~IPakStreamManager();
IPakStreamManager() = default;
virtual ~IPakStreamManager() = default;
IPakStreamManager& operator=(const IPakStreamManager& other) = delete;
IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete;
static std::unique_ptr<IPakStreamManager> Create(std::istream& stream, bool isLittleEndian);
[[nodiscard]] std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) const;
[[nodiscard]] virtual std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) = 0;
};
src/XMemCompress.lua
+49
View File
@@ -0,0 +1,49 @@
XMemCompress = {}
function XMemCompress:include(includes)
if includes:handle(self:name()) then
includedirs {
path.join(ProjectFolder(), "XMemCompress")
}
end
end
function XMemCompress:link(links)
links:add(self:name())
links:linkto(Utils)
links:linkto(lzx)
end
function XMemCompress:use()
end
function XMemCompress:name()
return "XMemCompress"
end
function XMemCompress:project()
local folder = ProjectFolder()
local includes = Includes:create()
project(self:name())
targetdir(TargetDirectoryLib)
location "%{wks.location}/src/%{prj.name}"
kind "StaticLib"
language "C++"
files {
path.join(folder, "XMemCompress/**.h"),
path.join(folder, "XMemCompress/**.cpp")
}
vpaths {
["*"] = {
path.join(folder, "XMemCompress")
}
}
self:include(includes)
Utils:include(includes)
lzx:include(includes)
end
src/XMemCompress/XMemDecompress.cpp
+141
View File
@@ -0,0 +1,141 @@
#include "XMemDecompress.h"
#include "Utils/Logging/Log.h"
#include <format>
#include <lzx.h>
namespace
{
uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize)
{
const auto value = input[offset];
offset++;
remainingSize--;
return value;
}
uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte)
{
return static_cast<uint16_t>(static_cast<uint16_t>(static_cast<uint16_t>(highByte) << 8u) | static_cast<uint16_t>(lowByte));
}
void LogErrorHeaderSpace(size_t remainingInputSize)
{
con::error("XMemDecompress: Not enough data for header: {}", remainingInputSize);
}
} // namespace
XMemDecompressContext::XMemDecompressContext()
: m_lzx_state(lzx_init(17))
{
}
XMemDecompressContext::~XMemDecompressContext()
{
if (m_lzx_state)
lzx_teardown(static_cast<lzx_state*>(m_lzx_state));
}
XMemDecompressContext::XMemDecompressContext(XMemDecompressContext&& other) noexcept
: m_lzx_state(other.m_lzx_state)
{
other.m_lzx_state = nullptr;
}
XMemDecompressContext& XMemDecompressContext::operator=(XMemDecompressContext&& other) noexcept
{
m_lzx_state = other.m_lzx_state;
other.m_lzx_state = nullptr;
return *this;
}
void XMemDecompressContext::Reset() const
{
lzx_reset(static_cast<lzx_state*>(m_lzx_state));
}
std::optional<size_t> XMemDecompressContext::Process(const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) const
{
size_t curInputOffset = 0uz;
size_t curInputSize = inputLength;
size_t curOutputOffset = 0uz;
size_t curOutputSize = outputBufferSize;
uint8_t lowByte;
uint16_t dstSize, srcSize;
while (curInputSize > 0)
{
uint8_t highByte = NextByte(input, curInputOffset, curInputSize);
uint8_t suffixSize;
if (highByte == 0xFF) // magic number: output is smaller than 0x8000
{
if (curInputSize < 4)
{
LogErrorHeaderSpace(curInputSize);
return std::nullopt;
}
highByte = NextByte(input, curInputOffset, curInputSize);
lowByte = NextByte(input, curInputOffset, curInputSize);
dstSize = CombineHighLow(highByte, lowByte);
highByte = NextByte(input, curInputOffset, curInputSize);
lowByte = NextByte(input, curInputOffset, curInputSize);
srcSize = CombineHighLow(highByte, lowByte);
// The game seems to skip a 5 byte suffix after these blocks, not sure why.
suffixSize = 5u;
}
else
{
if (curInputSize < 1)
{
LogErrorHeaderSpace(curInputSize);
return std::nullopt;
}
dstSize = 0x8000u;
lowByte = NextByte(input, curInputOffset, curInputSize);
srcSize = CombineHighLow(highByte, lowByte);
suffixSize = 0u;
}
if (srcSize == 0 || dstSize == 0)
{
// Other implementations do not handle this as a failure, game code suggests otherwise though
con::error("XMemDecompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize);
return std::nullopt;
}
if (static_cast<size_t>(srcSize) + suffixSize > curInputSize)
{
con::error("XMemDecompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize);
return std::nullopt;
}
if (dstSize > curOutputSize)
{
con::error("XMemDecompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize);
return std::nullopt;
}
auto ret = lzx_decompress(static_cast<lzx_state*>(m_lzx_state), &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize);
curInputOffset += srcSize + suffixSize;
curInputSize -= (srcSize + suffixSize);
curOutputOffset += dstSize;
curOutputSize -= srcSize;
if (ret != DECR_OK)
{
con::error("XMemDecompress: lzx decompression failed: {}", ret);
return std::nullopt;
}
}
return curOutputOffset;
}
src/XMemCompress/XMemDecompress.h
+21
View File
@@ -0,0 +1,21 @@
#pragma once
#include <cstdint>
#include <optional>
class XMemDecompressContext
{
public:
XMemDecompressContext();
~XMemDecompressContext();
XMemDecompressContext(const XMemDecompressContext& other) = delete;
XMemDecompressContext(XMemDecompressContext&& other) noexcept;
XMemDecompressContext& operator=(const XMemDecompressContext& other) = delete;
XMemDecompressContext& operator=(XMemDecompressContext&& other) noexcept;
void Reset() const;
std::optional<size_t> Process(const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) const;
private:
void* m_lzx_state;
};
src/ZoneCommon.lua
+2 -2
View File
@@ -6,6 +6,7 @@ function ZoneCommon:include(includes)
path.join(ProjectFolder(), "ZoneCommon")
}
Utils:include(includes)
XMemCompress:include(includes)
Common:include(includes)
ObjCommon:include(includes)
Parser:include(includes)
@@ -22,7 +23,7 @@ function ZoneCommon:link(links)
links:linkto(ObjCommon)
links:linkto(Parser)
links:linkto(Utils)
links:linkto(lzx)
links:linkto(XMemCompress)
ZoneCode:use()
end
@@ -58,7 +59,6 @@ function ZoneCommon:project()
}
self:include(includes)
lzx:include(includes)
ZoneCode:include(includes)
ZoneCode:use()
src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.cpp
+8 -116
View File
@@ -2,133 +2,25 @@
#include "Utils/Logging/Log.h"
#include <cstring>
#include <format>
#include <iostream>
#include <lzx.h>
namespace
{
uint8_t NextByte(const uint8_t* input, size_t& offset, size_t& remainingSize)
{
const auto value = input[offset];
offset++;
remainingSize--;
return value;
}
uint16_t CombineHighLow(const uint8_t highByte, const uint8_t lowByte)
{
return static_cast<uint16_t>(static_cast<uint16_t>(static_cast<uint16_t>(highByte) << 8u) | static_cast<uint16_t>(lowByte));
}
void LogErrorHeaderSpace(size_t remainingInputSize)
{
con::error("XMemCompress: Not enough data for header: {}", remainingInputSize);
}
} // namespace
XChunkProcessorLzxDecompress::XChunkProcessorLzxDecompress(const unsigned streamCount)
: m_lzx_states(streamCount)
: m_xmemdecompress_contexts(streamCount)
{
// T6 uses 17 for window bits
for (auto& lzxState : m_lzx_states)
lzxState = lzx_init(17);
}
XChunkProcessorLzxDecompress::~XChunkProcessorLzxDecompress()
{
for (auto* lzxState : m_lzx_states)
lzx_teardown(static_cast<lzx_state*>(lzxState));
}
size_t XChunkProcessorLzxDecompress::Process(
const unsigned streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
{
auto* state = static_cast<lzx_state*>(m_lzx_states[streamNumber]);
const auto& xMemDecompress = m_xmemdecompress_contexts[streamNumber];
// lzx state is reset before each chunk
lzx_reset(state);
xMemDecompress.Reset();
size_t curInputOffset = 0uz;
size_t curInputSize = inputLength;
size_t curOutputOffset = 0uz;
size_t curOutputSize = outputBufferSize;
uint8_t lowByte;
uint16_t dstSize, srcSize;
while (curInputSize > 0)
const auto maybeDecompressedSize = xMemDecompress.Process(input, inputLength, output, outputBufferSize);
if (!maybeDecompressedSize.has_value())
{
uint8_t highByte = NextByte(input, curInputOffset, curInputSize);
uint8_t suffixSize;
if (highByte == 0xFF) // magic number: output is smaller than 0x8000
{
if (curInputSize < 4)
{
LogErrorHeaderSpace(curInputSize);
return curOutputOffset;
con::error("Failed to decompress xchunk with XMemDecompress");
return 0;
}
highByte = NextByte(input, curInputOffset, curInputSize);
lowByte = NextByte(input, curInputOffset, curInputSize);
dstSize = CombineHighLow(highByte, lowByte);
highByte = NextByte(input, curInputOffset, curInputSize);
lowByte = NextByte(input, curInputOffset, curInputSize);
srcSize = CombineHighLow(highByte, lowByte);
// The game seems to skip a 5 byte suffix after these blocks, not sure why.
suffixSize = 5u;
}
else
{
if (curInputSize < 1)
{
LogErrorHeaderSpace(curInputSize);
return curOutputOffset;
}
dstSize = 0x8000u;
lowByte = NextByte(input, curInputOffset, curInputSize);
srcSize = CombineHighLow(highByte, lowByte);
suffixSize = 0u;
}
if (srcSize == 0 || dstSize == 0)
{
// Other implementations do not handle this as a failure, game code suggests otherwise though
con::error("XMemCompress: EOF: {} {}, {}", srcSize, dstSize, curInputSize);
return curOutputOffset;
}
if (static_cast<size_t>(srcSize) + suffixSize > curInputSize)
{
con::error("XMemCompress: block size bigger than remaining data: {} > {}", srcSize, curInputSize);
return curOutputOffset;
}
if (dstSize > curOutputSize)
{
con::error("XMemCompress: output size bigger than remaining data: {} > {}", dstSize, curOutputSize);
return curOutputOffset;
}
auto ret = lzx_decompress(state, &input[curInputOffset], &output[curOutputOffset], srcSize, dstSize);
curInputOffset += srcSize + suffixSize;
curInputSize -= (srcSize + suffixSize);
curOutputOffset += dstSize;
curOutputSize -= srcSize;
if (ret != DECR_OK)
{
con::error("XMemCompress: lzx decompression failed: {}", ret);
return curOutputOffset;
}
}
return curOutputOffset;
return *maybeDecompressedSize;
}
src/ZoneCommon/Zone/XChunk/XChunkProcessorLzxDecompress.h
+2 -2
View File
@@ -1,6 +1,7 @@
#pragma once
#include "IXChunkProcessor.h"
#include "XMemDecompress.h"
#include <vector>
@@ -8,9 +9,8 @@ class XChunkProcessorLzxDecompress final : public IXChunkProcessor
{
public:
explicit XChunkProcessorLzxDecompress(unsigned streamCount);
~XChunkProcessorLzxDecompress();
size_t Process(unsigned streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
private:
std::vector<void*> m_lzx_states;
std::vector<XMemDecompressContext> m_xmemdecompress_contexts;
};