zone/OpenAssetTools
zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-04-19 15:52:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add data writing for IPaks

Browse Source
This commit is contained in:
Jan 2023-10-15 17:54:10 +02:00
parent 2d0ef40335
commit abbb697d7c
5 changed files with 157 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/ObjCommon/ObjContainer/IPak/IPakTypes.h
View File

@ -16,7 +16,12 @@ namespace ipak_consts
static constexpr size_t IPAK_CHUNK_SIZE = 0x8000;
static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8;
static constexpr uint32_t IPAK_COMMAND_DEFAULT_SIZE = 0x7F00;
static constexpr uint32_t IPAK_COMMAND_UNCOMPRESSED = 0;
static constexpr uint32_t IPAK_COMMAND_COMPRESSED = 1;
static constexpr uint32_t IPAK_COMMAND_SKIP = 0xCF;
static_assert(IPAK_COMMAND_DEFAULT_SIZE <= IPAK_CHUNK_SIZE);
}
typedef uint32_t IPakHash;

6
src/ObjLoading/ObjContainer/IPak/IPak.cpp
View File

@ -70,8 +70,8 @@ class IPak::Impl : public ObjContainerReferenceable
{
IPakSection section{};
m_stream->read(reinterpret_cast<char*>(&section), sizeof section);
if (m_stream->gcount() != sizeof section)
m_stream->read(reinterpret_cast<char*>(&section), sizeof(section));
if (m_stream->gcount() != sizeof(section))
{
printf("Unexpected eof when trying to load section.\n");
return false;
@ -98,7 +98,7 @@ class IPak::Impl : public ObjContainerReferenceable
{
IPakHeader header{};
m_stream->read(reinterpret_cast<char*>(&header), sizeof header);
m_stream->read(reinterpret_cast<char*>(&header), sizeof(header));
if (m_stream->gcount() != sizeof header)
{
printf("Unexpected eof when trying to load header.\n");

152
src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp
View File

@ -2,6 +2,7 @@
#include <algorithm>
#include <iostream>
#include <minilzo.h>
#include <sstream>
#include <zlib.h>
@ -26,8 +27,14 @@ public:
m_data_section_offset(0),
m_data_section_size(0u),
m_index_section_offset(0),
m_branding_section_offset(0)
m_branding_section_offset(0),
m_file_offset(0u),
m_chunk_buffer_window_start(0),
m_current_block{},
m_current_block_header_offset(0)
{
m_decompressed_buffer = std::make_unique<char[]>(ipak_consts::IPAK_CHUNK_SIZE);
m_lzo_work_buffer = std::make_unique<char[]>(LZO1X_1_MEM_COMPRESS);
}
void AddImage(std::string imageName) override
@ -59,9 +66,14 @@ public:
}
}
void AlignToChunkWrite()
void AlignToChunk()
{
Pad(static_cast<size_t>(utils::Align(m_current_offset, 0x8000i64) - m_current_offset));
Pad(static_cast<size_t>(utils::Align(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE)) - m_current_offset));
}
void AlignToBlockHeader()
{
Pad(static_cast<size_t>(utils::Align(m_current_offset, static_cast<int64_t>(sizeof(IPakDataBlockHeader))) - m_current_offset));
}
void WriteHeaderData()
@ -128,6 +140,106 @@ public:
return imageData;
}
void FlushBlock()
{
if (m_current_block_header_offset > 0)
{
const auto previousOffset = m_current_offset;
GoTo(m_current_block_header_offset);
Write(&m_current_block, sizeof(m_current_block));
GoTo(previousOffset);
}
}
void FlushChunk()
{
FlushBlock();
AlignToBlockHeader();
const auto nextChunkOffset = utils::Align(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE));
const auto sizeToSkip = static_cast<size_t>(nextChunkOffset - m_current_offset);
if (sizeToSkip >= sizeof(IPakDataBlockHeader))
{
IPakDataBlockHeader skipBlockHeader{};
skipBlockHeader.countAndOffset.count = 1;
skipBlockHeader.commands[0].compressed = ipak_consts::IPAK_COMMAND_SKIP;
skipBlockHeader.commands[0].size = sizeToSkip - sizeof(IPakDataBlockHeader);
Write(&skipBlockHeader, sizeof(skipBlockHeader));
}
AlignToChunk();
m_chunk_buffer_window_start = m_current_offset;
}
void StartNewBlock()
{
AlignToBlockHeader();
m_current_block_header_offset = m_current_offset;
m_current_block = {};
m_current_block.countAndOffset.offset = static_cast<uint32_t>(m_file_offset);
}
void WriteChunkData(const void* data, const size_t dataSize)
{
auto dataOffset = 0u;
while (dataOffset < dataSize)
{
if (m_current_block.countAndOffset.count >= std::extent_v<decltype(IPakDataBlockHeader::commands)>)
{
FlushBlock();
StartNewBlock();
}
const auto remainingSize = dataSize - dataOffset;
const auto remainingChunkBufferWindowSize = std::max((ipak_consts::IPAK_CHUNK_COUNT_PER_READ * ipak_consts::IPAK_CHUNK_SIZE)
- static_cast<size_t>(m_current_offset - m_chunk_buffer_window_start), 0u);
const auto commandSize = std::min(std::min(remainingSize, ipak_consts::IPAK_COMMAND_DEFAULT_SIZE), remainingChunkBufferWindowSize);
auto writeUncompressed = true;
if (USE_COMPRESSION)
{
auto outLen = static_cast<lzo_uint>(ipak_consts::IPAK_CHUNK_SIZE);
const auto result = lzo1x_1_compress(&static_cast<const unsigned char*>(data)[dataOffset], commandSize, reinterpret_cast<unsigned char*>(m_decompressed_buffer.get()), &outLen,
m_lzo_work_buffer.get());
if (result == LZO_E_OK && outLen < commandSize)
{
writeUncompressed = false;
Write(m_decompressed_buffer.get(), outLen);
const auto currentCommand = m_current_block.countAndOffset.count;
m_current_block.commands[currentCommand].size = static_cast<uint32_t>(outLen);
m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_COMPRESSED;
m_current_block.countAndOffset.count = currentCommand + 1u;
}
}
if (writeUncompressed)
{
Write(data, dataSize);
const auto currentCommand = m_current_block.countAndOffset.count;
m_current_block.commands[currentCommand].size = commandSize;
m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_UNCOMPRESSED;
m_current_block.countAndOffset.count = currentCommand + 1u;
}
dataOffset += commandSize;
}
m_file_offset += dataSize;
}
void StartNewFile()
{
FlushBlock();
StartNewBlock();
m_file_offset = 0u;
m_chunk_buffer_window_start = utils::AlignToPrevious(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE));
}
bool WriteImageData(const std::string& imageName)
{
size_t imageSize;
@ -138,34 +250,39 @@ public:
const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0);
const auto dataHash = static_cast<unsigned>(crc32(0u, reinterpret_cast<const Bytef*>(imageData.get()), imageSize));
IPakIndexEntry indexEntry{};
IPakIndexEntry indexEntry;
indexEntry.key.nameHash = nameHash;
indexEntry.key.dataHash = dataHash & 0x1FFFFFFF;
indexEntry.offset = static_cast<uint32_t>(m_current_offset - m_data_section_offset);
size_t writtenImageSize = 0u;
// TODO: Write image data
StartNewFile();
const auto startOffset = m_current_offset;
WriteChunkData(imageData.get(), imageSize);
const auto writtenImageSize = static_cast<size_t>(m_current_offset - startOffset);
indexEntry.size = writtenImageSize;
m_index_entries.emplace_back(indexEntry);
m_data_section_size += writtenImageSize;
return true;
}
bool WriteDataSection()
{
AlignToChunkWrite();
AlignToChunk();
m_data_section_offset = m_current_offset;
m_data_section_size = 0u;
m_index_entries.reserve(m_images.size());
return std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName)
const auto result = std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName)
{
return WriteImageData(imageName);
});
FlushBlock();
m_data_section_size = static_cast<size_t>(m_current_offset - m_data_section_offset);
return result;
}
static bool CompareIndices(const IPakIndexEntry& entry1, const IPakIndexEntry& entry2)
@ -180,7 +297,7 @@ public:
void WriteIndexSection()
{
AlignToChunkWrite();
AlignToChunk();
m_index_section_offset = m_current_offset;
SortIndexSectionEntries();
@ -191,7 +308,7 @@ public:
void WriteBrandingSection()
{
AlignToChunkWrite();
AlignToChunk();
m_branding_section_offset = m_current_offset;
Write(BRANDING, std::extent_v<decltype(BRANDING)>);
@ -199,7 +316,7 @@ public:
void WriteFileEnding()
{
AlignToChunkWrite();
AlignToChunk();
m_total_size = m_current_offset;
}
@ -207,7 +324,7 @@ public:
{
// We will write the header and sections later since they need complementary data
GoTo(sizeof(IPakHeader) + sizeof(IPakSection) * SECTION_COUNT);
AlignToChunkWrite();
AlignToChunk();
if (!WriteDataSection())
return false;
@ -233,6 +350,13 @@ private:
size_t m_data_section_size;
int64_t m_index_section_offset;
int64_t m_branding_section_offset;
std::unique_ptr<char[]> m_decompressed_buffer;
std::unique_ptr<char[]> m_lzo_work_buffer;
size_t m_file_offset;
int64_t m_chunk_buffer_window_start;
IPakDataBlockHeader m_current_block;
int64_t m_current_block_header_offset;
};
std::unique_ptr<IPakWriter> IPakWriter::Create(std::ostream& stream, ISearchPath* assetSearchPath)

2
src/ObjWriting/ObjContainer/IPak/IPakWriter.h
View File

@ -7,6 +7,8 @@
class IPakWriter
{
public:
static constexpr auto USE_COMPRESSION = true;
IPakWriter() = default;
virtual ~IPakWriter() = default;

10
src/Utils/Utils/Alignment.h
View File

@ -3,10 +3,18 @@
namespace utils
{
template <typename T>
constexpr T Align(T value, T toNext)
constexpr T Align(const T value, const T toNext)
{
if (toNext > 0)
return (value + toNext - 1) / toNext * toNext;
return value;
}
template <typename T>
constexpr T AlignToPrevious(const T value, const T toPrevious)
{
if (toPrevious > 0)
return value / toPrevious * toPrevious;
return value;
}
}