diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index 94982262..3d516821 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -16,7 +16,12 @@ namespace ipak_consts static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; + static constexpr uint32_t IPAK_COMMAND_DEFAULT_SIZE = 0x7F00; + static constexpr uint32_t IPAK_COMMAND_UNCOMPRESSED = 0; + static constexpr uint32_t IPAK_COMMAND_COMPRESSED = 1; static constexpr uint32_t IPAK_COMMAND_SKIP = 0xCF; + + static_assert(IPAK_COMMAND_DEFAULT_SIZE <= IPAK_CHUNK_SIZE); } typedef uint32_t IPakHash; diff --git a/src/ObjLoading/ObjContainer/IPak/IPak.cpp b/src/ObjLoading/ObjContainer/IPak/IPak.cpp index 953cc81e..c8d6fff1 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPak.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPak.cpp @@ -70,8 +70,8 @@ class IPak::Impl : public ObjContainerReferenceable { IPakSection section{}; - m_stream->read(reinterpret_cast(§ion), sizeof section); - if (m_stream->gcount() != sizeof section) + m_stream->read(reinterpret_cast(§ion), sizeof(section)); + if (m_stream->gcount() != sizeof(section)) { printf("Unexpected eof when trying to load section.\n"); return false; @@ -98,7 +98,7 @@ class IPak::Impl : public ObjContainerReferenceable { IPakHeader header{}; - m_stream->read(reinterpret_cast(&header), sizeof header); + m_stream->read(reinterpret_cast(&header), sizeof(header)); if (m_stream->gcount() != sizeof header) { printf("Unexpected eof when trying to load header.\n"); diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp index c29387b3..e401d27b 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -26,8 +27,14 @@ public: m_data_section_offset(0), m_data_section_size(0u), m_index_section_offset(0), - m_branding_section_offset(0) + m_branding_section_offset(0), + m_file_offset(0u), + m_chunk_buffer_window_start(0), + m_current_block{}, + m_current_block_header_offset(0) { + m_decompressed_buffer = std::make_unique(ipak_consts::IPAK_CHUNK_SIZE); + m_lzo_work_buffer = std::make_unique(LZO1X_1_MEM_COMPRESS); } void AddImage(std::string imageName) override @@ -59,9 +66,14 @@ public: } } - void AlignToChunkWrite() + void AlignToChunk() { - Pad(static_cast(utils::Align(m_current_offset, 0x8000i64) - m_current_offset)); + Pad(static_cast(utils::Align(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)) - m_current_offset)); + } + + void AlignToBlockHeader() + { + Pad(static_cast(utils::Align(m_current_offset, static_cast(sizeof(IPakDataBlockHeader))) - m_current_offset)); } void WriteHeaderData() @@ -128,6 +140,106 @@ public: return imageData; } + void FlushBlock() + { + if (m_current_block_header_offset > 0) + { + const auto previousOffset = m_current_offset; + + GoTo(m_current_block_header_offset); + Write(&m_current_block, sizeof(m_current_block)); + GoTo(previousOffset); + } + } + + void FlushChunk() + { + FlushBlock(); + AlignToBlockHeader(); + + const auto nextChunkOffset = utils::Align(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)); + const auto sizeToSkip = static_cast(nextChunkOffset - m_current_offset); + + if (sizeToSkip >= sizeof(IPakDataBlockHeader)) + { + IPakDataBlockHeader skipBlockHeader{}; + skipBlockHeader.countAndOffset.count = 1; + skipBlockHeader.commands[0].compressed = ipak_consts::IPAK_COMMAND_SKIP; + skipBlockHeader.commands[0].size = sizeToSkip - sizeof(IPakDataBlockHeader); + Write(&skipBlockHeader, sizeof(skipBlockHeader)); + } + + AlignToChunk(); + m_chunk_buffer_window_start = m_current_offset; + } + + void StartNewBlock() + { + AlignToBlockHeader(); + m_current_block_header_offset = m_current_offset; + m_current_block = {}; + m_current_block.countAndOffset.offset = static_cast(m_file_offset); + } + + void WriteChunkData(const void* data, const size_t dataSize) + { + auto dataOffset = 0u; + while (dataOffset < dataSize) + { + if (m_current_block.countAndOffset.count >= std::extent_v) + { + FlushBlock(); + StartNewBlock(); + } + + const auto remainingSize = dataSize - dataOffset; + const auto remainingChunkBufferWindowSize = std::max((ipak_consts::IPAK_CHUNK_COUNT_PER_READ * ipak_consts::IPAK_CHUNK_SIZE) + - static_cast(m_current_offset - m_chunk_buffer_window_start), 0u); + const auto commandSize = std::min(std::min(remainingSize, ipak_consts::IPAK_COMMAND_DEFAULT_SIZE), remainingChunkBufferWindowSize); + + auto writeUncompressed = true; + if (USE_COMPRESSION) + { + auto outLen = static_cast(ipak_consts::IPAK_CHUNK_SIZE); + const auto result = lzo1x_1_compress(&static_cast(data)[dataOffset], commandSize, reinterpret_cast(m_decompressed_buffer.get()), &outLen, + m_lzo_work_buffer.get()); + + if (result == LZO_E_OK && outLen < commandSize) + { + writeUncompressed = false; + Write(m_decompressed_buffer.get(), outLen); + + const auto currentCommand = m_current_block.countAndOffset.count; + m_current_block.commands[currentCommand].size = static_cast(outLen); + m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_COMPRESSED; + m_current_block.countAndOffset.count = currentCommand + 1u; + } + } + + if (writeUncompressed) + { + Write(data, dataSize); + + const auto currentCommand = m_current_block.countAndOffset.count; + m_current_block.commands[currentCommand].size = commandSize; + m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_UNCOMPRESSED; + m_current_block.countAndOffset.count = currentCommand + 1u; + } + + dataOffset += commandSize; + } + + m_file_offset += dataSize; + } + + void StartNewFile() + { + FlushBlock(); + StartNewBlock(); + m_file_offset = 0u; + m_chunk_buffer_window_start = utils::AlignToPrevious(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)); + } + bool WriteImageData(const std::string& imageName) { size_t imageSize; @@ -138,34 +250,39 @@ public: const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0); const auto dataHash = static_cast(crc32(0u, reinterpret_cast(imageData.get()), imageSize)); - IPakIndexEntry indexEntry{}; + IPakIndexEntry indexEntry; indexEntry.key.nameHash = nameHash; indexEntry.key.dataHash = dataHash & 0x1FFFFFFF; indexEntry.offset = static_cast(m_current_offset - m_data_section_offset); - size_t writtenImageSize = 0u; - - // TODO: Write image data + StartNewFile(); + const auto startOffset = m_current_offset; + WriteChunkData(imageData.get(), imageSize); + const auto writtenImageSize = static_cast(m_current_offset - startOffset); indexEntry.size = writtenImageSize; m_index_entries.emplace_back(indexEntry); - m_data_section_size += writtenImageSize; return true; } bool WriteDataSection() { - AlignToChunkWrite(); + AlignToChunk(); m_data_section_offset = m_current_offset; m_data_section_size = 0u; m_index_entries.reserve(m_images.size()); - return std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName) + const auto result = std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName) { return WriteImageData(imageName); }); + + FlushBlock(); + m_data_section_size = static_cast(m_current_offset - m_data_section_offset); + + return result; } static bool CompareIndices(const IPakIndexEntry& entry1, const IPakIndexEntry& entry2) @@ -180,7 +297,7 @@ public: void WriteIndexSection() { - AlignToChunkWrite(); + AlignToChunk(); m_index_section_offset = m_current_offset; SortIndexSectionEntries(); @@ -191,7 +308,7 @@ public: void WriteBrandingSection() { - AlignToChunkWrite(); + AlignToChunk(); m_branding_section_offset = m_current_offset; Write(BRANDING, std::extent_v); @@ -199,7 +316,7 @@ public: void WriteFileEnding() { - AlignToChunkWrite(); + AlignToChunk(); m_total_size = m_current_offset; } @@ -207,7 +324,7 @@ public: { // We will write the header and sections later since they need complementary data GoTo(sizeof(IPakHeader) + sizeof(IPakSection) * SECTION_COUNT); - AlignToChunkWrite(); + AlignToChunk(); if (!WriteDataSection()) return false; @@ -233,6 +350,13 @@ private: size_t m_data_section_size; int64_t m_index_section_offset; int64_t m_branding_section_offset; + + std::unique_ptr m_decompressed_buffer; + std::unique_ptr m_lzo_work_buffer; + size_t m_file_offset; + int64_t m_chunk_buffer_window_start; + IPakDataBlockHeader m_current_block; + int64_t m_current_block_header_offset; }; std::unique_ptr IPakWriter::Create(std::ostream& stream, ISearchPath* assetSearchPath) diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h index f7051873..6b9f0a3a 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h @@ -7,6 +7,8 @@ class IPakWriter { public: + static constexpr auto USE_COMPRESSION = true; + IPakWriter() = default; virtual ~IPakWriter() = default; diff --git a/src/Utils/Utils/Alignment.h b/src/Utils/Utils/Alignment.h index 9b0ba20d..053a4636 100644 --- a/src/Utils/Utils/Alignment.h +++ b/src/Utils/Utils/Alignment.h @@ -3,10 +3,18 @@ namespace utils { template - constexpr T Align(T value, T toNext) + constexpr T Align(const T value, const T toNext) { if (toNext > 0) return (value + toNext - 1) / toNext * toNext; return value; } + + template + constexpr T AlignToPrevious(const T value, const T toPrevious) + { + if (toPrevious > 0) + return value / toPrevious * toPrevious; + return value; + } }