From 4baca812103c0f39e191a1de4d7cd3502c5423ed Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 9 Feb 2020 09:51:57 +0100 Subject: [PATCH] ObjLoading: Implement IPakEntryReadStream as a linear stream --- src/ObjLoading/Image/IwiLoader.cpp | 23 +-- .../ObjContainer/IPak/IPakEntryReadStream.cpp | 195 ++++++++++-------- .../ObjContainer/IPak/IPakEntryReadStream.h | 13 +- 3 files changed, 121 insertions(+), 110 deletions(-) diff --git a/src/ObjLoading/Image/IwiLoader.cpp b/src/ObjLoading/Image/IwiLoader.cpp index d85f6d14..400531bd 100644 --- a/src/ObjLoading/Image/IwiLoader.cpp +++ b/src/ObjLoading/Image/IwiLoader.cpp @@ -7,35 +7,18 @@ IwiLoader::IwiLoader(MemoryManager* memoryManager) Texture* IwiLoader::LoadIwi(FileAPI::IFile* file) { - struct IWIHeaderMeta + struct { char tag[3]; char version; } iwiHeaderMeta{}; - file->GotoEnd(); - auto iwiSize = static_cast(file->Pos()); - file->Goto(0); - - auto* buffer = new uint8_t[iwiSize]; - if(file->Read(buffer, 1, iwiSize) != iwiSize) - { - delete[] buffer; + if (file->Read(&iwiHeaderMeta, sizeof iwiHeaderMeta, 1) != 1) return nullptr; - } - auto* meta = reinterpret_cast(buffer); - - printf("Read IWI with version %i\n", meta->version); + printf("Read IWI with version %i\n", iwiHeaderMeta.version); // TODO: Read iwi based on version return nullptr; - - // if (file->Read(&iwiHeaderMeta, sizeof iwiHeaderMeta, 1) != 1) - // return nullptr; - // - // printf("Read IWI with version %i\n", iwiHeaderMeta.version); - // - // return nullptr; } diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index ac87eb36..2eb787fc 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -8,6 +8,7 @@ using namespace ipak_consts; IPakEntryReadStream::IPakEntryReadStream(IFile* file, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, const int64_t startOffset, const size_t entrySize) + : m_decompress_buffer{} { m_file = file; m_stream_manager_actions = streamManagerActions; @@ -17,14 +18,18 @@ IPakEntryReadStream::IPakEntryReadStream(IFile* file, IPakStreamManagerActions* m_file_head = 0; m_entry_size = entrySize; - m_file_buffer = new uint8_t[entrySize]; - m_base_pos = startOffset; m_end_pos = startOffset + entrySize; m_pos = m_base_pos; m_buffer_start_pos = 0; m_buffer_end_pos = 0; + m_current_block = nullptr; + m_next_command = 0; + m_current_command_buffer = nullptr; + m_current_command_length = 0; + m_current_command_offset = 0; + lzo_init(); } @@ -132,44 +137,13 @@ bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHea return true; } -bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const bool compressed) +bool IPakEntryReadStream::NextBlock() { - if (compressed) - { - lzo_uint outputSize = m_entry_size - m_file_head; - const auto result = lzo1x_decompress_safe(&m_chunk_buffer[m_pos - m_buffer_start_pos], commandSize, - &m_file_buffer[m_file_head], &outputSize, nullptr); + if (m_pos >= m_end_pos) + return false; - if (result != LZO_E_OK) - { - printf("Decompressing block with lzo failed: %i!\n", result); - return false; - } + m_pos = AlignForward(m_pos, sizeof IPakDataBlockHeader); - m_file_head += outputSize; - } - else - { - if (m_entry_size - m_file_head < commandSize) - { - printf( - "There is not enough space in output buffer to extract data from IPak block: %u required, %u available\n", - commandSize, m_entry_size - m_file_head); - return false; - } - - memcpy_s(&m_file_buffer[m_file_head], m_entry_size - m_file_head, &m_chunk_buffer[m_pos - m_buffer_start_pos], - commandSize); - - m_file_head += commandSize; - } - m_pos += commandSize; - - return true; -} - -bool IPakEntryReadStream::AdvanceStream() -{ const auto chunkStartPos = AlignBackwards(m_pos, IPAK_CHUNK_SIZE); const auto blockOffsetInChunk = static_cast(m_pos - chunkStartPos); @@ -183,24 +157,64 @@ bool IPakEntryReadStream::AdvanceStream() if (!SetChunkBufferWindow(chunkStartPos, estimatedChunksToRead)) return false; - const auto blockHeader = reinterpret_cast(&m_chunk_buffer[blockOffsetInChunk]); + m_current_block = reinterpret_cast(&m_chunk_buffer[blockOffsetInChunk]); - if (!ValidateBlockHeader(blockHeader)) + if (!ValidateBlockHeader(m_current_block)) return false; - if (!AdjustChunkBufferWindowForBlockHeader(blockHeader, blockOffsetInChunk)) + if (!AdjustChunkBufferWindowForBlockHeader(m_current_block, blockOffsetInChunk)) return false; m_pos += sizeof IPakDataBlockHeader; + m_next_command = 0; - for (unsigned commandIndex = 0; commandIndex < blockHeader->count; commandIndex++) + return true; +} + +bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int compressed) +{ + if (compressed > 0) { - if (!ProcessCommand(blockHeader->_commands[commandIndex].size, - blockHeader->_commands[commandIndex].compressed == 1)) + if (compressed == 1) + { + lzo_uint outputSize = sizeof m_decompress_buffer; + const auto result = lzo1x_decompress_safe(&m_chunk_buffer[m_pos - m_buffer_start_pos], commandSize, + m_decompress_buffer, &outputSize, nullptr); + + if (result != LZO_E_OK) + { + printf("Decompressing block with lzo failed: %i!\n", result); + return false; + } + + m_current_command_buffer = m_decompress_buffer; + m_current_command_length = outputSize; + m_current_command_offset = 0; + m_file_head += outputSize; + } + } + else + { + m_current_command_buffer = &m_chunk_buffer[m_pos - m_buffer_start_pos]; + m_current_command_length = commandSize; + m_current_command_offset = 0; + m_file_head += commandSize; + } + m_pos += commandSize; + + return true; +} + +bool IPakEntryReadStream::AdvanceStream() +{ + if(m_current_block == nullptr || m_next_command >= m_current_block->count) + { + if (!NextBlock()) return false; } - m_pos = AlignForward(m_pos, sizeof IPakDataBlockHeader); + ProcessCommand(m_current_block->_commands[m_next_command].size, m_current_block->_commands[m_next_command].compressed); + m_next_command++; return true; } @@ -212,29 +226,32 @@ bool IPakEntryReadStream::IsOpen() size_t IPakEntryReadStream::Read(void* buffer, const size_t elementSize, const size_t elementCount) { + auto* destBuffer = static_cast(buffer); const size_t bufferSize = elementCount * elementSize; - size_t sizeToRead = bufferSize <= m_entry_size - m_file_offset ? bufferSize : m_entry_size - m_file_offset; + size_t countRead = 0; - while (m_file_offset + sizeToRead > m_file_head && m_pos < m_end_pos) + while (countRead < bufferSize) { - if (!AdvanceStream()) + if (m_current_command_offset >= m_current_command_length) { - if (m_file_head - m_file_offset < sizeToRead) - sizeToRead = m_file_head - m_file_offset; + if (!AdvanceStream()) + break; + } + + size_t sizeToRead = bufferSize - countRead; + if (sizeToRead > m_current_command_length - m_current_command_offset) + sizeToRead = m_current_command_length - m_current_command_offset; + + if(sizeToRead > 0) + { + memcpy_s(&destBuffer[countRead], bufferSize - countRead, &m_current_command_buffer[m_current_command_offset], sizeToRead); + countRead += sizeToRead; + m_current_command_offset += sizeToRead; + m_file_offset += sizeToRead; } } - if(sizeToRead > m_file_head - m_file_offset) - { - sizeToRead = m_file_head - m_file_offset; - } - - if (sizeToRead > 0) - { - memcpy_s(buffer, bufferSize, &m_file_buffer[m_file_offset], sizeToRead); - } - - return sizeToRead; + return countRead / elementSize; } size_t IPakEntryReadStream::Write(const void* data, size_t elementSize, size_t elementCount) @@ -248,10 +265,24 @@ void IPakEntryReadStream::Skip(const int64_t amount) { if (amount > 0) { - m_file_offset += static_cast(amount); + const size_t targetOffset = m_file_offset + static_cast(amount); - if (m_file_offset > m_entry_size) - m_file_offset = m_entry_size; + while(m_file_head < targetOffset) + { + if (!AdvanceStream()) + break; + } + + if(targetOffset <= m_file_head) + { + m_current_command_offset = m_current_command_length - (m_file_head - targetOffset); + m_file_offset = targetOffset; + } + else + { + m_current_command_offset = m_current_command_length; + m_file_offset = m_file_head; + } } } @@ -269,34 +300,25 @@ int64_t IPakEntryReadStream::Pos() void IPakEntryReadStream::Goto(const int64_t pos) { - if (pos >= 0) + if (pos > m_file_offset) { - while (m_file_head < pos && m_pos < m_end_pos) - { - if (!AdvanceStream()) - { - break; - } - } - - m_file_offset = static_cast(pos); - - if (m_file_offset > m_file_head) - m_file_offset = m_file_head; + Skip(pos - m_file_offset); + } + else + { + // Not implemented due to being too time consuming. + // Can be added if necessary. + assert(false); + throw std::runtime_error("Operation not supported!"); } } void IPakEntryReadStream::GotoEnd() { - while (m_pos < m_end_pos) - { - if (!AdvanceStream()) - { - break; - } - } - - m_file_offset = m_file_head; + // Not implemented due to being too time consuming. + // Can be added if necessary. + assert(false); + throw std::runtime_error("Operation not supported!"); } void IPakEntryReadStream::Close() @@ -305,9 +327,6 @@ void IPakEntryReadStream::Close() { m_file = nullptr; - delete[] m_file_buffer; - m_file_buffer = nullptr; - m_stream_manager_actions->CloseStream(this); } } diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h index ce6bc2e9..47285334 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h @@ -7,8 +7,9 @@ class IPakEntryReadStream final : public FileAPI::IFile { + static constexpr size_t IPAK_DECOMPRESS_BUFFER_SIZE = 0x8000; + uint8_t* m_chunk_buffer; - uint8_t* m_file_buffer; IFile* m_file; IPakStreamManagerActions* m_stream_manager_actions; @@ -18,6 +19,13 @@ class IPakEntryReadStream final : public FileAPI::IFile size_t m_entry_size; + uint8_t m_decompress_buffer[IPAK_DECOMPRESS_BUFFER_SIZE]; + IPakDataBlockHeader* m_current_block; + unsigned m_next_command; + uint8_t* m_current_command_buffer; + size_t m_current_command_length; + size_t m_current_command_offset; + int64_t m_pos; int64_t m_base_pos; int64_t m_end_pos; @@ -39,7 +47,8 @@ class IPakEntryReadStream final : public FileAPI::IFile bool SetChunkBufferWindow(int64_t startPos, size_t chunkCount); bool ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const; bool AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, size_t blockOffsetInChunk); - bool ProcessCommand(size_t commandSize, bool compressed); + bool NextBlock(); + bool ProcessCommand(size_t commandSize, int compressed); bool AdvanceStream(); public: