diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index c8b77436..ee259f1d 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -4,6 +4,12 @@ typedef uint32_t IPakHash; +namespace ipak_consts +{ + static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; + static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; +} + struct IPakHeader { uint32_t magic; @@ -37,7 +43,7 @@ struct IPakIndexEntry uint32_t size; }; -struct IPakDataChunkHeader +struct IPakDataBlockHeader { union { diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index ba20c3ec..19602398 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -1,23 +1,30 @@ #include "IPakEntryReadStream.h" +#include "ObjContainer/IPak/IPakTypes.h" #include #include +#include -int64_t IPakEntryReadStream::Align(const int64_t num, const int64_t alignTo) -{ - return (num + alignTo - 1) / alignTo * alignTo; -} +using namespace ipak_consts; -IPakEntryReadStream::IPakEntryReadStream(IFile* file, IPakStreamManager* streamManager, std::mutex* readMutex, const int64_t startOffset, const size_t length) +IPakEntryReadStream::IPakEntryReadStream(IFile* file, IPakStreamManagerActions* streamManagerActions, + uint8_t* chunkBuffer, const int64_t startOffset, const size_t fileSize) { m_file = file; - m_base_pos = startOffset; - m_end_pos = startOffset + length; - m_buffer = new uint8_t[IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ]; - m_stream_manager = streamManager; - m_read_mutex = readMutex; + m_stream_manager_actions = streamManagerActions; + m_chunk_buffer = chunkBuffer; - m_buffer_pos = 0; + m_file_offset = 0; + m_file_head = 0; + m_file_length = fileSize; + + m_file_buffer = new uint8_t[fileSize]; + + m_base_pos = startOffset; m_pos = m_base_pos; + m_buffer_start_pos = 0; + m_buffer_end_pos = 0; + + lzo_init(); } IPakEntryReadStream::~IPakEntryReadStream() @@ -25,6 +32,175 @@ IPakEntryReadStream::~IPakEntryReadStream() Close(); } +bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, const size_t chunkCount) +{ + // Cannot load more than IPAK_CHUNK_COUNT_PER_READ chunks without overflowing the buffer + assert(chunkCount <= IPAK_CHUNK_COUNT_PER_READ); + + // The start position must be aligned to IPAK_CHUNK_SIZE + assert(startPos % IPAK_CHUNK_SIZE == 0); + + const int64_t endPos = startPos + static_cast(chunkCount) * IPAK_CHUNK_SIZE; + + uint8_t* readBuffer = m_chunk_buffer; + size_t readChunkCount = chunkCount; + + if (startPos >= m_buffer_start_pos && startPos < m_buffer_end_pos) + { + if (startPos != m_buffer_start_pos) + { + memmove_s(m_chunk_buffer, + IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ, + &m_chunk_buffer[startPos - m_buffer_start_pos], + static_cast(m_buffer_end_pos - startPos)); + readBuffer = &m_chunk_buffer[m_buffer_end_pos - startPos]; + } + + readChunkCount = endPos > m_buffer_end_pos + ? static_cast(endPos - m_buffer_end_pos) / IPAK_CHUNK_SIZE + : 0; + } + else if (endPos > m_buffer_start_pos && endPos <= m_buffer_end_pos) + { + memmove_s(&m_chunk_buffer[m_buffer_start_pos - startPos], + IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ - static_cast(m_buffer_start_pos - startPos), + m_chunk_buffer, + static_cast(endPos - m_buffer_start_pos)); + + readChunkCount = static_cast(m_buffer_start_pos - startPos) / IPAK_CHUNK_SIZE; + } + + if (readChunkCount == 0) + return true; + + m_stream_manager_actions->StartReading(); + m_file->Goto(startPos); + const auto readSize = m_file->Read(readBuffer, 1, readChunkCount * IPAK_CHUNK_SIZE); + m_stream_manager_actions->StopReading(); + + m_buffer_start_pos = startPos; + m_buffer_end_pos = AlignBackwards(m_buffer_start_pos + readSize, IPAK_CHUNK_SIZE); + + return readSize == readChunkCount * IPAK_CHUNK_SIZE; +} + +bool IPakEntryReadStream::ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const +{ + if (blockHeader->count > 31) + { + printf("IPak block has more than 31 commands: %u -> Invalid\n", blockHeader->count); + return false; + } + if (blockHeader->offset > m_file_length) + { + printf("IPak block offset is larger than file itself: %u > %u -> Invalid\n", blockHeader->offset, + m_file_length); + return false; + } + + return true; +} + +bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, + const size_t blockOffsetInChunk) +{ + size_t commandsSize = 0; + for (unsigned commandIndex = 0; commandIndex < blockHeader->count; commandIndex++) + { + commandsSize += blockHeader->_commands[commandIndex].size; + } + + const size_t requiredChunkCount = AlignForward( + blockOffsetInChunk + sizeof IPakDataBlockHeader + commandsSize, IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE; + + const size_t amountOfReadBlocks = static_cast(m_buffer_end_pos - m_buffer_start_pos) / IPAK_CHUNK_SIZE; + + if (requiredChunkCount > amountOfReadBlocks) + { + if (requiredChunkCount > IPAK_CHUNK_COUNT_PER_READ) + { + printf("IPak block spans over more than %u blocks (%u), which is not supported.\n", + IPAK_CHUNK_COUNT_PER_READ, requiredChunkCount); + return false; + } + + if (!SetChunkBufferWindow(m_buffer_start_pos, requiredChunkCount)) + return false; + } + + return true; +} + +bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const bool compressed) +{ + if (compressed) + { + lzo_uint outputSize = m_file_length - m_file_head; + const auto result = lzo1x_decompress(&m_chunk_buffer[m_pos - m_buffer_start_pos], commandSize, + &m_file_buffer[m_file_head], &outputSize, nullptr); + + if (result != LZO_E_OK) + { + printf("Decompressing block with lzo failed: %i!\n", result); + return false; + } + + m_file_head += outputSize; + } + else + { + if (m_file_length - m_file_head < commandSize) + { + printf( + "There is not enough space in output buffer to extract data from IPak block: %u required, %u available\n", + commandSize, m_file_length - m_file_head); + return false; + } + + memcpy_s(&m_file_buffer[m_file_head], m_file_length - m_file_head, &m_chunk_buffer[m_pos - m_buffer_start_pos], + commandSize); + } + + return true; +} + +bool IPakEntryReadStream::AdvanceStream() +{ + const auto chunkStartPos = AlignBackwards(m_pos, IPAK_CHUNK_SIZE); + const auto blockOffsetInChunk = static_cast(m_pos - chunkStartPos); + + const size_t sizeLeftToRead = m_file_length - m_file_head; + size_t estimatedChunksToRead = AlignForward(blockOffsetInChunk + sizeLeftToRead, IPAK_CHUNK_SIZE) + / IPAK_CHUNK_SIZE; + + if (estimatedChunksToRead > IPAK_CHUNK_COUNT_PER_READ) + estimatedChunksToRead = IPAK_CHUNK_COUNT_PER_READ; + + if (!SetChunkBufferWindow(chunkStartPos, estimatedChunksToRead)) + return false; + + const auto blockHeader = reinterpret_cast(&m_chunk_buffer[blockOffsetInChunk]); + + if (!ValidateBlockHeader(blockHeader)) + return false; + + if (!AdjustChunkBufferWindowForBlockHeader(blockHeader, blockOffsetInChunk)) + return false; + + m_pos += sizeof IPakDataBlockHeader; + + for (unsigned commandIndex = 0; commandIndex < blockHeader->count; commandIndex++) + { + if (!ProcessCommand(blockHeader->_commands[commandIndex].size, + blockHeader->_commands[commandIndex].compressed == 1)) + return false; + } + + m_pos = AlignForward(m_pos, sizeof IPakDataBlockHeader); + + return true; +} + bool IPakEntryReadStream::IsOpen() { return m_file != nullptr; @@ -32,10 +208,27 @@ bool IPakEntryReadStream::IsOpen() size_t IPakEntryReadStream::Read(void* buffer, const size_t elementSize, const size_t elementCount) { - const size_t readSize = elementCount * elementSize; - size_t chunksToRead = Align((m_pos % IPAK_CHUNK_SIZE) + readSize, IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE; + const size_t bufferSize = elementCount * elementSize; + size_t sizeToRead = bufferSize <= m_file_length - m_file_offset ? bufferSize : m_file_length - m_file_offset; - return 0; + if (sizeToRead) + sizeToRead = m_file_length - m_file_offset; + + while (m_file_offset + sizeToRead > m_file_head) + { + if (!AdvanceStream()) + { + if (m_file_head - m_file_offset < sizeToRead) + sizeToRead = m_file_head - m_file_offset; + } + } + + if (sizeToRead > 0) + { + memcpy_s(buffer, bufferSize, &m_file_buffer[m_file_offset], sizeToRead); + } + + return sizeToRead; } size_t IPakEntryReadStream::Write(const void* data, size_t elementSize, size_t elementCount) @@ -47,14 +240,12 @@ size_t IPakEntryReadStream::Write(const void* data, size_t elementSize, size_t e void IPakEntryReadStream::Skip(const int64_t amount) { - if(amount > 0) + if (amount > 0) { - m_pos += amount; + m_file_offset += static_cast(amount); - if (m_pos > m_end_pos) - m_pos = m_end_pos; - else if (m_pos < m_base_pos) - m_pos = m_base_pos; + if (m_file_offset > m_file_length) + m_file_offset = m_file_length; } } @@ -67,39 +258,34 @@ size_t IPakEntryReadStream::Printf(const char* fmt, ...) int64_t IPakEntryReadStream::Pos() { - return m_pos - m_base_pos; + return m_file_offset; } void IPakEntryReadStream::Goto(const int64_t pos) { - if(pos <= 0) + if (pos <= 0) { - m_pos = m_base_pos; - } - else if(pos < m_end_pos - m_base_pos) - { - m_pos = pos + m_base_pos; - } - else - { - m_pos = m_end_pos; + m_file_offset = static_cast(pos); + + if (m_file_offset > m_file_length) + m_file_offset = m_file_length; } } void IPakEntryReadStream::GotoEnd() { - m_pos = m_end_pos; + m_pos = m_file_length; } void IPakEntryReadStream::Close() { - if(IsOpen()) + if (IsOpen()) { m_file = nullptr; - delete[] m_buffer; - m_buffer = nullptr; + delete[] m_file_buffer; + m_file_buffer = nullptr; - m_stream_manager->OnCloseStream(this); + m_stream_manager_actions->CloseStream(this); } } diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h index 08cc05c9..099aa226 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h @@ -2,28 +2,48 @@ #include "IPakStreamManager.h" #include "Utils/FileAPI.h" +#include "ObjContainer/IPak/IPakTypes.h" #include class IPakEntryReadStream final : public FileAPI::IFile { - static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; - static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; + uint8_t* m_chunk_buffer; + uint8_t* m_file_buffer; - uint8_t* m_buffer; IFile* m_file; - IPakStreamManager* m_stream_manager; - std::mutex* m_read_mutex; + IPakStreamManagerActions* m_stream_manager_actions; + + size_t m_file_offset; + size_t m_file_head; + size_t m_file_length; int64_t m_pos; int64_t m_base_pos; - int64_t m_end_pos; - int64_t m_buffer_pos; + int64_t m_buffer_start_pos; + int64_t m_buffer_end_pos; - static int64_t Align(int64_t num, int64_t alignTo); + template + static T AlignForward(const T num, const T alignTo) + { + return (num + alignTo - 1) / alignTo * alignTo; + } + + template + static T AlignBackwards(const T num, const T alignTo) + { + return num / alignTo * alignTo; + } + + bool SetChunkBufferWindow(int64_t startPos, size_t chunkCount); + bool ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const; + bool AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, size_t blockOffsetInChunk); + bool ProcessCommand(size_t commandSize, bool compressed); + bool AdvanceStream(); public: - IPakEntryReadStream(IFile* file, IPakStreamManager* streamManager, std::mutex* readMutex, int64_t startOffset, size_t length); + IPakEntryReadStream(IFile* file, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t fileSize); ~IPakEntryReadStream() override; + bool IsOpen() override; size_t Read(void* buffer, size_t elementSize, size_t elementCount) override; size_t Write(const void* data, size_t elementSize, size_t elementCount) override; diff --git a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp index d15104e8..9cbbae64 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp @@ -1,41 +1,154 @@ #include "IPakStreamManager.h" #include "IPakEntryReadStream.h" +#include "ObjContainer/IPak/IPakTypes.h" +#include + +using namespace ipak_consts; + +class IPakStreamManager::Impl final : public IPakStreamManagerActions +{ + static constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3; + + class ChunkBuffer + { + public: + IPakEntryReadStream* m_using_stream = nullptr; + uint8_t m_buffer[IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ]{}; + }; + + class ManagedStream + { + public: + IPakEntryReadStream* m_stream; + ChunkBuffer* m_chunk_buffer; + + ManagedStream(IPakEntryReadStream* stream, ChunkBuffer* chunkBuffer) + { + m_stream = stream; + m_chunk_buffer = chunkBuffer; + } + }; + + FileAPI::IFile* m_file; + + std::mutex m_read_mutex; + std::mutex m_stream_mutex; + + std::vector m_open_streams; + std::vector m_chunk_buffers; + +public: + explicit Impl(FileAPI::IFile* file) + { + m_file = file; + + m_chunk_buffers.push_back(new ChunkBuffer()); + } + + Impl(const Impl& other) = delete; + Impl(Impl&& other) noexcept = delete; + + virtual ~Impl() + { + m_stream_mutex.lock(); + + for (const auto& openStream : m_open_streams) + { + openStream.m_stream->Close(); + } + m_open_streams.clear(); + + m_stream_mutex.unlock(); + } + + Impl& operator=(const Impl& other) = delete; + Impl& operator=(Impl&& other) noexcept = delete; + + FileAPI::IFile* OpenStream(const int64_t startPosition, const size_t length) + { + m_stream_mutex.lock(); + + ChunkBuffer* reservedChunkBuffer; + const auto freeChunkBuffer = std::find_if(m_chunk_buffers.begin(), m_chunk_buffers.end(), + [](ChunkBuffer* chunkBuffer) + { + return chunkBuffer->m_using_stream == nullptr; + }); + + if (freeChunkBuffer == m_chunk_buffers.end()) + { + reservedChunkBuffer = new ChunkBuffer(); + m_chunk_buffers.push_back(reservedChunkBuffer); + } + else + reservedChunkBuffer = *freeChunkBuffer; + + auto* stream = new IPakEntryReadStream(m_file, this, reservedChunkBuffer->m_buffer, startPosition, length); + + reservedChunkBuffer->m_using_stream = stream; + + m_open_streams.emplace_back(stream, reservedChunkBuffer); + + m_stream_mutex.unlock(); + + return stream; + } + + void StartReading() override + { + m_read_mutex.lock(); + } + + void StopReading() override + { + m_read_mutex.unlock(); + } + + void CloseStream(FileAPI::IFile* stream) override + { + m_stream_mutex.lock(); + + const auto openStreamEntry = std::find_if(m_open_streams.begin(), m_open_streams.end(), + [stream](const ManagedStream& managedStream) + { + return managedStream.m_stream == stream; + }); + + if (openStreamEntry != m_open_streams.end()) + { + auto* chunkBuffer = openStreamEntry->m_chunk_buffer; + m_open_streams.erase(openStreamEntry); + chunkBuffer->m_using_stream = nullptr; + + // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers + if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT) + { + const auto chunkBufferEntry = std::find(m_chunk_buffers.begin(), m_chunk_buffers.end(), chunkBuffer); + + if (chunkBufferEntry != m_chunk_buffers.end()) + { + m_chunk_buffers.erase(chunkBufferEntry); + delete chunkBuffer; + } + } + } + + m_stream_mutex.unlock(); + } +}; IPakStreamManager::IPakStreamManager(FileAPI::IFile* file) { - m_file = file; + m_impl = new Impl(file); } IPakStreamManager::~IPakStreamManager() { - m_stream_mutex.lock(); - for(const auto& openStream : m_open_streams) - { - openStream->Close(); - } - m_open_streams.clear(); - m_stream_mutex.unlock(); + delete m_impl; + m_impl = nullptr; } -FileAPI::IFile* IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) +FileAPI::IFile* IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) const { - auto* stream = new IPakEntryReadStream(m_file, this, &m_read_mutex, startPosition, length); - - m_stream_mutex.lock(); - m_open_streams.push_back(stream); - m_stream_mutex.unlock(); - - return stream; -} - -void IPakStreamManager::OnCloseStream(FileAPI::IFile* stream) -{ - m_stream_mutex.lock(); - const auto openStreamEntry = std::find(m_open_streams.begin(), m_open_streams.end(), stream); - - if(openStreamEntry != m_open_streams.end()) - { - m_open_streams.erase(openStreamEntry); - } - m_stream_mutex.unlock(); + return m_impl->OpenStream(startPosition, length); } diff --git a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h index 80f16816..00ecf6b5 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h @@ -1,15 +1,22 @@ #pragma once #include "Utils/FileAPI.h" -#include +#include #include +class IPakStreamManagerActions +{ +public: + virtual void StartReading() = 0; + virtual void StopReading() = 0; + + virtual void CloseStream(FileAPI::IFile* stream) = 0; +}; + class IPakStreamManager { - FileAPI::IFile* m_file; - std::vector m_open_streams; - std::mutex m_read_mutex; - std::mutex m_stream_mutex; + class Impl; + Impl* m_impl; public: explicit IPakStreamManager(FileAPI::IFile* file); @@ -20,6 +27,5 @@ public: IPakStreamManager& operator=(const IPakStreamManager& other) = delete; IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete; - FileAPI::IFile* OpenStream(int64_t startPosition, size_t length); - void OnCloseStream(FileAPI::IFile* stream); + FileAPI::IFile* OpenStream(int64_t startPosition, size_t length) const; }; \ No newline at end of file