ObjLoading: Initial commit for IPak entry reading

src/ObjCommon/ObjContainer/IPak/IPakTypes.h

@@ -4,6 +4,12 @@
 
 typedef uint32_t IPakHash;
 
+namespace ipak_consts
+{
+	static constexpr size_t IPAK_CHUNK_SIZE = 0x8000;
+	static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8;
+}
+
 struct IPakHeader
 {
 	uint32_t magic;
@@ -37,7 +43,7 @@ struct IPakIndexEntry
 	uint32_t size;
 };
 
-struct IPakDataChunkHeader
+struct IPakDataBlockHeader
 {
 	union
 	{

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp

@@ -1,23 +1,30 @@
 #include "IPakEntryReadStream.h"
+#include "ObjContainer/IPak/IPakTypes.h"
 #include <cassert>
 #include <stdexcept>
+#include <minilzo.h>
 
-int64_t IPakEntryReadStream::Align(const int64_t num, const int64_t alignTo)
-{
-    return (num + alignTo - 1) / alignTo * alignTo;
-}
+using namespace ipak_consts;
 
-IPakEntryReadStream::IPakEntryReadStream(IFile* file, IPakStreamManager* streamManager, std::mutex* readMutex, const int64_t startOffset, const size_t length)
+IPakEntryReadStream::IPakEntryReadStream(IFile* file, IPakStreamManagerActions* streamManagerActions,
+                                         uint8_t* chunkBuffer, const int64_t startOffset, const size_t fileSize)
 {
     m_file = file;
-    m_base_pos = startOffset;
-    m_end_pos = startOffset + length;
-    m_buffer = new uint8_t[IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ];
-    m_stream_manager = streamManager;
-    m_read_mutex = readMutex;
+    m_stream_manager_actions = streamManagerActions;
+    m_chunk_buffer = chunkBuffer;
 
-    m_buffer_pos = 0;
+    m_file_offset = 0;
+    m_file_head = 0;
+    m_file_length = fileSize;
+
+    m_file_buffer = new uint8_t[fileSize];
+
+    m_base_pos = startOffset;
     m_pos = m_base_pos;
+    m_buffer_start_pos = 0;
+    m_buffer_end_pos = 0;
+
+    lzo_init();
 }
 
 IPakEntryReadStream::~IPakEntryReadStream()
@@ -25,6 +32,175 @@ IPakEntryReadStream::~IPakEntryReadStream()
     Close();
 }
 
+bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, const size_t chunkCount)
+{
+    // Cannot load more than IPAK_CHUNK_COUNT_PER_READ chunks without overflowing the buffer
+    assert(chunkCount <= IPAK_CHUNK_COUNT_PER_READ);
+
+    // The start position must be aligned to IPAK_CHUNK_SIZE
+    assert(startPos % IPAK_CHUNK_SIZE == 0);
+
+    const int64_t endPos = startPos + static_cast<int64_t>(chunkCount) * IPAK_CHUNK_SIZE;
+
+    uint8_t* readBuffer = m_chunk_buffer;
+    size_t readChunkCount = chunkCount;
+
+    if (startPos >= m_buffer_start_pos && startPos < m_buffer_end_pos)
+    {
+        if (startPos != m_buffer_start_pos)
+        {
+            memmove_s(m_chunk_buffer,
+                      IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ,
+                      &m_chunk_buffer[startPos - m_buffer_start_pos],
+                      static_cast<size_t>(m_buffer_end_pos - startPos));
+            readBuffer = &m_chunk_buffer[m_buffer_end_pos - startPos];
+        }
+
+        readChunkCount = endPos > m_buffer_end_pos
+                             ? static_cast<size_t>(endPos - m_buffer_end_pos) / IPAK_CHUNK_SIZE
+                             : 0;
+    }
+    else if (endPos > m_buffer_start_pos && endPos <= m_buffer_end_pos)
+    {
+        memmove_s(&m_chunk_buffer[m_buffer_start_pos - startPos],
+                  IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ - static_cast<size_t>(m_buffer_start_pos - startPos),
+                  m_chunk_buffer,
+                  static_cast<size_t>(endPos - m_buffer_start_pos));
+
+        readChunkCount = static_cast<size_t>(m_buffer_start_pos - startPos) / IPAK_CHUNK_SIZE;
+    }
+
+    if (readChunkCount == 0)
+        return true;
+
+    m_stream_manager_actions->StartReading();
+    m_file->Goto(startPos);
+    const auto readSize = m_file->Read(readBuffer, 1, readChunkCount * IPAK_CHUNK_SIZE);
+    m_stream_manager_actions->StopReading();
+
+    m_buffer_start_pos = startPos;
+    m_buffer_end_pos = AlignBackwards<int64_t>(m_buffer_start_pos + readSize, IPAK_CHUNK_SIZE);
+
+    return readSize == readChunkCount * IPAK_CHUNK_SIZE;
+}
+
+bool IPakEntryReadStream::ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const
+{
+    if (blockHeader->count > 31)
+    {
+        printf("IPak block has more than 31 commands: %u -> Invalid\n", blockHeader->count);
+        return false;
+    }
+    if (blockHeader->offset > m_file_length)
+    {
+        printf("IPak block offset is larger than file itself: %u > %u -> Invalid\n", blockHeader->offset,
+               m_file_length);
+        return false;
+    }
+
+    return true;
+}
+
+bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader,
+                                                                const size_t blockOffsetInChunk)
+{
+    size_t commandsSize = 0;
+    for (unsigned commandIndex = 0; commandIndex < blockHeader->count; commandIndex++)
+    {
+        commandsSize += blockHeader->_commands[commandIndex].size;
+    }
+
+    const size_t requiredChunkCount = AlignForward<size_t>(
+        blockOffsetInChunk + sizeof IPakDataBlockHeader + commandsSize, IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE;
+
+    const size_t amountOfReadBlocks = static_cast<size_t>(m_buffer_end_pos - m_buffer_start_pos) / IPAK_CHUNK_SIZE;
+
+    if (requiredChunkCount > amountOfReadBlocks)
+    {
+        if (requiredChunkCount > IPAK_CHUNK_COUNT_PER_READ)
+        {
+            printf("IPak block spans over more than %u blocks (%u), which is not supported.\n",
+                   IPAK_CHUNK_COUNT_PER_READ, requiredChunkCount);
+            return false;
+        }
+
+        if (!SetChunkBufferWindow(m_buffer_start_pos, requiredChunkCount))
+            return false;
+    }
+
+    return true;
+}
+
+bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const bool compressed)
+{
+    if (compressed)
+    {
+        lzo_uint outputSize = m_file_length - m_file_head;
+        const auto result = lzo1x_decompress(&m_chunk_buffer[m_pos - m_buffer_start_pos], commandSize,
+                                             &m_file_buffer[m_file_head], &outputSize, nullptr);
+
+        if (result != LZO_E_OK)
+        {
+            printf("Decompressing block with lzo failed: %i!\n", result);
+            return false;
+        }
+
+        m_file_head += outputSize;
+    }
+    else
+    {
+        if (m_file_length - m_file_head < commandSize)
+        {
+            printf(
+                "There is not enough space in output buffer to extract data from IPak block: %u required, %u available\n",
+                commandSize, m_file_length - m_file_head);
+            return false;
+        }
+
+        memcpy_s(&m_file_buffer[m_file_head], m_file_length - m_file_head, &m_chunk_buffer[m_pos - m_buffer_start_pos],
+                 commandSize);
+    }
+
+    return true;
+}
+
+bool IPakEntryReadStream::AdvanceStream()
+{
+    const auto chunkStartPos = AlignBackwards<int64_t>(m_pos, IPAK_CHUNK_SIZE);
+    const auto blockOffsetInChunk = static_cast<size_t>(m_pos - chunkStartPos);
+
+    const size_t sizeLeftToRead = m_file_length - m_file_head;
+    size_t estimatedChunksToRead = AlignForward(blockOffsetInChunk + sizeLeftToRead, IPAK_CHUNK_SIZE)
+        / IPAK_CHUNK_SIZE;
+
+    if (estimatedChunksToRead > IPAK_CHUNK_COUNT_PER_READ)
+        estimatedChunksToRead = IPAK_CHUNK_COUNT_PER_READ;
+
+    if (!SetChunkBufferWindow(chunkStartPos, estimatedChunksToRead))
+        return false;
+
+    const auto blockHeader = reinterpret_cast<IPakDataBlockHeader*>(&m_chunk_buffer[blockOffsetInChunk]);
+
+    if (!ValidateBlockHeader(blockHeader))
+        return false;
+
+    if (!AdjustChunkBufferWindowForBlockHeader(blockHeader, blockOffsetInChunk))
+        return false;
+
+    m_pos += sizeof IPakDataBlockHeader;
+
+    for (unsigned commandIndex = 0; commandIndex < blockHeader->count; commandIndex++)
+    {
+        if (!ProcessCommand(blockHeader->_commands[commandIndex].size,
+                            blockHeader->_commands[commandIndex].compressed == 1))
+            return false;
+    }
+
+    m_pos = AlignForward<int64_t>(m_pos, sizeof IPakDataBlockHeader);
+
+    return true;
+}
+
 bool IPakEntryReadStream::IsOpen()
 {
     return m_file != nullptr;
@@ -32,10 +208,27 @@ bool IPakEntryReadStream::IsOpen()
 
 size_t IPakEntryReadStream::Read(void* buffer, const size_t elementSize, const size_t elementCount)
 {
-    const size_t readSize = elementCount * elementSize;
-    size_t chunksToRead = Align((m_pos % IPAK_CHUNK_SIZE) + readSize, IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE;
+    const size_t bufferSize = elementCount * elementSize;
+    size_t sizeToRead = bufferSize <= m_file_length - m_file_offset ? bufferSize : m_file_length - m_file_offset;
 
-    return 0;
+    if (sizeToRead)
+        sizeToRead = m_file_length - m_file_offset;
+
+    while (m_file_offset + sizeToRead > m_file_head)
+    {
+        if (!AdvanceStream())
+        {
+            if (m_file_head - m_file_offset < sizeToRead)
+                sizeToRead = m_file_head - m_file_offset;
+        }
+    }
+
+    if (sizeToRead > 0)
+    {
+        memcpy_s(buffer, bufferSize, &m_file_buffer[m_file_offset], sizeToRead);
+    }
+
+    return sizeToRead;
 }
 
 size_t IPakEntryReadStream::Write(const void* data, size_t elementSize, size_t elementCount)
@@ -49,12 +242,10 @@ void IPakEntryReadStream::Skip(const int64_t amount)
 {
     if (amount > 0)
     {
-        m_pos += amount;
+        m_file_offset += static_cast<size_t>(amount);
 
-        if (m_pos > m_end_pos)
-            m_pos = m_end_pos;
-        else if (m_pos < m_base_pos)
-            m_pos = m_base_pos;
+        if (m_file_offset > m_file_length)
+            m_file_offset = m_file_length;
     }
 }
 
@@ -67,28 +258,23 @@ size_t IPakEntryReadStream::Printf(const char* fmt, ...)
 
 int64_t IPakEntryReadStream::Pos()
 {
-    return m_pos - m_base_pos;
+    return m_file_offset;
 }
 
 void IPakEntryReadStream::Goto(const int64_t pos)
 {
     if (pos <= 0)
     {
-        m_pos = m_base_pos;
-    }
-    else if(pos < m_end_pos - m_base_pos)
-    {
-        m_pos = pos + m_base_pos;
-    }
-    else
-    {
-        m_pos = m_end_pos;
+        m_file_offset = static_cast<size_t>(pos);
+
+        if (m_file_offset > m_file_length)
+            m_file_offset = m_file_length;
     }
 }
 
 void IPakEntryReadStream::GotoEnd()
 {
-    m_pos = m_end_pos;
+    m_pos = m_file_length;
 }
 
 void IPakEntryReadStream::Close()
@@ -97,9 +283,9 @@ void IPakEntryReadStream::Close()
     {
         m_file = nullptr;
 
-        delete[] m_buffer;
-        m_buffer = nullptr;
+        delete[] m_file_buffer;
+        m_file_buffer = nullptr;
 
-        m_stream_manager->OnCloseStream(this);
+        m_stream_manager_actions->CloseStream(this);
     }
 }

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h

@@ -2,28 +2,48 @@
 
 #include "IPakStreamManager.h"
 #include "Utils/FileAPI.h"
+#include "ObjContainer/IPak/IPakTypes.h"
 #include <mutex>
 
 class IPakEntryReadStream final : public FileAPI::IFile
 {
-    static constexpr size_t IPAK_CHUNK_SIZE = 0x8000;
-    static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8;
+    uint8_t* m_chunk_buffer;
+    uint8_t* m_file_buffer;
 
-    uint8_t* m_buffer;
     IFile* m_file;
-    IPakStreamManager* m_stream_manager;
-    std::mutex* m_read_mutex;
+    IPakStreamManagerActions* m_stream_manager_actions;
+
+    size_t m_file_offset;
+    size_t m_file_head;
+    size_t m_file_length;
 
     int64_t m_pos;
     int64_t m_base_pos;
-    int64_t m_end_pos;
-    int64_t m_buffer_pos;
+    int64_t m_buffer_start_pos;
+    int64_t m_buffer_end_pos;
 
-    static int64_t Align(int64_t num, int64_t alignTo);
+    template<typename T>
+    static T AlignForward(const T num, const T alignTo)
+    {
+        return (num + alignTo - 1) / alignTo * alignTo;
+    }
+
+    template<typename T>
+    static T AlignBackwards(const T num, const T alignTo)
+    {
+        return num / alignTo * alignTo;
+    }
+
+    bool SetChunkBufferWindow(int64_t startPos, size_t chunkCount);
+    bool ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const;
+    bool AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, size_t blockOffsetInChunk);
+    bool ProcessCommand(size_t commandSize, bool compressed);
+    bool AdvanceStream();
 
 public:
-    IPakEntryReadStream(IFile* file, IPakStreamManager* streamManager, std::mutex* readMutex, int64_t startOffset, size_t length);
+    IPakEntryReadStream(IFile* file, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t fileSize);
     ~IPakEntryReadStream() override;
+
     bool IsOpen() override;
     size_t Read(void* buffer, size_t elementSize, size_t elementCount) override;
     size_t Write(const void* data, size_t elementSize, size_t elementCount) override;

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp

@@ -1,41 +1,154 @@
 #include "IPakStreamManager.h"
 #include "IPakEntryReadStream.h"
+#include "ObjContainer/IPak/IPakTypes.h"
+#include <vector>
 
-IPakStreamManager::IPakStreamManager(FileAPI::IFile* file)
+using namespace ipak_consts;
+
+class IPakStreamManager::Impl final : public IPakStreamManagerActions
+{
+    static constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
+
+    class ChunkBuffer
+    {
+    public:
+        IPakEntryReadStream* m_using_stream = nullptr;
+        uint8_t m_buffer[IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ]{};
+    };
+
+    class ManagedStream
+    {
+    public:
+        IPakEntryReadStream* m_stream;
+        ChunkBuffer* m_chunk_buffer;
+
+        ManagedStream(IPakEntryReadStream* stream, ChunkBuffer* chunkBuffer)
+        {
+            m_stream = stream;
+            m_chunk_buffer = chunkBuffer;
+        }
+    };
+
+    FileAPI::IFile* m_file;
+
+    std::mutex m_read_mutex;
+    std::mutex m_stream_mutex;
+
+    std::vector<ManagedStream> m_open_streams;
+    std::vector<ChunkBuffer*> m_chunk_buffers;
+
+public:
+    explicit Impl(FileAPI::IFile* file)
     {
         m_file = file;
+
+        m_chunk_buffers.push_back(new ChunkBuffer());
     }
 
-IPakStreamManager::~IPakStreamManager()
+    Impl(const Impl& other) = delete;
+    Impl(Impl&& other) noexcept = delete;
+
+    virtual ~Impl()
     {
         m_stream_mutex.lock();
+
         for (const auto& openStream : m_open_streams)
         {
-        openStream->Close();
+            openStream.m_stream->Close();
         }
         m_open_streams.clear();
+
         m_stream_mutex.unlock();
     }
 
-FileAPI::IFile* IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length)
-{
-    auto* stream = new IPakEntryReadStream(m_file, this, &m_read_mutex, startPosition, length);
+    Impl& operator=(const Impl& other) = delete;
+    Impl& operator=(Impl&& other) noexcept = delete;
 
+    FileAPI::IFile* OpenStream(const int64_t startPosition, const size_t length)
+    {
         m_stream_mutex.lock();
-    m_open_streams.push_back(stream);
+
+        ChunkBuffer* reservedChunkBuffer;
+        const auto freeChunkBuffer = std::find_if(m_chunk_buffers.begin(), m_chunk_buffers.end(),
+                                               [](ChunkBuffer* chunkBuffer)
+                                               {
+                                                   return chunkBuffer->m_using_stream == nullptr;
+                                               });
+
+        if (freeChunkBuffer == m_chunk_buffers.end())
+        {
+            reservedChunkBuffer = new ChunkBuffer();
+            m_chunk_buffers.push_back(reservedChunkBuffer);
+        }
+        else
+            reservedChunkBuffer = *freeChunkBuffer;
+
+        auto* stream = new IPakEntryReadStream(m_file, this, reservedChunkBuffer->m_buffer, startPosition, length);
+
+        reservedChunkBuffer->m_using_stream = stream;
+
+        m_open_streams.emplace_back(stream, reservedChunkBuffer);
+
         m_stream_mutex.unlock();
 
         return stream;
     }
 
-void IPakStreamManager::OnCloseStream(FileAPI::IFile* stream)
+    void StartReading() override
+    {
+        m_read_mutex.lock();
+    }
+
+    void StopReading() override
+    {
+        m_read_mutex.unlock();
+    }
+
+    void CloseStream(FileAPI::IFile* stream) override
     {
         m_stream_mutex.lock();
-    const auto openStreamEntry = std::find(m_open_streams.begin(), m_open_streams.end(), stream);
+
+        const auto openStreamEntry = std::find_if(m_open_streams.begin(), m_open_streams.end(),
+                                               [stream](const ManagedStream& managedStream)
+                                               {
+                                                   return managedStream.m_stream == stream;
+                                               });
 
         if (openStreamEntry != m_open_streams.end())
         {
+            auto* chunkBuffer = openStreamEntry->m_chunk_buffer;
             m_open_streams.erase(openStreamEntry);
+            chunkBuffer->m_using_stream = nullptr;
+
+            // Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers
+            if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT)
+            {
+                const auto chunkBufferEntry = std::find(m_chunk_buffers.begin(), m_chunk_buffers.end(), chunkBuffer);
+
+                if (chunkBufferEntry != m_chunk_buffers.end())
+                {
+                    m_chunk_buffers.erase(chunkBufferEntry);
+                    delete chunkBuffer;
                 }
+            }
+        }
+
         m_stream_mutex.unlock();
     }
+};
+
+IPakStreamManager::IPakStreamManager(FileAPI::IFile* file)
+{
+    m_impl = new Impl(file);
+}
+
+IPakStreamManager::~IPakStreamManager()
+{
+    delete m_impl;
+    m_impl = nullptr;
+}
+
+FileAPI::IFile* IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) const
+{
+    return m_impl->OpenStream(startPosition, length);
+}

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h

@@ -1,15 +1,22 @@
 #pragma once
 
 #include "Utils/FileAPI.h"
-#include <vector>
+#include <cstdint>
 #include <mutex>
 
+class IPakStreamManagerActions
+{
+public:
+    virtual void StartReading() = 0;
+    virtual void StopReading() = 0;
+
+    virtual void CloseStream(FileAPI::IFile* stream) = 0;
+};
+
 class IPakStreamManager
 {
-    FileAPI::IFile* m_file;
-    std::vector<FileAPI::IFile*> m_open_streams;
-    std::mutex m_read_mutex;
-    std::mutex m_stream_mutex;
+    class Impl;
+    Impl* m_impl;
 
 public:
     explicit IPakStreamManager(FileAPI::IFile* file);
@@ -20,6 +27,5 @@ public:
     IPakStreamManager& operator=(const IPakStreamManager& other) = delete;
     IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete;
 
-    FileAPI::IFile* OpenStream(int64_t startPosition, size_t length);
-    void OnCloseStream(FileAPI::IFile* stream);
+    FileAPI::IFile* OpenStream(int64_t startPosition, size_t length) const;
 };