ObjLoading: Add basis for IPakEntryReadStream to read ipak entries

src/ObjLoading/Game/T6/ObjLoaderT6.cpp

@@ -112,32 +112,39 @@ void ObjLoaderT6::LoadImageFromIwi(T6::GfxImage* image, ISearchPath* searchPath,
     Texture* loadedTexture = nullptr;
     IwiLoader loader(zone->GetMemory());
 
-    const std::string imageFileName = "images/" + std::string(image->name) + ".iwi";
+    if (image->streamedPartCount > 0)
-    auto* filePathImage = searchPath->Open(imageFileName);
-
-    if (filePathImage != nullptr && filePathImage->IsOpen())
-    {
-        loadedTexture = loader.LoadIwi(filePathImage);
-
-        filePathImage->Close();
-        delete filePathImage;
-    }
-    else if (image->streamedPartCount > 0)
     {
         for (auto* ipak : IPak::Repository)
         {
-            auto* ipakEntry = ipak->GetEntryData(image->hash, image->streamedParts[0].hash);
+            auto* ipakStream = ipak->GetEntryStream(image->hash, image->streamedParts[0].hash);
 
-            if (ipakEntry != nullptr && ipakEntry->IsOpen())
+            if (ipakStream != nullptr)
             {
-                loadedTexture = loader.LoadIwi(ipakEntry);
+                loadedTexture = loader.LoadIwi(ipakStream);
 
-                ipakEntry->Close();
+                ipakStream->Close();
-                delete ipakEntry;
+                delete ipakStream;
+
+                if (loadedTexture != nullptr)
+                    break;
             }
         }
     }
 
+    if(loadedTexture == nullptr)
+    {
+        const std::string imageFileName = "images/" + std::string(image->name) + ".iwi";
+        auto* filePathImage = searchPath->Open(imageFileName);
+
+        if (filePathImage != nullptr)
+        {
+            loadedTexture = loader.LoadIwi(filePathImage);
+
+            filePathImage->Close();
+            delete filePathImage;
+        }
+    }
+
     if(loadedTexture != nullptr)
     {
         image->texture.texture = loadedTexture;

src/ObjLoading/Image/IwiLoader.cpp

@@ -7,18 +7,35 @@ IwiLoader::IwiLoader(MemoryManager* memoryManager)
 
 Texture* IwiLoader::LoadIwi(FileAPI::IFile* file)
 {
-    struct
+    struct IWIHeaderMeta
     {
         char tag[3];
         char version;
     } iwiHeaderMeta{};
 
-    if (file->Read(&iwiHeaderMeta, sizeof iwiHeaderMeta, 1) != 1)
+    file->GotoEnd();
-        return nullptr;
+    auto iwiSize = static_cast<size_t>(file->Pos());
+    file->Goto(0);
 
-    printf("Read IWI with version %i\n", iwiHeaderMeta.version);
+    auto* buffer = new uint8_t[iwiSize];
+    if(file->Read(buffer, 1, iwiSize) != iwiSize)
+    {
+        delete[] buffer;
+        return nullptr;
+    }
+
+    auto* meta = reinterpret_cast<IWIHeaderMeta*>(buffer);
+
+    printf("Read IWI with version %i\n", meta->version);
 
     // TODO: Read iwi based on version
 
     return nullptr;
+
+    // if (file->Read(&iwiHeaderMeta, sizeof iwiHeaderMeta, 1) != 1)
+    //     return nullptr;
+    //
+    // printf("Read IWI with version %i\n", iwiHeaderMeta.version);
+    //
+    // return nullptr;
 }

src/ObjLoading/ObjContainer/IPak/IPak.cpp

@@ -3,6 +3,7 @@
 #include "Exception/IPakLoadException.h"
 #include "ObjContainer/IPak/IPakTypes.h"
 #include "Utils/PathUtils.h"
+#include "IPakStreamManager.h"
 
 #include <sstream>
 #include <vector>
@@ -24,6 +25,8 @@ class IPak::Impl : public ObjContainerReferenceable
 
     std::vector<IPakIndexEntry> m_index_entries;
 
+    IPakStreamManager m_stream_manager;
+
     static uint32_t R_HashString(const char* str, uint32_t hash)
     {
         for (const char* pos = str; *pos; pos++)
@@ -50,6 +53,12 @@ class IPak::Impl : public ObjContainerReferenceable
             m_index_entries.push_back(indexEntry);
         }
 
+        std::sort(m_index_entries.begin(), m_index_entries.end(),
+                  [](const IPakIndexEntry& entry1, const IPakIndexEntry& entry2)
+                  {
+                      return entry1.key.combinedKey < entry2.key.combinedKey;
+                  });
+
         return true;
     }
 
@@ -128,6 +137,7 @@ class IPak::Impl : public ObjContainerReferenceable
 
 public:
     Impl(std::string path, FileAPI::IFile* file)
+        : m_stream_manager(file)
     {
         m_path = std::move(path);
         m_file = file;
@@ -164,12 +174,19 @@ public:
 
     FileAPI::IFile* GetEntryData(const Hash nameHash, const Hash dataHash)
     {
-        for(auto& entry : m_index_entries)
+        IPakIndexEntryKey wantedKey{};
+        wantedKey.nameHash = nameHash;
+        wantedKey.dataHash = dataHash;
+
+        for (auto& entry : m_index_entries)
         {
-            if(entry.key.nameHash == nameHash && entry.key.dataHash == dataHash)
+            if (entry.key.combinedKey == wantedKey.combinedKey)
             {
-                // TODO: Implement ipak reader as IFile interface and use here
+                return m_stream_manager.OpenStream(entry.offset, entry.size);
-                __asm nop;
+            }
+            else if (entry.key.combinedKey > wantedKey.combinedKey)
+            {
+                // The index entries are sorted so if the current entry is higher than the wanted entry we can cancel here
                 return nullptr;
             }
         }
@@ -209,7 +226,7 @@ bool IPak::Initialize() const
     return m_impl->Initialize();
 }
 
-FileAPI::IFile* IPak::GetEntryData(const Hash nameHash, const Hash dataHash) const
+FileAPI::IFile* IPak::GetEntryStream(const Hash nameHash, const Hash dataHash) const
 {
     return m_impl->GetEntryData(nameHash, dataHash);
 }
@@ -222,4 +239,4 @@ IPak::Hash IPak::HashString(const std::string& str)
 IPak::Hash IPak::HashData(const void* data, const size_t dataSize)
 {
     return Impl::HashData(data, dataSize);
-}
+}

src/ObjLoading/ObjContainer/IPak/IPak.h

@@ -21,7 +21,7 @@ public:
     std::string GetName() override;
 
     bool Initialize() const;
-    FileAPI::IFile* GetEntryData(Hash nameHash, Hash dataHash) const;
+    FileAPI::IFile* GetEntryStream(Hash nameHash, Hash dataHash) const;
 
     static Hash HashString(const std::string& str);
     static Hash HashData(const void* data, size_t dataSize);

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp

@@ -0,0 +1,105 @@
+#include "IPakEntryReadStream.h"
+#include <cassert>
+#include <stdexcept>
+
+int64_t IPakEntryReadStream::Align(const int64_t num, const int64_t alignTo)
+{
+    return (num + alignTo - 1) / alignTo * alignTo;
+}
+
+IPakEntryReadStream::IPakEntryReadStream(IFile* file, IPakStreamManager* streamManager, std::mutex* readMutex, const int64_t startOffset, const size_t length)
+{
+    m_file = file;
+    m_base_pos = startOffset;
+    m_end_pos = startOffset + length;
+    m_buffer = new uint8_t[IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ];
+    m_stream_manager = streamManager;
+    m_read_mutex = readMutex;
+
+    m_buffer_pos = 0;
+    m_pos = m_base_pos;
+}
+
+IPakEntryReadStream::~IPakEntryReadStream()
+{
+    Close();
+}
+
+bool IPakEntryReadStream::IsOpen()
+{
+    return m_file != nullptr;
+}
+
+size_t IPakEntryReadStream::Read(void* buffer, const size_t elementSize, const size_t elementCount)
+{
+    const size_t readSize = elementCount * elementSize;
+    size_t chunksToRead = Align((m_pos % IPAK_CHUNK_SIZE) + readSize, IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE;
+
+    return 0;
+}
+
+size_t IPakEntryReadStream::Write(const void* data, size_t elementSize, size_t elementCount)
+{
+    // This is not meant for writing.
+    assert(false);
+    throw std::runtime_error("This is not a stream for output!");
+}
+
+void IPakEntryReadStream::Skip(const int64_t amount)
+{
+    if(amount > 0)
+    {
+        m_pos += amount;
+
+        if (m_pos > m_end_pos)
+            m_pos = m_end_pos;
+        else if (m_pos < m_base_pos)
+            m_pos = m_base_pos;
+    }
+}
+
+size_t IPakEntryReadStream::Printf(const char* fmt, ...)
+{
+    // This is not meant for writing.
+    assert(false);
+    throw std::runtime_error("This is not a stream for output!");
+}
+
+int64_t IPakEntryReadStream::Pos()
+{
+    return m_pos - m_base_pos;
+}
+
+void IPakEntryReadStream::Goto(const int64_t pos)
+{
+    if(pos <= 0)
+    {
+        m_pos = m_base_pos;
+    }
+    else if(pos < m_end_pos - m_base_pos)
+    {
+        m_pos = pos + m_base_pos;
+    }
+    else
+    {
+        m_pos = m_end_pos;
+    }
+}
+
+void IPakEntryReadStream::GotoEnd()
+{
+    m_pos = m_end_pos;
+}
+
+void IPakEntryReadStream::Close()
+{
+    if(IsOpen())
+    {
+        m_file = nullptr;
+
+        delete[] m_buffer;
+        m_buffer = nullptr;
+
+        m_stream_manager->OnCloseStream(this);
+    }
+}

src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "IPakStreamManager.h"
+#include "Utils/FileAPI.h"
+#include <mutex>
+
+class IPakEntryReadStream final : public FileAPI::IFile
+{
+    static constexpr size_t IPAK_CHUNK_SIZE = 0x8000;
+    static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8;
+
+    uint8_t* m_buffer;
+    IFile* m_file;
+    IPakStreamManager* m_stream_manager;
+    std::mutex* m_read_mutex;
+
+    int64_t m_pos;
+    int64_t m_base_pos;
+    int64_t m_end_pos;
+    int64_t m_buffer_pos;
+
+    static int64_t Align(int64_t num, int64_t alignTo);
+
+public:
+    IPakEntryReadStream(IFile* file, IPakStreamManager* streamManager, std::mutex* readMutex, int64_t startOffset, size_t length);
+    ~IPakEntryReadStream() override;
+    bool IsOpen() override;
+    size_t Read(void* buffer, size_t elementSize, size_t elementCount) override;
+    size_t Write(const void* data, size_t elementSize, size_t elementCount) override;
+    void Skip(int64_t amount) override;
+    size_t Printf(const char* fmt, ...) override;
+    int64_t Pos() override;
+    void Goto(int64_t pos) override;
+    void GotoEnd() override;
+    void Close() override;
+};

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp

@@ -0,0 +1,41 @@
+#include "IPakStreamManager.h"
+#include "IPakEntryReadStream.h"
+
+IPakStreamManager::IPakStreamManager(FileAPI::IFile* file)
+{
+    m_file = file;
+}
+
+IPakStreamManager::~IPakStreamManager()
+{
+    m_stream_mutex.lock();
+    for(const auto& openStream : m_open_streams)
+    {
+        openStream->Close();
+    }
+    m_open_streams.clear();
+    m_stream_mutex.unlock();
+}
+
+FileAPI::IFile* IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length)
+{
+    auto* stream = new IPakEntryReadStream(m_file, this, &m_read_mutex, startPosition, length);
+
+    m_stream_mutex.lock();
+    m_open_streams.push_back(stream);
+    m_stream_mutex.unlock();
+
+    return stream;
+}
+
+void IPakStreamManager::OnCloseStream(FileAPI::IFile* stream)
+{
+    m_stream_mutex.lock();
+    const auto openStreamEntry = std::find(m_open_streams.begin(), m_open_streams.end(), stream);
+
+    if(openStreamEntry != m_open_streams.end())
+    {
+        m_open_streams.erase(openStreamEntry);
+    }
+    m_stream_mutex.unlock();
+}

src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "Utils/FileAPI.h"
+#include <vector>
+#include <mutex>
+
+class IPakStreamManager
+{
+    FileAPI::IFile* m_file;
+    std::vector<FileAPI::IFile*> m_open_streams;
+    std::mutex m_read_mutex;
+    std::mutex m_stream_mutex;
+
+public:
+    explicit IPakStreamManager(FileAPI::IFile* file);
+    IPakStreamManager(const IPakStreamManager& other) = delete;
+    IPakStreamManager(IPakStreamManager&& other) noexcept = delete;
+    ~IPakStreamManager();
+
+    IPakStreamManager& operator=(const IPakStreamManager& other) = delete;
+    IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete;
+
+    FileAPI::IFile* OpenStream(int64_t startPosition, size_t length);
+    void OnCloseStream(FileAPI::IFile* stream);
+};