ZoneLoading: Simulate the vanilla edge case that the chunk size is in the end of the loadbuffer and cannot quite fit into it so the space is padded and the chunk size is loaded from the beginning of the buffer

src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp

@@ -28,6 +28,7 @@ const int ZoneLoaderFactoryT6::VERSION = 147;
 
 const int ZoneLoaderFactoryT6::STREAM_COUNT = 4;
 const int ZoneLoaderFactoryT6::XCHUNK_SIZE = 0x8000;
+const int ZoneLoaderFactoryT6::VANILLA_BUFFER_SIZE = 0x80000;
 const int ZoneLoaderFactoryT6::OFFSET_BLOCK_BIT_COUNT = 3;
 const block_t ZoneLoaderFactoryT6::INSERT_BLOCK = T6::XFILE_BLOCK_VIRTUAL;
 
@@ -259,7 +260,7 @@ class ZoneLoaderFactoryT6::ZoneLoaderFactoryT6Impl
     static ISignatureDataProvider* AddXChunkProcessor(bool isEncrypted, ZoneLoader* zoneLoader, std::string& fileName)
     {
         ISignatureDataProvider* result = nullptr;
-        auto* xChunkProcessor = new ProcessorXChunks(STREAM_COUNT, XCHUNK_SIZE);
+        auto* xChunkProcessor = new ProcessorXChunks(STREAM_COUNT, XCHUNK_SIZE, VANILLA_BUFFER_SIZE);
 
         if(isEncrypted)
         {

src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.h

@@ -13,6 +13,7 @@ class ZoneLoaderFactoryT6 final : public IZoneLoaderFactory
 
     static const int STREAM_COUNT;
     static const int XCHUNK_SIZE;
+    static const int VANILLA_BUFFER_SIZE;
     static const int OFFSET_BLOCK_BIT_COUNT;
     static const block_t INSERT_BLOCK;
 

src/ZoneLoading/Loading/ILoadingStream.h

@@ -1,9 +1,12 @@
 #pragma once
 
+#include <cstdint>
+
 class ILoadingStream
 {
 public:
     virtual ~ILoadingStream() = default;
 
     virtual size_t Load(void* buffer, size_t length) = 0;
+    virtual int64_t Pos() = 0;
 };

src/ZoneLoading/Loading/LoadingFileStream.cpp

@@ -9,3 +9,8 @@ size_t LoadingFileStream::Load(void* buffer, const size_t length)
 {
     return m_file->Read(buffer, 1, length);
 }
+
+int64_t LoadingFileStream::Pos()
+{
+    return m_file->Pos();
+}

src/ZoneLoading/Loading/LoadingFileStream.h

@@ -2,7 +2,7 @@
 #include "ILoadingStream.h"
 #include "Utils/FileAPI.h"
 
-class LoadingFileStream : public ILoadingStream
+class LoadingFileStream final : public ILoadingStream
 {
     FileAPI::File* m_file;
 
@@ -10,4 +10,5 @@ public:
     explicit LoadingFileStream(FileAPI::File* file);
 
     size_t Load(void* buffer, size_t length) override;
+    int64_t Pos() override;
 };

src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp

@@ -33,9 +33,9 @@ class DBLoadStream
 
         bool firstProcessor = true;
 
-        for(auto processor : m_processors)
+        for (auto processor : m_processors)
         {
-            if(!firstProcessor)
+            if (!firstProcessor)
             {
                 uint8_t* previousInputBuffer = m_input_buffer;
                 m_input_buffer = m_output_buffer;
@@ -55,7 +55,8 @@ class DBLoadStream
     }
 
 public:
-    DBLoadStream(const int streamIndex, const size_t chunkSize, std::vector<IXChunkProcessor*>& chunkProcessors) : m_processors(chunkProcessors)
+    DBLoadStream(const int streamIndex, const size_t chunkSize,
+                 std::vector<IXChunkProcessor*>& chunkProcessors) : m_processors(chunkProcessors)
     {
         m_index = streamIndex;
         m_chunk_size = chunkSize;
@@ -85,11 +86,11 @@ public:
 
     void StartLoading(const size_t inputSize)
     {
-        if(inputSize > 0)
+        if (inputSize > 0)
         {
             std::unique_lock<std::mutex> lock(m_load_mutex);
 
-            if(m_is_loading)
+            if (m_is_loading)
             {
                 m_loading_finished.wait(lock);
             }
@@ -111,7 +112,7 @@ public:
         assert(pSize != nullptr);
 
         std::unique_lock<std::mutex> lock(m_load_mutex);
-        if(m_is_loading)
+        if (m_is_loading)
         {
             m_loading_finished.wait(lock);
         }
@@ -127,6 +128,7 @@ class ProcessorXChunks::ProcessorXChunksImpl
 
     std::vector<DBLoadStream*> m_streams;
     size_t m_chunk_size;
+    size_t m_vanilla_buffer_size;
     std::vector<IXChunkProcessor*> m_chunk_processors;
 
     bool m_initialized_streams;
@@ -134,6 +136,7 @@ class ProcessorXChunks::ProcessorXChunksImpl
     const uint8_t* m_current_chunk;
     size_t m_current_chunk_size;
     size_t m_current_chunk_offset;
+    size_t m_vanilla_buffer_offset;
 
     bool m_eof_reached;
     unsigned int m_eof_stream;
@@ -142,20 +145,31 @@ class ProcessorXChunks::ProcessorXChunksImpl
     {
         assert(streamNum >= 0 && streamNum < m_streams.size());
 
-        if(m_eof_reached)
+        if (m_eof_reached)
             return;
 
         xchunk_size_t chunkSize;
-        const size_t readSize = m_base->m_base_stream->Load(&chunkSize, sizeof(chunkSize));
+        if (m_vanilla_buffer_size > 0)
+        {
+            if (m_vanilla_buffer_offset + sizeof chunkSize > m_vanilla_buffer_size)
+            {
+                m_base->m_base_stream->Load(&chunkSize, m_vanilla_buffer_size - m_vanilla_buffer_offset);
+                m_vanilla_buffer_offset = 0;
+            }
 
-        if(readSize == 0)
+            m_vanilla_buffer_offset = (m_vanilla_buffer_offset + sizeof chunkSize) % m_vanilla_buffer_size;
+        }
+
+        const size_t readSize = m_base->m_base_stream->Load(&chunkSize, sizeof chunkSize);
+
+        if (readSize == 0)
         {
             m_eof_reached = true;
             m_eof_stream = streamNum;
             return;
         }
 
-        if(chunkSize > m_chunk_size)
+        if (chunkSize > m_chunk_size)
         {
             throw InvalidChunkSizeException(chunkSize, m_chunk_size);
         }
@@ -163,11 +177,16 @@ class ProcessorXChunks::ProcessorXChunksImpl
         auto* stream = m_streams[streamNum];
         const size_t loadedChunkSize = m_base->m_base_stream->Load(stream->GetInputBuffer(), chunkSize);
 
-        if(loadedChunkSize != chunkSize)
+        if (loadedChunkSize != chunkSize)
         {
             throw InvalidChunkSizeException(chunkSize);
         }
 
+        if (m_vanilla_buffer_size > 0)
+        {
+            m_vanilla_buffer_offset = (m_vanilla_buffer_offset + loadedChunkSize) % m_vanilla_buffer_size;
+        }
+
         stream->StartLoading(loadedChunkSize);
     }
 
@@ -183,9 +202,10 @@ class ProcessorXChunks::ProcessorXChunksImpl
     void InitStreams()
     {
         m_initialized_streams = true;
+        m_vanilla_buffer_offset = static_cast<size_t>(m_base->m_base_stream->Pos());
 
         const unsigned int streamCount = m_streams.size();
-        for(unsigned int streamNum = 0; streamNum < streamCount; streamNum++)
+        for (unsigned int streamNum = 0; streamNum < streamCount; streamNum++)
         {
             AdvanceStream(streamNum);
         }
@@ -209,32 +229,40 @@ public:
 
         m_base = base;
 
-        for(int streamIndex = 0; streamIndex < numStreams; streamIndex++)
+        for (int streamIndex = 0; streamIndex < numStreams; streamIndex++)
         {
             m_streams.push_back(new DBLoadStream(streamIndex, xChunkSize, m_chunk_processors));
         }
 
         m_chunk_size = xChunkSize;
+        m_vanilla_buffer_size = 0;
 
         m_initialized_streams = false;
         m_current_stream = 0;
         m_current_chunk = nullptr;
         m_current_chunk_size = 0;
         m_current_chunk_offset = 0;
+        m_vanilla_buffer_offset = 0;
 
         m_eof_reached = false;
         m_eof_stream = 0;
     }
 
+    ProcessorXChunksImpl(ProcessorXChunks* base, const int numStreams, const size_t xChunkSize,
+                         const size_t vanillaBufferSize) : ProcessorXChunksImpl(base, numStreams, xChunkSize)
+    {
+        m_vanilla_buffer_size = vanillaBufferSize;
+    }
+
     ~ProcessorXChunksImpl()
     {
-        for(auto* stream : m_streams)
+        for (auto* stream : m_streams)
         {
             delete stream;
         }
         m_streams.clear();
 
-        for(auto* processor : m_chunk_processors)
+        for (auto* processor : m_chunk_processors)
         {
             delete processor;
         }
@@ -252,19 +280,19 @@ public:
     {
         assert(buffer != nullptr);
 
-        if(!m_initialized_streams)
+        if (!m_initialized_streams)
         {
             InitStreams();
         }
 
         size_t loadedSize = 0;
-        while(!EndOfStream() && loadedSize < length)
+        while (!EndOfStream() && loadedSize < length)
         {
             auto* bufferPos = static_cast<uint8_t*>(buffer) + loadedSize;
             const size_t sizeToRead = length - loadedSize;
             const size_t bytesLeftInCurrentChunk = m_current_chunk_size - m_current_chunk_offset;
 
-            if(sizeToRead > bytesLeftInCurrentChunk)
+            if (sizeToRead > bytesLeftInCurrentChunk)
             {
                 memcpy_s(bufferPos, sizeToRead, &m_current_chunk[m_current_chunk_offset], bytesLeftInCurrentChunk);
                 loadedSize += bytesLeftInCurrentChunk;
@@ -277,7 +305,7 @@ public:
                 loadedSize += sizeToRead;
                 m_current_chunk_offset += sizeToRead;
 
-                if(m_current_chunk_offset == m_current_chunk_size)
+                if (m_current_chunk_offset == m_current_chunk_size)
                 {
                     NextStream();
                 }
@@ -286,6 +314,11 @@ public:
 
         return loadedSize;
     }
+
+    int64_t Pos() const
+    {
+        return m_base->m_base_stream->Pos();
+    }
 };
 
 ProcessorXChunks::ProcessorXChunks(const int numStreams, const size_t xChunkSize)
@@ -293,6 +326,11 @@ ProcessorXChunks::ProcessorXChunks(const int numStreams, const size_t xChunkSize
     m_impl = new ProcessorXChunksImpl(this, numStreams, xChunkSize);
 }
 
+ProcessorXChunks::ProcessorXChunks(const int numStreams, const size_t xChunkSize, const size_t vanillaBufferSize)
+{
+    m_impl = new ProcessorXChunksImpl(this, numStreams, xChunkSize, vanillaBufferSize);
+}
+
 ProcessorXChunks::~ProcessorXChunks()
 {
     delete m_impl;
@@ -308,3 +346,8 @@ size_t ProcessorXChunks::Load(void* buffer, const size_t length)
 {
     return m_impl->Load(buffer, length);
 }
+
+int64_t ProcessorXChunks::Pos()
+{
+    return m_impl->Pos();
+}

src/ZoneLoading/Loading/Processor/ProcessorXChunks.h

@@ -9,9 +9,11 @@ class ProcessorXChunks : public StreamProcessor
 
 public:
     ProcessorXChunks(int numStreams, size_t xChunkSize);
+    ProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize);
     ~ProcessorXChunks() override;
 
     size_t Load(void* buffer, size_t length) override;
+    int64_t Pos() override;
 
     void AddChunkProcessor(IXChunkProcessor* chunkProcessor) const;
 };