From d224eb8ce5d8b2b462f4456227a3351c9421af44 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 24 Dec 2019 17:21:58 +0100 Subject: [PATCH] ZoneLoading: Simulate the vanilla edge case that the chunk size is in the end of the loadbuffer and cannot quite fit into it so the space is padded and the chunk size is loaded from the beginning of the buffer --- .../Game/T6/ZoneLoaderFactoryT6.cpp | 3 +- src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.h | 1 + src/ZoneLoading/Loading/ILoadingStream.h | 3 + src/ZoneLoading/Loading/LoadingFileStream.cpp | 5 ++ src/ZoneLoading/Loading/LoadingFileStream.h | 3 +- .../Loading/Processor/ProcessorXChunks.cpp | 83 ++++++++++++++----- .../Loading/Processor/ProcessorXChunks.h | 2 + 7 files changed, 78 insertions(+), 22 deletions(-) diff --git a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp index b3ccbc9c..c4094ef9 100644 --- a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp +++ b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.cpp @@ -28,6 +28,7 @@ const int ZoneLoaderFactoryT6::VERSION = 147; const int ZoneLoaderFactoryT6::STREAM_COUNT = 4; const int ZoneLoaderFactoryT6::XCHUNK_SIZE = 0x8000; +const int ZoneLoaderFactoryT6::VANILLA_BUFFER_SIZE = 0x80000; const int ZoneLoaderFactoryT6::OFFSET_BLOCK_BIT_COUNT = 3; const block_t ZoneLoaderFactoryT6::INSERT_BLOCK = T6::XFILE_BLOCK_VIRTUAL; @@ -259,7 +260,7 @@ class ZoneLoaderFactoryT6::ZoneLoaderFactoryT6Impl static ISignatureDataProvider* AddXChunkProcessor(bool isEncrypted, ZoneLoader* zoneLoader, std::string& fileName) { ISignatureDataProvider* result = nullptr; - auto* xChunkProcessor = new ProcessorXChunks(STREAM_COUNT, XCHUNK_SIZE); + auto* xChunkProcessor = new ProcessorXChunks(STREAM_COUNT, XCHUNK_SIZE, VANILLA_BUFFER_SIZE); if(isEncrypted) { diff --git a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.h b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.h index 4240c4ef..19196ef2 100644 --- a/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.h +++ b/src/ZoneLoading/Game/T6/ZoneLoaderFactoryT6.h @@ -13,6 +13,7 @@ class ZoneLoaderFactoryT6 final : public IZoneLoaderFactory static const int STREAM_COUNT; static const int XCHUNK_SIZE; + static const int VANILLA_BUFFER_SIZE; static const int OFFSET_BLOCK_BIT_COUNT; static const block_t INSERT_BLOCK; diff --git a/src/ZoneLoading/Loading/ILoadingStream.h b/src/ZoneLoading/Loading/ILoadingStream.h index 00c92f9e..65cd4ba5 100644 --- a/src/ZoneLoading/Loading/ILoadingStream.h +++ b/src/ZoneLoading/Loading/ILoadingStream.h @@ -1,9 +1,12 @@ #pragma once +#include + class ILoadingStream { public: virtual ~ILoadingStream() = default; virtual size_t Load(void* buffer, size_t length) = 0; + virtual int64_t Pos() = 0; }; \ No newline at end of file diff --git a/src/ZoneLoading/Loading/LoadingFileStream.cpp b/src/ZoneLoading/Loading/LoadingFileStream.cpp index 022b8778..1d6cc4ff 100644 --- a/src/ZoneLoading/Loading/LoadingFileStream.cpp +++ b/src/ZoneLoading/Loading/LoadingFileStream.cpp @@ -8,4 +8,9 @@ LoadingFileStream::LoadingFileStream(FileAPI::File* file) size_t LoadingFileStream::Load(void* buffer, const size_t length) { return m_file->Read(buffer, 1, length); +} + +int64_t LoadingFileStream::Pos() +{ + return m_file->Pos(); } \ No newline at end of file diff --git a/src/ZoneLoading/Loading/LoadingFileStream.h b/src/ZoneLoading/Loading/LoadingFileStream.h index f20a9acf..ceec1f42 100644 --- a/src/ZoneLoading/Loading/LoadingFileStream.h +++ b/src/ZoneLoading/Loading/LoadingFileStream.h @@ -2,7 +2,7 @@ #include "ILoadingStream.h" #include "Utils/FileAPI.h" -class LoadingFileStream : public ILoadingStream +class LoadingFileStream final : public ILoadingStream { FileAPI::File* m_file; @@ -10,4 +10,5 @@ public: explicit LoadingFileStream(FileAPI::File* file); size_t Load(void* buffer, size_t length) override; + int64_t Pos() override; }; diff --git a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp index 13934b5f..ecfb89ca 100644 --- a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp +++ b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp @@ -33,9 +33,9 @@ class DBLoadStream bool firstProcessor = true; - for(auto processor : m_processors) + for (auto processor : m_processors) { - if(!firstProcessor) + if (!firstProcessor) { uint8_t* previousInputBuffer = m_input_buffer; m_input_buffer = m_output_buffer; @@ -55,7 +55,8 @@ class DBLoadStream } public: - DBLoadStream(const int streamIndex, const size_t chunkSize, std::vector& chunkProcessors) : m_processors(chunkProcessors) + DBLoadStream(const int streamIndex, const size_t chunkSize, + std::vector& chunkProcessors) : m_processors(chunkProcessors) { m_index = streamIndex; m_chunk_size = chunkSize; @@ -85,11 +86,11 @@ public: void StartLoading(const size_t inputSize) { - if(inputSize > 0) + if (inputSize > 0) { std::unique_lock lock(m_load_mutex); - if(m_is_loading) + if (m_is_loading) { m_loading_finished.wait(lock); } @@ -111,7 +112,7 @@ public: assert(pSize != nullptr); std::unique_lock lock(m_load_mutex); - if(m_is_loading) + if (m_is_loading) { m_loading_finished.wait(lock); } @@ -127,6 +128,7 @@ class ProcessorXChunks::ProcessorXChunksImpl std::vector m_streams; size_t m_chunk_size; + size_t m_vanilla_buffer_size; std::vector m_chunk_processors; bool m_initialized_streams; @@ -134,6 +136,7 @@ class ProcessorXChunks::ProcessorXChunksImpl const uint8_t* m_current_chunk; size_t m_current_chunk_size; size_t m_current_chunk_offset; + size_t m_vanilla_buffer_offset; bool m_eof_reached; unsigned int m_eof_stream; @@ -142,20 +145,31 @@ class ProcessorXChunks::ProcessorXChunksImpl { assert(streamNum >= 0 && streamNum < m_streams.size()); - if(m_eof_reached) + if (m_eof_reached) return; xchunk_size_t chunkSize; - const size_t readSize = m_base->m_base_stream->Load(&chunkSize, sizeof(chunkSize)); + if (m_vanilla_buffer_size > 0) + { + if (m_vanilla_buffer_offset + sizeof chunkSize > m_vanilla_buffer_size) + { + m_base->m_base_stream->Load(&chunkSize, m_vanilla_buffer_size - m_vanilla_buffer_offset); + m_vanilla_buffer_offset = 0; + } - if(readSize == 0) + m_vanilla_buffer_offset = (m_vanilla_buffer_offset + sizeof chunkSize) % m_vanilla_buffer_size; + } + + const size_t readSize = m_base->m_base_stream->Load(&chunkSize, sizeof chunkSize); + + if (readSize == 0) { m_eof_reached = true; m_eof_stream = streamNum; return; } - if(chunkSize > m_chunk_size) + if (chunkSize > m_chunk_size) { throw InvalidChunkSizeException(chunkSize, m_chunk_size); } @@ -163,11 +177,16 @@ class ProcessorXChunks::ProcessorXChunksImpl auto* stream = m_streams[streamNum]; const size_t loadedChunkSize = m_base->m_base_stream->Load(stream->GetInputBuffer(), chunkSize); - if(loadedChunkSize != chunkSize) + if (loadedChunkSize != chunkSize) { throw InvalidChunkSizeException(chunkSize); } + if (m_vanilla_buffer_size > 0) + { + m_vanilla_buffer_offset = (m_vanilla_buffer_offset + loadedChunkSize) % m_vanilla_buffer_size; + } + stream->StartLoading(loadedChunkSize); } @@ -183,9 +202,10 @@ class ProcessorXChunks::ProcessorXChunksImpl void InitStreams() { m_initialized_streams = true; + m_vanilla_buffer_offset = static_cast(m_base->m_base_stream->Pos()); const unsigned int streamCount = m_streams.size(); - for(unsigned int streamNum = 0; streamNum < streamCount; streamNum++) + for (unsigned int streamNum = 0; streamNum < streamCount; streamNum++) { AdvanceStream(streamNum); } @@ -209,32 +229,40 @@ public: m_base = base; - for(int streamIndex = 0; streamIndex < numStreams; streamIndex++) + for (int streamIndex = 0; streamIndex < numStreams; streamIndex++) { m_streams.push_back(new DBLoadStream(streamIndex, xChunkSize, m_chunk_processors)); } m_chunk_size = xChunkSize; + m_vanilla_buffer_size = 0; m_initialized_streams = false; m_current_stream = 0; m_current_chunk = nullptr; m_current_chunk_size = 0; m_current_chunk_offset = 0; + m_vanilla_buffer_offset = 0; m_eof_reached = false; m_eof_stream = 0; } + ProcessorXChunksImpl(ProcessorXChunks* base, const int numStreams, const size_t xChunkSize, + const size_t vanillaBufferSize) : ProcessorXChunksImpl(base, numStreams, xChunkSize) + { + m_vanilla_buffer_size = vanillaBufferSize; + } + ~ProcessorXChunksImpl() { - for(auto* stream : m_streams) + for (auto* stream : m_streams) { delete stream; } m_streams.clear(); - for(auto* processor : m_chunk_processors) + for (auto* processor : m_chunk_processors) { delete processor; } @@ -252,19 +280,19 @@ public: { assert(buffer != nullptr); - if(!m_initialized_streams) + if (!m_initialized_streams) { InitStreams(); } size_t loadedSize = 0; - while(!EndOfStream() && loadedSize < length) + while (!EndOfStream() && loadedSize < length) { auto* bufferPos = static_cast(buffer) + loadedSize; const size_t sizeToRead = length - loadedSize; const size_t bytesLeftInCurrentChunk = m_current_chunk_size - m_current_chunk_offset; - if(sizeToRead > bytesLeftInCurrentChunk) + if (sizeToRead > bytesLeftInCurrentChunk) { memcpy_s(bufferPos, sizeToRead, &m_current_chunk[m_current_chunk_offset], bytesLeftInCurrentChunk); loadedSize += bytesLeftInCurrentChunk; @@ -277,7 +305,7 @@ public: loadedSize += sizeToRead; m_current_chunk_offset += sizeToRead; - if(m_current_chunk_offset == m_current_chunk_size) + if (m_current_chunk_offset == m_current_chunk_size) { NextStream(); } @@ -286,11 +314,21 @@ public: return loadedSize; } + + int64_t Pos() const + { + return m_base->m_base_stream->Pos(); + } }; ProcessorXChunks::ProcessorXChunks(const int numStreams, const size_t xChunkSize) { - m_impl = new ProcessorXChunksImpl(this, numStreams, xChunkSize); + m_impl = new ProcessorXChunksImpl(this, numStreams, xChunkSize); +} + +ProcessorXChunks::ProcessorXChunks(const int numStreams, const size_t xChunkSize, const size_t vanillaBufferSize) +{ + m_impl = new ProcessorXChunksImpl(this, numStreams, xChunkSize, vanillaBufferSize); } ProcessorXChunks::~ProcessorXChunks() @@ -307,4 +345,9 @@ void ProcessorXChunks::AddChunkProcessor(IXChunkProcessor* chunkProcessor) const size_t ProcessorXChunks::Load(void* buffer, const size_t length) { return m_impl->Load(buffer, length); +} + +int64_t ProcessorXChunks::Pos() +{ + return m_impl->Pos(); } \ No newline at end of file diff --git a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.h b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.h index 9cb0c0d9..b8a7cced 100644 --- a/src/ZoneLoading/Loading/Processor/ProcessorXChunks.h +++ b/src/ZoneLoading/Loading/Processor/ProcessorXChunks.h @@ -9,9 +9,11 @@ class ProcessorXChunks : public StreamProcessor public: ProcessorXChunks(int numStreams, size_t xChunkSize); + ProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize); ~ProcessorXChunks() override; size_t Load(void* buffer, size_t length) override; + int64_t Pos() override; void AddChunkProcessor(IXChunkProcessor* chunkProcessor) const; };