Rename ZoneLoader and ZoneWriter components to ZoneLoading and ZoneWriting to make a difference between the executive class and the component class

src/ZoneLoading/Loading/Processor/ProcessorInflate.cpp

@@ -0,0 +1,73 @@
+#include "ProcessorInflate.h"
+#include "zlib.h"
+#include <exception>
+#include "zutil.h"
+#include <cstdint>
+
+class ProcessorInflate::ProcessorInflateImpl
+{
+    z_stream m_stream{};
+    uint8_t m_in_buffer[0x800];
+    ProcessorInflate* m_base;
+
+public:
+    ProcessorInflateImpl(ProcessorInflate* baseClass)
+    {
+        m_base = baseClass;
+
+        m_stream.zalloc = Z_NULL;
+        m_stream.zfree = Z_NULL;
+        m_stream.opaque = Z_NULL;
+        m_stream.avail_in = 0;
+        m_stream.next_in = Z_NULL;
+
+        const int ret = inflateInit2(&m_stream, -DEF_WBITS);
+
+        if(ret != Z_OK)
+        {
+            throw std::exception("Initializing inflate failed");
+        }
+    }
+
+    ~ProcessorInflateImpl()
+    {
+        inflateEnd(&m_stream);
+    }
+
+    size_t Load(void* buffer, size_t length)
+    {
+        m_stream.next_out = static_cast<Bytef*>(buffer);
+        m_stream.avail_out = length;
+
+        while(m_stream.avail_out > 0)
+        {
+            if(m_stream.avail_in == 0)
+            {
+                m_stream.avail_in = m_base->m_base_stream->Load(m_in_buffer, sizeof(m_in_buffer));
+
+                if(m_stream.avail_in == 0) // EOF
+                    return length - m_stream.avail_out;
+            }
+
+            inflate(&m_stream, Z_FULL_FLUSH);
+        }
+
+        return m_stream.avail_out;
+    }
+};
+
+ProcessorInflate::ProcessorInflate()
+{
+    m_impl = new ProcessorInflateImpl(this);
+}
+
+ProcessorInflate::~ProcessorInflate()
+{
+    delete m_impl;
+    m_impl = nullptr;
+}
+
+size_t ProcessorInflate::Load(void* buffer, const size_t length)
+{
+    return m_impl->Load(buffer, length);
+}

src/ZoneLoading/Loading/Processor/ProcessorInflate.h

@@ -0,0 +1,14 @@
+#pragma once
+#include "Loading/StreamProcessor.h"
+
+class ProcessorInflate final : public StreamProcessor
+{
+    class ProcessorInflateImpl;
+    ProcessorInflateImpl* m_impl;
+
+public:
+    ProcessorInflate();
+    ~ProcessorInflate() override;
+
+    size_t Load(void* buffer, size_t length) override;
+};

src/ZoneLoading/Loading/Processor/ProcessorStreamCipher.cpp

@@ -0,0 +1,28 @@
+#include "ProcessorStreamCipher.h"
+
+ProcessorStreamCipher::ProcessorStreamCipher(IStreamCipher* cipher)
+{
+    m_cipher = cipher;
+}
+
+ProcessorStreamCipher::~ProcessorStreamCipher()
+{
+    delete m_cipher;
+    m_cipher = nullptr;
+}
+
+
+size_t ProcessorStreamCipher::Load(void* buffer, const size_t length)
+{
+    if(m_base_stream != nullptr)
+    {
+        const size_t readSize = m_base_stream->Load(buffer, length);
+
+        if(readSize > 0)
+            m_cipher->Process(buffer, buffer, readSize);
+
+        return readSize;
+    }
+
+    return 0;
+}

src/ZoneLoading/Loading/Processor/ProcessorStreamCipher.h

@@ -0,0 +1,14 @@
+#pragma once
+#include "Loading/StreamProcessor.h"
+#include "Crypto.h"
+
+class ProcessorStreamCipher final : public StreamProcessor
+{
+    IStreamCipher* m_cipher;
+
+public:
+    explicit ProcessorStreamCipher(IStreamCipher* cipher);
+    ~ProcessorStreamCipher() override;
+
+    size_t Load(void* buffer, size_t length) override;
+};

src/ZoneLoading/Loading/Processor/ProcessorXChunks.cpp

@@ -0,0 +1,310 @@
+#include "ProcessorXChunks.h"
+#include "Zone/ZoneTypes.h"
+#include "Loading/Exception/InvalidChunkSizeException.h"
+
+#include <vector>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <cassert>
+
+class DBLoadStream
+{
+    int m_index;
+
+    uint8_t* m_input_buffer;
+    size_t m_input_size;
+
+    uint8_t* m_output_buffer;
+    size_t m_output_size;
+
+    size_t m_chunk_size;
+
+    bool m_is_loading;
+    std::mutex m_load_mutex;
+    std::condition_variable m_loading_finished;
+    std::thread m_load_thread;
+
+    std::vector<IXChunkProcessor*>& m_processors;
+
+    void Load()
+    {
+        std::lock_guard<std::mutex> lock(m_load_mutex);
+
+        bool firstProcessor = true;
+
+        for(auto processor : m_processors)
+        {
+            if(!firstProcessor)
+            {
+                uint8_t* previousInputBuffer = m_input_buffer;
+                m_input_buffer = m_output_buffer;
+                m_output_buffer = previousInputBuffer;
+
+                m_input_size = m_output_size;
+                m_output_size = 0;
+            }
+
+            m_output_size = processor->Process(m_index, m_input_buffer, m_input_size, m_output_buffer, m_chunk_size);
+
+            firstProcessor = false;
+        }
+
+        m_is_loading = false;
+        m_loading_finished.notify_all();
+    }
+
+public:
+    DBLoadStream(const int streamIndex, const size_t chunkSize, std::vector<IXChunkProcessor*>& chunkProcessors) : m_processors(chunkProcessors)
+    {
+        m_index = streamIndex;
+        m_chunk_size = chunkSize;
+
+        m_input_buffer = new uint8_t[chunkSize];
+        m_output_buffer = new uint8_t[chunkSize];
+
+        m_input_size = 0;
+        m_output_size = 0;
+
+        m_is_loading = false;
+    }
+
+    ~DBLoadStream()
+    {
+        delete[] m_input_buffer;
+        m_input_buffer = nullptr;
+
+        delete[] m_output_buffer;
+        m_output_buffer = nullptr;
+    }
+
+    uint8_t* GetInputBuffer() const
+    {
+        return m_input_buffer;
+    }
+
+    void StartLoading(const size_t inputSize)
+    {
+        if(inputSize > 0)
+        {
+            std::unique_lock<std::mutex> lock(m_load_mutex);
+
+            if(m_is_loading)
+            {
+                m_loading_finished.wait(lock);
+            }
+
+            m_input_size = inputSize;
+            m_is_loading = true;
+            m_load_thread = std::thread(&DBLoadStream::Load, this);
+            m_load_thread.detach();
+        }
+        else
+        {
+            m_output_size = 0;
+        }
+    }
+
+    void GetOutput(const uint8_t** pBuffer, size_t* pSize)
+    {
+        assert(pBuffer != nullptr);
+        assert(pSize != nullptr);
+
+        std::unique_lock<std::mutex> lock(m_load_mutex);
+        if(m_is_loading)
+        {
+            m_loading_finished.wait(lock);
+        }
+
+        *pBuffer = m_output_buffer;
+        *pSize = m_output_size;
+    }
+};
+
+class ProcessorXChunks::ProcessorXChunksImpl
+{
+    ProcessorXChunks* m_base;
+
+    std::vector<DBLoadStream*> m_streams;
+    size_t m_chunk_size;
+    std::vector<IXChunkProcessor*> m_chunk_processors;
+
+    bool m_initialized_streams;
+    unsigned int m_current_stream;
+    const uint8_t* m_current_chunk;
+    size_t m_current_chunk_size;
+    size_t m_current_chunk_offset;
+
+    bool m_eof_reached;
+    unsigned int m_eof_stream;
+
+    void AdvanceStream(const unsigned int streamNum)
+    {
+        assert(streamNum >= 0 && streamNum < m_streams.size());
+
+        if(m_eof_reached)
+            return;
+
+        xchunk_size_t chunkSize;
+        const size_t readSize = m_base->m_base_stream->Load(&chunkSize, sizeof(chunkSize));
+
+        if(readSize == 0)
+        {
+            m_eof_reached = true;
+            m_eof_stream = streamNum;
+            return;
+        }
+
+        if(chunkSize > m_chunk_size)
+        {
+            throw InvalidChunkSizeException(chunkSize, m_chunk_size);
+        }
+
+        auto* stream = m_streams[streamNum];
+        const size_t loadedChunkSize = m_base->m_base_stream->Load(stream->GetInputBuffer(), chunkSize);
+
+        if(loadedChunkSize != chunkSize)
+        {
+            throw InvalidChunkSizeException(chunkSize);
+        }
+
+        stream->StartLoading(loadedChunkSize);
+    }
+
+    void NextStream()
+    {
+        AdvanceStream(m_current_stream);
+
+        m_current_stream = (m_current_stream + 1) % m_streams.size();
+        m_current_chunk_offset = 0;
+        m_streams[m_current_stream]->GetOutput(&m_current_chunk, &m_current_chunk_size);
+    }
+
+    void InitStreams()
+    {
+        m_initialized_streams = true;
+
+        const unsigned int streamCount = m_streams.size();
+        for(unsigned int streamNum = 0; streamNum < streamCount; streamNum++)
+        {
+            AdvanceStream(streamNum);
+        }
+
+        m_current_stream = 0;
+        m_current_chunk_offset = 0;
+        m_streams[0]->GetOutput(&m_current_chunk, &m_current_chunk_size);
+    }
+
+    bool EndOfStream() const
+    {
+        return m_eof_reached && m_eof_stream == m_current_stream;
+    }
+
+public:
+    ProcessorXChunksImpl(ProcessorXChunks* base, const int numStreams, const size_t xChunkSize)
+    {
+        assert(base != nullptr);
+        assert(numStreams > 0);
+        assert(xChunkSize > 0);
+
+        m_base = base;
+
+        for(int streamIndex = 0; streamIndex < numStreams; streamIndex++)
+        {
+            m_streams.push_back(new DBLoadStream(streamIndex, xChunkSize, m_chunk_processors));
+        }
+
+        m_chunk_size = xChunkSize;
+
+        m_initialized_streams = false;
+        m_current_stream = 0;
+        m_current_chunk = nullptr;
+        m_current_chunk_size = 0;
+        m_current_chunk_offset = 0;
+
+        m_eof_reached = false;
+        m_eof_stream = 0;
+    }
+
+    ~ProcessorXChunksImpl()
+    {
+        for(auto* stream : m_streams)
+        {
+            delete stream;
+        }
+        m_streams.clear();
+
+        for(auto* processor : m_chunk_processors)
+        {
+            delete processor;
+        }
+        m_chunk_processors.clear();
+    }
+
+    void AddChunkProcessor(IXChunkProcessor* streamProcessor)
+    {
+        assert(streamProcessor != nullptr);
+
+        m_chunk_processors.push_back(streamProcessor);
+    }
+
+    size_t Load(void* buffer, const size_t length)
+    {
+        assert(buffer != nullptr);
+
+        if(!m_initialized_streams)
+        {
+            InitStreams();
+        }
+
+        size_t loadedSize = 0;
+        while(!EndOfStream() && loadedSize < length)
+        {
+            auto* bufferPos = static_cast<uint8_t*>(buffer) + loadedSize;
+            const size_t sizeToRead = length - loadedSize;
+            const size_t bytesLeftInCurrentChunk = m_current_chunk_size - m_current_chunk_offset;
+
+            if(sizeToRead > bytesLeftInCurrentChunk)
+            {
+                memcpy_s(bufferPos, sizeToRead, &m_current_chunk[m_current_chunk_offset], bytesLeftInCurrentChunk);
+                loadedSize += bytesLeftInCurrentChunk;
+
+                NextStream();
+            }
+            else
+            {
+                memcpy_s(bufferPos, sizeToRead, &m_current_chunk[m_current_chunk_offset], sizeToRead);
+                loadedSize += sizeToRead;
+                m_current_chunk_offset += sizeToRead;
+
+                if(m_current_chunk_offset == m_current_chunk_size)
+                {
+                    NextStream();
+                }
+            }
+        }
+
+        return loadedSize;
+    }
+};
+
+ProcessorXChunks::ProcessorXChunks(const int numStreams, const size_t xChunkSize)
+{
+   m_impl = new ProcessorXChunksImpl(this, numStreams, xChunkSize);
+}
+
+ProcessorXChunks::~ProcessorXChunks()
+{
+    delete m_impl;
+    m_impl = nullptr;
+}
+
+void ProcessorXChunks::AddChunkProcessor(IXChunkProcessor* chunkProcessor) const
+{
+    m_impl->AddChunkProcessor(chunkProcessor);
+}
+
+size_t ProcessorXChunks::Load(void* buffer, const size_t length)
+{
+    return m_impl->Load(buffer, length);
+}

src/ZoneLoading/Loading/Processor/ProcessorXChunks.h

@@ -0,0 +1,17 @@
+#pragma once
+#include "Loading/StreamProcessor.h"
+#include "XChunks/IXChunkProcessor.h"
+
+class ProcessorXChunks : public StreamProcessor
+{
+    class ProcessorXChunksImpl;
+    ProcessorXChunksImpl* m_impl;
+
+public:
+    ProcessorXChunks(int numStreams, size_t xChunkSize);
+    ~ProcessorXChunks() override;
+
+    size_t Load(void* buffer, size_t length) override;
+
+    void AddChunkProcessor(IXChunkProcessor* chunkProcessor) const;
+};

src/ZoneLoading/Loading/Processor/XChunks/ChunkProcessorInflate.cpp

@@ -0,0 +1,36 @@
+#include "ChunkProcessorInflate.h"
+#include "zlib.h"
+#include "zutil.h"
+#include <exception>
+#include "Loading/Exception/InvalidCompressionException.h"
+
+size_t ChunkProcessorInflate::Process(int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
+{
+    z_stream stream{};
+    stream.zalloc = Z_NULL;
+    stream.zfree = Z_NULL;
+    stream.opaque = Z_NULL;
+
+    int ret = inflateInit2(&stream, -DEF_WBITS);
+    if(ret != Z_OK)
+    {
+        throw std::exception("Initializing inflate failed.");
+    }
+
+    stream.avail_in = inputLength;
+    stream.next_in = input;
+    stream.avail_out = outputBufferSize;
+    stream.next_out = output;
+
+    ret = inflate(&stream, Z_FULL_FLUSH);
+    if(ret != Z_STREAM_END)
+    {
+        throw InvalidCompressionException();
+    }
+
+    const size_t outputSize = stream.total_out;
+
+    inflateEnd(&stream);
+
+    return outputSize;
+}

src/ZoneLoading/Loading/Processor/XChunks/ChunkProcessorInflate.h

@@ -0,0 +1,8 @@
+#pragma once
+#include "IXChunkProcessor.h"
+
+class ChunkProcessorInflate : public IXChunkProcessor
+{
+public:
+    size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
+};

src/ZoneLoading/Loading/Processor/XChunks/ChunkProcessorSalsa20.cpp

@@ -0,0 +1,156 @@
+#include "ChunkProcessorSalsa20.h"
+#include "Crypto.h"
+#include <cassert>
+
+class StreamContextSalsa20
+{
+public:
+    IStreamCipher* m_salsa20;
+    IHashFunction* m_sha1;
+
+    StreamContextSalsa20()
+    {
+        m_salsa20 = nullptr;
+        m_sha1 = nullptr;
+    }
+
+    ~StreamContextSalsa20()
+    {
+        delete m_salsa20;
+        m_salsa20 = nullptr;
+
+        delete m_sha1;
+        m_sha1 = nullptr;
+    }
+};
+
+class ChunkProcessorSalsa20::ChunkProcessorSalsa20Impl
+{
+    static const int BLOCK_HASHES_COUNT = 200;
+    static const int SHA1_HASH_SIZE = 20;
+    static const int SALSA20_IV_SIZE = 8;
+
+    int m_stream_count;
+    StreamContextSalsa20* m_stream_contexts;
+
+    // m_block_hashes[BLOCK_HASHES_COUNT][numStreams][HASH_SIZE]
+    uint8_t* m_block_hashes;
+    unsigned int* m_stream_block_indices;
+
+    uint8_t* GetHashBlock(const int streamNumber) const
+    {
+        const size_t blockIndexOffset = m_stream_block_indices[streamNumber] * m_stream_count * SHA1_HASH_SIZE;
+        const size_t streamOffset = streamNumber * SHA1_HASH_SIZE;
+
+        return &m_block_hashes[blockIndexOffset + streamOffset];
+    }
+
+public:
+    ChunkProcessorSalsa20Impl(const int streamCount, std::string& zoneName, const uint8_t* salsa20Key, size_t keySize)
+    {
+        m_stream_count = streamCount;
+        m_stream_contexts = new StreamContextSalsa20[streamCount];
+        m_block_hashes = new uint8_t[BLOCK_HASHES_COUNT * streamCount * SHA1_HASH_SIZE];
+        m_stream_block_indices = new unsigned int[streamCount];
+
+        InitStreams(zoneName, salsa20Key, keySize);
+    }
+
+    ~ChunkProcessorSalsa20Impl()
+    {
+        delete[] m_stream_contexts;
+
+        delete[] m_block_hashes;
+        m_block_hashes = nullptr;
+
+        delete[] m_stream_block_indices;
+        m_stream_block_indices = nullptr;
+    }
+
+    void InitStreams(std::string& zoneName, const uint8_t* salsa20Key, size_t keySize) const
+    {
+        const int zoneNameLength = zoneName.length();
+        const size_t blockHashBufferSize = BLOCK_HASHES_COUNT * m_stream_count * SHA1_HASH_SIZE;
+
+        assert(blockHashBufferSize % 4 == 0);
+
+        size_t zoneNameOffset = 0;
+        for(size_t i = 0; i < blockHashBufferSize; i += 4)
+        {
+            *reinterpret_cast<uint32_t*>(&m_block_hashes[i]) = 0x1010101 * zoneName[zoneNameOffset++];
+
+            zoneNameOffset %= zoneNameLength;
+        }
+
+        for(int stream = 0; stream < m_stream_count; stream++)
+        {
+            m_stream_block_indices[stream] = 0;
+
+            m_stream_contexts[stream].m_salsa20 = Crypto::CreateSalsa20(salsa20Key, keySize);
+            m_stream_contexts[stream].m_sha1 = Crypto::CreateSHA1();
+        }
+    }
+
+    size_t Process(const int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize) const
+    {
+        assert(streamNumber >= 0 && streamNumber < m_stream_count);
+        assert(input != nullptr);
+        assert(output != nullptr);
+        assert(inputLength <= outputBufferSize);
+
+        StreamContextSalsa20& streamContext = m_stream_contexts[streamNumber];
+
+        // Initialize Salsa20 with an IV of the first 8 bytes of the current hash block
+        streamContext.m_salsa20->SetIV(GetHashBlock(streamNumber), SALSA20_IV_SIZE);
+        streamContext.m_salsa20->Process(input, output, inputLength);
+
+        // Hash decrypted XChunk
+        uint8_t blockSha1Hash[SHA1_HASH_SIZE];
+        streamContext.m_sha1->Init();
+        streamContext.m_sha1->Process(output, inputLength);
+        streamContext.m_sha1->Finish(&blockSha1Hash);
+
+        // Advance index to next hash block
+        m_stream_block_indices[streamNumber] = (m_stream_block_indices[streamNumber] + 1) % BLOCK_HASHES_COUNT;
+
+        uint8_t* nextHashBlock = GetHashBlock(streamNumber);
+
+        // XOR the upcoming hash block with the hash of the XChunk utilizing the previous hash block
+        for(unsigned int hashOffset = 0; hashOffset < sizeof(blockSha1Hash); hashOffset++)
+        {
+            nextHashBlock[hashOffset] ^= blockSha1Hash[hashOffset];
+        }
+
+        return inputLength;
+    }
+
+    void GetSignatureData(const uint8_t** pSignatureData, size_t* pSize) const
+    {
+        assert(pSignatureData != nullptr);
+        assert(pSize != nullptr);
+
+        *pSignatureData = m_block_hashes;
+        *pSize = BLOCK_HASHES_COUNT * m_stream_count * SHA1_HASH_SIZE;
+    }
+};
+
+ChunkProcessorSalsa20::ChunkProcessorSalsa20(const int streamCount, std::string& zoneName, const uint8_t* salsa20Key, size_t keySize)
+{
+    m_impl = new ChunkProcessorSalsa20Impl(streamCount, zoneName, salsa20Key, keySize);
+}
+
+ChunkProcessorSalsa20::~ChunkProcessorSalsa20()
+{
+    delete m_impl;
+    m_impl = nullptr;
+}
+
+size_t ChunkProcessorSalsa20::Process(const int streamNumber, const uint8_t* input, const size_t inputLength, uint8_t* output, const size_t outputBufferSize)
+{
+    return m_impl->Process(streamNumber, input, inputLength, output, outputBufferSize);
+}
+
+void ChunkProcessorSalsa20::GetSignatureData(const uint8_t** pSignatureData, size_t* pSize)
+{
+    m_impl->GetSignatureData(pSignatureData, pSize);
+}

src/ZoneLoading/Loading/Processor/XChunks/ChunkProcessorSalsa20.h

@@ -0,0 +1,17 @@
+#pragma once
+#include "IXChunkProcessor.h"
+#include "Loading/ISignatureDataProvider.h"
+#include <string>
+
+class ChunkProcessorSalsa20 : public IXChunkProcessor, public ISignatureDataProvider
+{
+    class ChunkProcessorSalsa20Impl;
+    ChunkProcessorSalsa20Impl* m_impl;
+
+public:
+    ChunkProcessorSalsa20(int streamCount, std::string& zoneName, const uint8_t* salsa20Key, size_t keySize);
+    ~ChunkProcessorSalsa20() override;
+
+    size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) override;
+    void GetSignatureData(const uint8_t** pSignatureData, size_t* pSize) override;
+};

src/ZoneLoading/Loading/Processor/XChunks/IXChunkProcessor.h

@@ -0,0 +1,9 @@
+#pragma once
+#include <cstdint>
+
+class IXChunkProcessor
+{
+public:
+    virtual ~IXChunkProcessor() = default;
+    virtual size_t Process(int streamNumber, const uint8_t* input, size_t inputLength, uint8_t* output, size_t outputBufferSize) = 0;
+};