Write XChunks in fastfiles

src/Linker/Linker.cpp

@@ -443,7 +443,7 @@ class Linker::Impl
 
         fs::create_directories(zoneFolderPath);
 
-        std::ofstream stream(zoneFilePath);
+        std::ofstream stream(zoneFilePath, std::fstream::out | std::fstream::binary);
         if (!stream.is_open())
             return false;
 

src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp

@@ -1,5 +1,7 @@
 #include "XChunkProcessorDeflate.h"
 
+
+#include <cassert>
 #include <zlib.h>
 #include <zutil.h>
 
@@ -23,7 +25,7 @@ size_t XChunkProcessorDeflate::Process(int streamNumber, const uint8_t* input, c
 
     ret = deflate(&stream, Z_FINISH);
     if (ret != Z_STREAM_END)
-        throw XChunkException("Zone has invalid or unsupported compression. Deflate failed");
+        throw XChunkException("Failed to deflate memory of zone.");
 
     const size_t outputSize = stream.total_out;
 

src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp

@@ -5,7 +5,7 @@
 #include "Crypto.h"
 #include "AbstractSalsa20Processor.h"
 
-XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const int streamCount, std::string& zoneName, const uint8_t* salsa20Key, size_t keySize)
+XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const int streamCount, std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize)
     : AbstractSalsa20Processor(streamCount, zoneName, salsa20Key, keySize)
 {
 }

src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp

@@ -15,6 +15,7 @@
 #include "Writing/Steps/StepWriteZoneContentToFile.h"
 #include "Writing/Steps/StepWriteZoneContentToMemory.h"
 #include "Writing/Steps/StepWriteZoneHeader.h"
+#include "Writing/Steps/StepWriteZoneSizes.h"
 
 using namespace T6;
 
@@ -74,6 +75,9 @@ public:
         ICapturedDataProvider* result = nullptr;
         auto xChunkProcessor = std::make_unique<OutputProcessorXChunks>(ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, ZoneConstants::VANILLA_BUFFER_SIZE);
 
+        // Decompress the chunks using zlib
+        xChunkProcessor->AddChunkProcessor(std::make_unique<XChunkProcessorDeflate>());
+
         if (isEncrypted)
         {
             // If zone is encrypted, the decryption is applied before the decompression. T6 Zones always use Salsa20.
@@ -83,8 +87,6 @@ public:
             xChunkProcessor->AddChunkProcessor(std::move(chunkProcessorSalsa20));
         }
 
-        // Decompress the chunks using zlib
-        xChunkProcessor->AddChunkProcessor(std::make_unique<XChunkProcessorDeflate>());
         m_writer->AddWritingStep(std::make_unique<StepAddOutputProcessor>(std::move(xChunkProcessor)));
 
         // If there is encryption, the signed data of the zone is the final hash blocks provided by the Salsa20 IV adaption algorithm
@@ -111,6 +113,7 @@ public:
 
         // Start of the XFile struct
         //m_writer->AddWritingStep(std::make_unique<StepSkipBytes>(8)); // Skip size and externalSize fields since they are not interesting for us
+        m_writer->AddWritingStep(std::make_unique<StepWriteZoneSizes>(contentInMemoryPtr));
         m_writer->AddWritingStep(std::make_unique<StepWriteXBlockSizes>(m_zone));
 
         // Start of the zone content

src/ZoneWriting/Writing/IWritingStream.h

@@ -14,5 +14,6 @@ public:
     IWritingStream& operator=(IWritingStream&& other) noexcept = default;
 
     virtual void Write(const void* buffer, size_t length) = 0;
+    virtual void Flush() = 0;
     virtual int64_t Pos() = 0;
 };

src/ZoneWriting/Writing/InMemoryZoneData.h

@@ -5,7 +5,7 @@
 
 class InMemoryZoneData
 {
-    static constexpr size_t BUFFER_SIZE = 0x10000;
+    static constexpr size_t BUFFER_SIZE = 0x400000;
 
 public:
     class MemoryBuffer

src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.cpp

@@ -1,46 +1,123 @@
 #include "OutputProcessorXChunks.h"
 
-class OutputProcessorXChunks::Impl
+#include <cassert>
-{
 
-};
+#include "Writing/WritingException.h"
+#include "Zone/ZoneTypes.h"
+#include "Zone/XChunk/XChunkException.h"
 
-OutputProcessorXChunks::OutputProcessorXChunks(int numStreams, size_t xChunkSize)
+void OutputProcessorXChunks::Init()
 {
+    if (m_vanilla_buffer_size > 0)
+        m_vanilla_buffer_offset = static_cast<size_t>(m_base_stream->Pos()) % m_vanilla_buffer_size;
+
+    m_initialized = true;
 }
 
-OutputProcessorXChunks::OutputProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize)
+void OutputProcessorXChunks::WriteChunk()
 {
+    if (m_vanilla_buffer_size > 0)
+    {
+        if (m_vanilla_buffer_offset + sizeof(xchunk_size_t) > m_vanilla_buffer_size)
+        {
+            xchunk_size_t zeroMem = 0;
+            m_base_stream->Write(&zeroMem, m_vanilla_buffer_size - m_vanilla_buffer_offset);
+            m_vanilla_buffer_offset = 0;
+        }
     }
 
-OutputProcessorXChunks::~OutputProcessorXChunks()
+    try
     {
-    delete m_impl;
+        for (const auto& processor : m_chunk_processors)
-    m_impl = nullptr;
+        {
+            m_input_size = processor->Process(m_current_stream, m_input_buffer, m_input_size, m_output_buffer, m_chunk_size);
+            auto* swap = m_input_buffer;
+            m_input_buffer = m_output_buffer;
+            m_output_buffer = swap;
+        }
+    }
+    catch (XChunkException& e)
+    {
+        throw WritingException(e.Message());
     }
 
-OutputProcessorXChunks::OutputProcessorXChunks(OutputProcessorXChunks&& other) noexcept
+    auto chunkSize = static_cast<xchunk_size_t>(m_input_size);
-    : m_impl(other.m_impl)
+    m_base_stream->Write(&chunkSize, sizeof(chunkSize));
+    m_base_stream->Write(m_input_buffer, m_input_size);
+
+    if (m_vanilla_buffer_size > 0)
     {
-    other.m_impl = nullptr;
+        m_vanilla_buffer_offset += sizeof(chunkSize) + m_input_size;
+        m_vanilla_buffer_offset %= m_vanilla_buffer_size;
     }
 
-OutputProcessorXChunks& OutputProcessorXChunks::operator=(OutputProcessorXChunks&& other) noexcept
+    m_current_stream = (m_current_stream + 1) % m_stream_count;
-{
-    m_impl = other.m_impl;
-    other.m_impl = nullptr;
-    return *this;
 }
 
-void OutputProcessorXChunks::AddChunkProcessor(std::unique_ptr<IXChunkProcessor> chunkProcessor) const
+OutputProcessorXChunks::OutputProcessorXChunks(const int numStreams, const size_t xChunkSize)
+    : m_stream_count(numStreams),
+      m_chunk_size(xChunkSize),
+      m_vanilla_buffer_size(0),
+      m_initialized(false),
+      m_current_stream(0),
+      m_vanilla_buffer_offset(0),
+      m_input_buffer(nullptr),
+      m_output_buffer(nullptr),
+      m_input_size(0)
 {
+    assert(numStreams > 0);
+    assert(xChunkSize > 0);
+
+    for (auto i = 0u; i < 2u; i++)
+        m_buffers.emplace_back(std::make_unique<uint8_t[]>(xChunkSize));
+
+    m_input_buffer = m_buffers[0].get();
+    m_output_buffer = m_buffers[1].get();
 }
 
-void OutputProcessorXChunks::Write(const void* buffer, size_t length)
+OutputProcessorXChunks::OutputProcessorXChunks(const int numStreams, const size_t xChunkSize, const size_t vanillaBufferSize)
+    : OutputProcessorXChunks(numStreams, xChunkSize)
 {
+    m_vanilla_buffer_size = vanillaBufferSize;
+}
+
+void OutputProcessorXChunks::AddChunkProcessor(std::unique_ptr<IXChunkProcessor> chunkProcessor)
+{
+    assert(chunkProcessor != nullptr);
+
+    m_chunk_processors.emplace_back(std::move(chunkProcessor));
+}
+
+void OutputProcessorXChunks::Write(const void* buffer, const size_t length)
+{
+    assert(buffer != nullptr);
+
+    if (!m_initialized)
+        Init();
+
+    auto sizeRemaining = length;
+    while (sizeRemaining > 0)
+    {
+        const auto toWrite = std::min(m_chunk_size - m_input_size, sizeRemaining);
+
+        memcpy(&m_input_buffer[m_input_size], &static_cast<const char*>(buffer)[length - sizeRemaining], toWrite);
+        m_input_size += toWrite;
+        if (m_input_size >= m_chunk_size)
+            WriteChunk();
+
+        sizeRemaining -= toWrite;
+    }
+}
+
+void OutputProcessorXChunks::Flush()
+{
+    if (m_input_size)
+        WriteChunk();
+
+    m_base_stream->Flush();
 }
 
 int64_t OutputProcessorXChunks::Pos()
 {
-    return 0;
+    return m_base_stream->Pos();
 }

src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.h

@@ -1,26 +1,45 @@
 #pragma once
 #include <memory>
+#include <vector>
+#include <cstdint>
+#include <cstddef>
 
 #include "Writing/OutputStreamProcessor.h"
 #include "Zone/XChunk/IXChunkProcessor.h"
 
 class OutputProcessorXChunks final : public OutputStreamProcessor
 {
-    class Impl;
+    std::vector<std::unique_ptr<IXChunkProcessor>> m_chunk_processors;
-    Impl* m_impl;
+
+    int m_stream_count;
+    size_t m_chunk_size;
+    size_t m_vanilla_buffer_size;
+
+    bool m_initialized;
+    int m_current_stream;
+    size_t m_vanilla_buffer_offset;
+
+    std::vector<std::unique_ptr<uint8_t[]>> m_buffers;
+    uint8_t* m_input_buffer;
+    uint8_t* m_output_buffer;
+    size_t m_input_size;
+
+    void Init();
+    void WriteChunk();
 
 public:
     OutputProcessorXChunks(int numStreams, size_t xChunkSize);
     OutputProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize);
-    ~OutputProcessorXChunks() override;
+    ~OutputProcessorXChunks() override = default;
 
     OutputProcessorXChunks(const OutputProcessorXChunks& other) = delete;
-    OutputProcessorXChunks(OutputProcessorXChunks&& other) noexcept;
+    OutputProcessorXChunks(OutputProcessorXChunks&& other) noexcept = default;
     OutputProcessorXChunks& operator=(const OutputProcessorXChunks& other) = delete;
-    OutputProcessorXChunks& operator=(OutputProcessorXChunks&& other) noexcept;
+    OutputProcessorXChunks& operator=(OutputProcessorXChunks&& other) noexcept = default;
 
-    void AddChunkProcessor(std::unique_ptr<IXChunkProcessor> chunkProcessor) const;
+    void AddChunkProcessor(std::unique_ptr<IXChunkProcessor> chunkProcessor);
 
     void Write(const void* buffer, size_t length) override;
+    void Flush() override;
     int64_t Pos() override;
 };

src/ZoneWriting/Writing/Steps/StepWriteXBlockSizes.cpp

@@ -7,4 +7,9 @@ StepWriteXBlockSizes::StepWriteXBlockSizes(Zone* zone)
 
 void StepWriteXBlockSizes::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream)
 {
+    for(const auto& block : zoneWriter->m_blocks)
+    {
+        auto blockSize = static_cast<xblock_size_t>(block->m_buffer_size);
+        stream->Write(&blockSize, sizeof(blockSize));
+    }
 }

src/ZoneWriting/Writing/Steps/StepWriteZoneContentToFile.cpp

@@ -7,4 +7,8 @@ StepWriteZoneContentToFile::StepWriteZoneContentToFile(StepWriteZoneContentToMem
 
 void StepWriteZoneContentToFile::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream)
 {
+    for(const auto& dataBuffer : m_memory->GetData()->m_buffers)
+    {
+        stream->Write(dataBuffer.m_data.get(), dataBuffer.m_size);
+    }
 }

src/ZoneWriting/Writing/WritingFileStream.cpp

@@ -10,6 +10,11 @@ void WritingFileStream::Write(const void* buffer, const size_t length)
     m_stream.write(static_cast<const char*>(buffer), length);
 }
 
+void WritingFileStream::Flush()
+{
+    m_stream.flush();
+}
+
 int64_t WritingFileStream::Pos()
 {
     return m_stream.tellp();

src/ZoneWriting/Writing/WritingFileStream.h

@@ -11,5 +11,6 @@ public:
     explicit WritingFileStream(std::ostream& stream);
 
     void Write(const void* buffer, size_t length) override;
+    void Flush() override;
     int64_t Pos() override;
 };

src/ZoneWriting/Writing/ZoneWriter.cpp

@@ -75,5 +75,7 @@ bool ZoneWriter::WriteZone(std::ostream& stream)
         return false;
     }
 
+    endStream->Flush();
+
     return true;
 }