diff --git a/src/Linker/Linker.cpp b/src/Linker/Linker.cpp index 6ea8434e..f0477427 100644 --- a/src/Linker/Linker.cpp +++ b/src/Linker/Linker.cpp @@ -443,7 +443,7 @@ class Linker::Impl fs::create_directories(zoneFolderPath); - std::ofstream stream(zoneFilePath); + std::ofstream stream(zoneFilePath, std::fstream::out | std::fstream::binary); if (!stream.is_open()) return false; diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp index 5f1cf5ca..0556ff10 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorDeflate.cpp @@ -1,5 +1,7 @@ #include "XChunkProcessorDeflate.h" + +#include #include #include @@ -11,7 +13,7 @@ size_t XChunkProcessorDeflate::Process(int streamNumber, const uint8_t* input, c stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; - + auto ret = deflateInit2(&stream, Z_BEST_COMPRESSION, Z_DEFLATED, -DEF_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); if (ret != Z_OK) throw XChunkException("Initializing deflate failed."); @@ -23,7 +25,7 @@ size_t XChunkProcessorDeflate::Process(int streamNumber, const uint8_t* input, c ret = deflate(&stream, Z_FINISH); if (ret != Z_STREAM_END) - throw XChunkException("Zone has invalid or unsupported compression. Deflate failed"); + throw XChunkException("Failed to deflate memory of zone."); const size_t outputSize = stream.total_out; diff --git a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp index b4b6887f..e4499068 100644 --- a/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp +++ b/src/ZoneCommon/Zone/XChunk/XChunkProcessorSalsa20Decryption.cpp @@ -5,7 +5,7 @@ #include "Crypto.h" #include "AbstractSalsa20Processor.h" -XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const int streamCount, std::string& zoneName, const uint8_t* salsa20Key, size_t keySize) +XChunkProcessorSalsa20Decryption::XChunkProcessorSalsa20Decryption(const int streamCount, std::string& zoneName, const uint8_t* salsa20Key, const size_t keySize) : AbstractSalsa20Processor(streamCount, zoneName, salsa20Key, keySize) { } diff --git a/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp b/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp index 95308590..95eaee99 100644 --- a/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp +++ b/src/ZoneWriting/Game/T6/ZoneWriterFactoryT6.cpp @@ -15,6 +15,7 @@ #include "Writing/Steps/StepWriteZoneContentToFile.h" #include "Writing/Steps/StepWriteZoneContentToMemory.h" #include "Writing/Steps/StepWriteZoneHeader.h" +#include "Writing/Steps/StepWriteZoneSizes.h" using namespace T6; @@ -74,6 +75,9 @@ public: ICapturedDataProvider* result = nullptr; auto xChunkProcessor = std::make_unique(ZoneConstants::STREAM_COUNT, ZoneConstants::XCHUNK_SIZE, ZoneConstants::VANILLA_BUFFER_SIZE); + // Decompress the chunks using zlib + xChunkProcessor->AddChunkProcessor(std::make_unique()); + if (isEncrypted) { // If zone is encrypted, the decryption is applied before the decompression. T6 Zones always use Salsa20. @@ -83,8 +87,6 @@ public: xChunkProcessor->AddChunkProcessor(std::move(chunkProcessorSalsa20)); } - // Decompress the chunks using zlib - xChunkProcessor->AddChunkProcessor(std::make_unique()); m_writer->AddWritingStep(std::make_unique(std::move(xChunkProcessor))); // If there is encryption, the signed data of the zone is the final hash blocks provided by the Salsa20 IV adaption algorithm @@ -111,6 +113,7 @@ public: // Start of the XFile struct //m_writer->AddWritingStep(std::make_unique(8)); // Skip size and externalSize fields since they are not interesting for us + m_writer->AddWritingStep(std::make_unique(contentInMemoryPtr)); m_writer->AddWritingStep(std::make_unique(m_zone)); // Start of the zone content diff --git a/src/ZoneWriting/Writing/IWritingStream.h b/src/ZoneWriting/Writing/IWritingStream.h index 5332e586..6324e133 100644 --- a/src/ZoneWriting/Writing/IWritingStream.h +++ b/src/ZoneWriting/Writing/IWritingStream.h @@ -14,5 +14,6 @@ public: IWritingStream& operator=(IWritingStream&& other) noexcept = default; virtual void Write(const void* buffer, size_t length) = 0; + virtual void Flush() = 0; virtual int64_t Pos() = 0; }; \ No newline at end of file diff --git a/src/ZoneWriting/Writing/InMemoryZoneData.h b/src/ZoneWriting/Writing/InMemoryZoneData.h index 52ec3453..0d43bdb8 100644 --- a/src/ZoneWriting/Writing/InMemoryZoneData.h +++ b/src/ZoneWriting/Writing/InMemoryZoneData.h @@ -5,7 +5,7 @@ class InMemoryZoneData { - static constexpr size_t BUFFER_SIZE = 0x10000; + static constexpr size_t BUFFER_SIZE = 0x400000; public: class MemoryBuffer diff --git a/src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.cpp b/src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.cpp index a3a45a2d..b4a008db 100644 --- a/src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.cpp +++ b/src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.cpp @@ -1,46 +1,123 @@ #include "OutputProcessorXChunks.h" -class OutputProcessorXChunks::Impl -{ - -}; +#include -OutputProcessorXChunks::OutputProcessorXChunks(int numStreams, size_t xChunkSize) +#include "Writing/WritingException.h" +#include "Zone/ZoneTypes.h" +#include "Zone/XChunk/XChunkException.h" + +void OutputProcessorXChunks::Init() { + if (m_vanilla_buffer_size > 0) + m_vanilla_buffer_offset = static_cast(m_base_stream->Pos()) % m_vanilla_buffer_size; + + m_initialized = true; } -OutputProcessorXChunks::OutputProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize) +void OutputProcessorXChunks::WriteChunk() { + if (m_vanilla_buffer_size > 0) + { + if (m_vanilla_buffer_offset + sizeof(xchunk_size_t) > m_vanilla_buffer_size) + { + xchunk_size_t zeroMem = 0; + m_base_stream->Write(&zeroMem, m_vanilla_buffer_size - m_vanilla_buffer_offset); + m_vanilla_buffer_offset = 0; + } + } + + try + { + for (const auto& processor : m_chunk_processors) + { + m_input_size = processor->Process(m_current_stream, m_input_buffer, m_input_size, m_output_buffer, m_chunk_size); + auto* swap = m_input_buffer; + m_input_buffer = m_output_buffer; + m_output_buffer = swap; + } + } + catch (XChunkException& e) + { + throw WritingException(e.Message()); + } + + auto chunkSize = static_cast(m_input_size); + m_base_stream->Write(&chunkSize, sizeof(chunkSize)); + m_base_stream->Write(m_input_buffer, m_input_size); + + if (m_vanilla_buffer_size > 0) + { + m_vanilla_buffer_offset += sizeof(chunkSize) + m_input_size; + m_vanilla_buffer_offset %= m_vanilla_buffer_size; + } + + m_current_stream = (m_current_stream + 1) % m_stream_count; } -OutputProcessorXChunks::~OutputProcessorXChunks() +OutputProcessorXChunks::OutputProcessorXChunks(const int numStreams, const size_t xChunkSize) + : m_stream_count(numStreams), + m_chunk_size(xChunkSize), + m_vanilla_buffer_size(0), + m_initialized(false), + m_current_stream(0), + m_vanilla_buffer_offset(0), + m_input_buffer(nullptr), + m_output_buffer(nullptr), + m_input_size(0) { - delete m_impl; - m_impl = nullptr; + assert(numStreams > 0); + assert(xChunkSize > 0); + + for (auto i = 0u; i < 2u; i++) + m_buffers.emplace_back(std::make_unique(xChunkSize)); + + m_input_buffer = m_buffers[0].get(); + m_output_buffer = m_buffers[1].get(); } -OutputProcessorXChunks::OutputProcessorXChunks(OutputProcessorXChunks&& other) noexcept - : m_impl(other.m_impl) +OutputProcessorXChunks::OutputProcessorXChunks(const int numStreams, const size_t xChunkSize, const size_t vanillaBufferSize) + : OutputProcessorXChunks(numStreams, xChunkSize) { - other.m_impl = nullptr; + m_vanilla_buffer_size = vanillaBufferSize; } -OutputProcessorXChunks& OutputProcessorXChunks::operator=(OutputProcessorXChunks&& other) noexcept +void OutputProcessorXChunks::AddChunkProcessor(std::unique_ptr chunkProcessor) { - m_impl = other.m_impl; - other.m_impl = nullptr; - return *this; + assert(chunkProcessor != nullptr); + + m_chunk_processors.emplace_back(std::move(chunkProcessor)); } -void OutputProcessorXChunks::AddChunkProcessor(std::unique_ptr chunkProcessor) const +void OutputProcessorXChunks::Write(const void* buffer, const size_t length) { + assert(buffer != nullptr); + + if (!m_initialized) + Init(); + + auto sizeRemaining = length; + while (sizeRemaining > 0) + { + const auto toWrite = std::min(m_chunk_size - m_input_size, sizeRemaining); + + memcpy(&m_input_buffer[m_input_size], &static_cast(buffer)[length - sizeRemaining], toWrite); + m_input_size += toWrite; + if (m_input_size >= m_chunk_size) + WriteChunk(); + + sizeRemaining -= toWrite; + } } -void OutputProcessorXChunks::Write(const void* buffer, size_t length) +void OutputProcessorXChunks::Flush() { + if (m_input_size) + WriteChunk(); + + m_base_stream->Flush(); } int64_t OutputProcessorXChunks::Pos() { - return 0; + return m_base_stream->Pos(); } diff --git a/src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.h b/src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.h index c9f16d25..827fecb6 100644 --- a/src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.h +++ b/src/ZoneWriting/Writing/Processor/OutputProcessorXChunks.h @@ -1,26 +1,45 @@ #pragma once #include +#include +#include +#include #include "Writing/OutputStreamProcessor.h" #include "Zone/XChunk/IXChunkProcessor.h" class OutputProcessorXChunks final : public OutputStreamProcessor { - class Impl; - Impl* m_impl; + std::vector> m_chunk_processors; + + int m_stream_count; + size_t m_chunk_size; + size_t m_vanilla_buffer_size; + + bool m_initialized; + int m_current_stream; + size_t m_vanilla_buffer_offset; + + std::vector> m_buffers; + uint8_t* m_input_buffer; + uint8_t* m_output_buffer; + size_t m_input_size; + + void Init(); + void WriteChunk(); public: OutputProcessorXChunks(int numStreams, size_t xChunkSize); OutputProcessorXChunks(int numStreams, size_t xChunkSize, size_t vanillaBufferSize); - ~OutputProcessorXChunks() override; + ~OutputProcessorXChunks() override = default; OutputProcessorXChunks(const OutputProcessorXChunks& other) = delete; - OutputProcessorXChunks(OutputProcessorXChunks&& other) noexcept; + OutputProcessorXChunks(OutputProcessorXChunks&& other) noexcept = default; OutputProcessorXChunks& operator=(const OutputProcessorXChunks& other) = delete; - OutputProcessorXChunks& operator=(OutputProcessorXChunks&& other) noexcept; + OutputProcessorXChunks& operator=(OutputProcessorXChunks&& other) noexcept = default; - void AddChunkProcessor(std::unique_ptr chunkProcessor) const; + void AddChunkProcessor(std::unique_ptr chunkProcessor); void Write(const void* buffer, size_t length) override; + void Flush() override; int64_t Pos() override; }; diff --git a/src/ZoneWriting/Writing/Steps/StepWriteXBlockSizes.cpp b/src/ZoneWriting/Writing/Steps/StepWriteXBlockSizes.cpp index eae6a854..baa59ff5 100644 --- a/src/ZoneWriting/Writing/Steps/StepWriteXBlockSizes.cpp +++ b/src/ZoneWriting/Writing/Steps/StepWriteXBlockSizes.cpp @@ -7,4 +7,9 @@ StepWriteXBlockSizes::StepWriteXBlockSizes(Zone* zone) void StepWriteXBlockSizes::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream) { + for(const auto& block : zoneWriter->m_blocks) + { + auto blockSize = static_cast(block->m_buffer_size); + stream->Write(&blockSize, sizeof(blockSize)); + } } diff --git a/src/ZoneWriting/Writing/Steps/StepWriteZoneContentToFile.cpp b/src/ZoneWriting/Writing/Steps/StepWriteZoneContentToFile.cpp index 8ba6d1bc..8348c331 100644 --- a/src/ZoneWriting/Writing/Steps/StepWriteZoneContentToFile.cpp +++ b/src/ZoneWriting/Writing/Steps/StepWriteZoneContentToFile.cpp @@ -7,4 +7,8 @@ StepWriteZoneContentToFile::StepWriteZoneContentToFile(StepWriteZoneContentToMem void StepWriteZoneContentToFile::PerformStep(ZoneWriter* zoneWriter, IWritingStream* stream) { + for(const auto& dataBuffer : m_memory->GetData()->m_buffers) + { + stream->Write(dataBuffer.m_data.get(), dataBuffer.m_size); + } } diff --git a/src/ZoneWriting/Writing/WritingFileStream.cpp b/src/ZoneWriting/Writing/WritingFileStream.cpp index 86e891ce..1f6d8b23 100644 --- a/src/ZoneWriting/Writing/WritingFileStream.cpp +++ b/src/ZoneWriting/Writing/WritingFileStream.cpp @@ -10,6 +10,11 @@ void WritingFileStream::Write(const void* buffer, const size_t length) m_stream.write(static_cast(buffer), length); } +void WritingFileStream::Flush() +{ + m_stream.flush(); +} + int64_t WritingFileStream::Pos() { return m_stream.tellp(); diff --git a/src/ZoneWriting/Writing/WritingFileStream.h b/src/ZoneWriting/Writing/WritingFileStream.h index 2960dc41..117d711f 100644 --- a/src/ZoneWriting/Writing/WritingFileStream.h +++ b/src/ZoneWriting/Writing/WritingFileStream.h @@ -11,5 +11,6 @@ public: explicit WritingFileStream(std::ostream& stream); void Write(const void* buffer, size_t length) override; + void Flush() override; int64_t Pos() override; }; diff --git a/src/ZoneWriting/Writing/ZoneWriter.cpp b/src/ZoneWriting/Writing/ZoneWriter.cpp index 9ac765ed..70e262ae 100644 --- a/src/ZoneWriting/Writing/ZoneWriter.cpp +++ b/src/ZoneWriting/Writing/ZoneWriter.cpp @@ -75,5 +75,7 @@ bool ZoneWriter::WriteZone(std::ostream& stream) return false; } + endStream->Flush(); + return true; }