diff --git a/generate.bat b/generate.bat index 3e68b64..0934437 100644 --- a/generate.bat +++ b/generate.bat @@ -2,4 +2,4 @@ echo Updating submodules git submodule update --init --recursive -tools\premake5.exe vs2019 \ No newline at end of file +tools\premake5.exe vs2022 \ No newline at end of file diff --git a/premake5.lua b/premake5.lua index a4da8b1..050a362 100644 --- a/premake5.lua +++ b/premake5.lua @@ -44,7 +44,7 @@ workspace "zonetool" defines "CPU_64BIT" filter {} - buildoptions "/std:c++latest" + buildoptions "/std:c++17" buildoptions "/Zc:strictStrings-" systemversion "latest" symbols "On" diff --git a/src/IW4/Assets/GfxImage.cpp b/src/IW4/Assets/GfxImage.cpp index 5ab655a..be82f25 100644 --- a/src/IW4/Assets/GfxImage.cpp +++ b/src/IW4/Assets/GfxImage.cpp @@ -8,7 +8,7 @@ // ======================================================== #include "stdafx.hpp" #include "IW5/Assets/GfxImage.hpp" - +#include "Utils/String.hpp" namespace ZoneTool { @@ -249,14 +249,14 @@ namespace ZoneTool alpha_image.cached = false; alpha_image.cardMemory.platform[0] = pixels.size(); - if (this->name().starts_with("*refle")) + if (Utils::StartsWith(this->name(), "*refle")) { alpha_image.format = 0x18280186; alpha_image.mapType = 5; alpha_image.semantic = 1; alpha_image.category = 1; } - else if (this->name().starts_with("*light")) + else if (Utils::StartsWith(this->name(), "*light")) { alpha_image.format = 0x2800017A; alpha_image.mapType = 3; diff --git a/src/IW5/Assets/ClipMap.cpp b/src/IW5/Assets/ClipMap.cpp index 99ee642..be05e35 100644 --- a/src/IW5/Assets/ClipMap.cpp +++ b/src/IW5/Assets/ClipMap.cpp @@ -826,7 +826,7 @@ colmap->dynEntDefList[_num1][_num2]._item = newEntDef[_num2]._item; END_LOG_STREAM; buf->pop_stream(); - encrypt_data(dest, sizeof clipMap_t); + //encrypt_data(dest, sizeof clipMap_t); #ifdef USE_VMPROTECT VMProtectEnd(); diff --git a/src/IW5/Assets/ComWorld.cpp b/src/IW5/Assets/ComWorld.cpp index c66b6c7..c1b8655 100644 --- a/src/IW5/Assets/ComWorld.cpp +++ b/src/IW5/Assets/ComWorld.cpp @@ -142,7 +142,7 @@ namespace ZoneTool buf->pop_stream(); - encrypt_data(dest, sizeof ComWorld); + //encrypt_data(dest, sizeof ComWorld); } void IComWorld::dump(ComWorld* asset, bool fromIW5) diff --git a/src/IW5/Assets/FxWorld.cpp b/src/IW5/Assets/FxWorld.cpp index ca4677a..7ed23f4 100644 --- a/src/IW5/Assets/FxWorld.cpp +++ b/src/IW5/Assets/FxWorld.cpp @@ -230,7 +230,7 @@ namespace ZoneTool END_LOG_STREAM; buf->pop_stream(); - encrypt_data(dest, sizeof FxWorld); + //encrypt_data(dest, sizeof FxWorld); } void IFxWorld::dump(FxWorld* asset) diff --git a/src/IW5/Assets/GfxWorld.cpp b/src/IW5/Assets/GfxWorld.cpp index c1d67c3..2e1a102 100644 --- a/src/IW5/Assets/GfxWorld.cpp +++ b/src/IW5/Assets/GfxWorld.cpp @@ -1055,7 +1055,7 @@ namespace ZoneTool END_LOG_STREAM; buf->pop_stream(); - encrypt_data(dest, sizeof GfxWorld); + //encrypt_data(dest, sizeof GfxWorld); #ifdef USE_VMPROTECT VMProtectEnd(); #endif diff --git a/src/IW5/Assets/MapEnts.cpp b/src/IW5/Assets/MapEnts.cpp index 66d7a71..dd7a676 100644 --- a/src/IW5/Assets/MapEnts.cpp +++ b/src/IW5/Assets/MapEnts.cpp @@ -251,7 +251,7 @@ namespace ZoneTool END_LOG_STREAM; buf->pop_stream(); - encrypt_data(dest, sizeof MapEnts); + //encrypt_data(dest, sizeof MapEnts); } void IMapEnts::dump(MapEnts* asset) diff --git a/src/IW5/Assets/Techset.cpp b/src/IW5/Assets/Techset.cpp index 8e2919c..fd43977 100644 --- a/src/IW5/Assets/Techset.cpp +++ b/src/IW5/Assets/Techset.cpp @@ -271,7 +271,7 @@ namespace ZoneTool END_LOG_STREAM; buf->pop_stream(); - encrypt_data(dest, sizeof MaterialTechniqueSet); + //encrypt_data(dest, sizeof MaterialTechniqueSet); } void ITechset::dump_technique(MaterialTechnique* asset) diff --git a/src/IW5/Assets/WeaponDef.cpp b/src/IW5/Assets/WeaponDef.cpp index 34b7d8f..964e339 100644 --- a/src/IW5/Assets/WeaponDef.cpp +++ b/src/IW5/Assets/WeaponDef.cpp @@ -843,7 +843,12 @@ namespace ZoneTool #define WEAPON_SUBASSET(__field__,__type__,__struct__) \ if (data->__field__) \ { \ + ZONETOOL_INFO("field %s was added", #__field__); \ zone->add_asset_of_type(__type__, data->__field__->name); \ + } \ + else \ + { \ + ZONETOOL_INFO("field %s was null", #__field__); \ } for (auto i = 0u; i < 16; i++) @@ -1585,7 +1590,7 @@ namespace ZoneTool END_LOG_STREAM; buf->pop_stream(); - encrypt_data(dest, sizeof WeaponCompleteDef); + //encrypt_data(dest, sizeof WeaponCompleteDef); } #define WEAPON_DUMP_FIELD(__field__) \ @@ -2176,6 +2181,8 @@ namespace ZoneTool } } + ZONETOOL_INFO("hideTags done dumping for %s", asset->szInternalName); + for (int i = 0; i < 6; i++) { if (asset->scopes && asset->scopes[i]) @@ -2210,6 +2217,8 @@ namespace ZoneTool } } + ZONETOOL_INFO("scopes/others/underBarrels done dumping for %s", asset->szInternalName); + for (int i = 0; i < 42; i++) { if (asset->szXAnims && asset->szXAnims[i]) @@ -2222,6 +2231,8 @@ namespace ZoneTool } } + ZONETOOL_INFO("szXAnims done dumping for %s", asset->szInternalName); + for (int i = 0; i < asset->numAnimOverrides; i++) { data["animOverrides"][i]["altmodeAnim"] = (asset->animOverrides[i].altmodeAnim) @@ -2237,6 +2248,8 @@ namespace ZoneTool data["animOverrides"][i]["animTreeType"] = asset->animOverrides[i].animTreeType; } + ZONETOOL_INFO("animOverrides done dumping for %s", asset->szInternalName); + for (int i = 0; i < asset->numSoundOverrides; i++) { data["soundOverrides"][i]["altmodeSound"] = (asset->soundOverrides[i].altmodeSound) @@ -2250,6 +2263,8 @@ namespace ZoneTool data["soundOverrides"][i]["soundType"] = asset->soundOverrides[i].soundType; } + ZONETOOL_INFO("soundOverrides done dumping for %s", asset->szInternalName); + for (int i = 0; i < asset->numFXOverrides; i++) { data["fxOverrides"][i]["altmodeFX"] = (asset->fxOverrides[i].altmodeFX) @@ -2263,6 +2278,8 @@ namespace ZoneTool : ""; } + ZONETOOL_INFO("fxOverrides done dumping for %s", asset->szInternalName); + for (int i = 0; i < asset->numReloadStateTimerOverrides; i++) { data["reloadOverrides"][i]["attachment"] = asset->reloadOverrides[i].attachment; @@ -2270,6 +2287,8 @@ namespace ZoneTool data["reloadOverrides"][i]["reloadEmptyAddTime"] = asset->reloadOverrides[i].reloadEmptyAddTime; } + ZONETOOL_INFO("reloadOverrides done dumping for %s", asset->szInternalName); + for (int i = 0; i < asset->numNotetrackOverrides; i++) { data["notetrackOverrides"][i]["attachment"] = asset->notetrackOverrides[i].attachment; @@ -2283,6 +2302,8 @@ namespace ZoneTool } } + ZONETOOL_INFO("notetrackOverrides done dumping for %s", asset->szInternalName); + WEAPON_DUMP_FIELD(szInternalName); WEAPON_DUMP_FIELD(szDisplayName); WEAPON_DUMP_FIELD(numAnimOverrides); @@ -2325,6 +2346,8 @@ namespace ZoneTool void IWeaponDef::dump(WeaponCompleteDef* asset, const std::function& convertToString) { + ZONETOOL_INFO("Dumping WeaponCompleteDef %s", asset->szInternalName); + std::string path = "weapons/mp/"s + asset->szInternalName; std::string json = dump_complete(asset, convertToString).dump(4); diff --git a/src/IW5/Zone.cpp b/src/IW5/Zone.cpp index 451090b..d9afc7b 100644 --- a/src/IW5/Zone.cpp +++ b/src/IW5/Zone.cpp @@ -16,7 +16,7 @@ namespace ZoneTool { if (name.empty()) { - return nullptr; + return nullptr; } for (std::size_t idx = 0; idx < m_assets.size(); idx++) @@ -245,12 +245,12 @@ namespace ZoneTool buf->save("debug\\" + this->name_ + ".zone"); // Compress buffer - auto buf_compressed = buf->compress_zstd(); + auto buf_compressed = buf->compress_zlib(); // Generate FF header auto header = this->m_zonemem->Alloc(); strcpy(header->header, "IWffu100"); - header->version = 2000; + header->version = 1; header->allowOnlineUpdate = 0; // Save fastfile @@ -260,15 +260,16 @@ namespace ZoneTool fastfile.write(buf_compressed.data(), buf_compressed.size()); - std::string localappdata = getenv("LOCALAPPDATA"); - fastfile.save(localappdata + "\\Plutonium-staging\\storage\\iw5\\zone\\" + this->name_ + ".ff"); + //std::string localappdata = getenv("LOCALAPPDATA"); + //fastfile.save(localappdata + "\\Plutonium-staging\\storage\\iw5\\zone\\" + this->name_ + ".ff"); // oxygen output paths // fastfile.save("C:\\Users\\RektInator\\AppData\\Local\\Plutonium\\storage\\iw5\\zone\\" + this->name_ + ".ff"); - fastfile.save("zone\\english\\" + this->name_ + ".ff"); + CreateDirectoryA("zone_out", nullptr); + fastfile.save("zone_out\\" + this->name_ + ".ff"); ZONETOOL_INFO("Successfully compiled fastfile \"%s\"!", this->name_.data()); - ZONETOOL_INFO("Compiling took %u msec.", GetTickCount64() - startTime); + ZONETOOL_INFO("Compiling took %llu msec.", GetTickCount64() - startTime); // this->m_linker->UnloadZones(); diff --git a/src/ZoneUtils/Utils/String.cpp b/src/ZoneUtils/Utils/String.cpp new file mode 100644 index 0000000..7204118 --- /dev/null +++ b/src/ZoneUtils/Utils/String.cpp @@ -0,0 +1,250 @@ +#include "stdafx.hpp" +#include "String.hpp" + +#include +#include + +#ifdef ENABLE_BASE128 +#include "base128.h" +#endif + +namespace ZoneTool::Utils +{ + const char* VA(const char* fmt, ...) + { + static thread_local VAProvider<8, 256> provider; + + va_list ap; + va_start(ap, fmt); + + const char* result; + result = provider.get(fmt, ap); + + va_end(ap); + return result; + } + + std::string ToLower(const std::string& text) + { + std::string result; + std::transform(text.begin(), text.end(), std::back_inserter(result), [](const unsigned char input) -> char + { + return static_cast(std::tolower(input)); + }); + + return result; + } + + std::string ToUpper(const std::string& text) + { + std::string result; + std::transform(text.begin(), text.end(), std::back_inserter(result), [](const unsigned char input) -> char + { + return static_cast(std::toupper(input)); + }); + + return result; + } + + bool Compare(const std::string& lhs, const std::string& rhs) + { + return false; + } + + std::string DumpHex(const std::string& data, const std::string& separator) + { + std::string result; + + for (std::size_t i = 0; i < data.size(); ++i) + { + if (i > 0) + { + result.append(separator); + } + + result.append(VA("%02X", data[i] & 0xFF)); + } + + return result; + } + + std::string XOR(std::string str, char value) + { + for (std::size_t i = 0; i < str.size(); ++i) + { + str[i] ^= static_cast(value); + } + + return str; + } + + std::vector Split(const std::string& str, const char delim) + { + std::stringstream ss(str); + std::string item; + std::vector elems; + + while (std::getline(ss, item, delim)) + { + elems.push_back(item); // elems.push_back(std::move(item)); // if C++11 (based on comment from S1x) + } + + return elems; + } + + void Replace(std::string& str, const std::string& from, const std::string& to) + { + std::size_t nPos = 0; + + while ((nPos = str.find(from, nPos)) != std::string::npos) + { + str = str.replace(nPos, from.length(), to); + nPos += to.length(); + } + } + + bool StartsWith(const std::string& haystack, const std::string& needle) + { + return haystack.find(needle) == 0; // If the pos of the first found char is 0, string starts with 'needle' + } + + bool EndsWith(const std::string& haystack, const std::string& needle) + { + if (needle.size() > haystack.size()) return false; + return std::equal(needle.rbegin(), needle.rend(), haystack.rbegin()); + } + + bool IsNumber(const std::string& str) + { + return !str.empty() && std::find_if(str.begin(), str.end(), [](unsigned char input) + { + return !std::isdigit(input); + }) == str.end(); + } + + // Trim from start + std::string& LTrim(std::string& str) + { + str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](const unsigned char input) + { + return !std::isspace(input); + })); + + return str; + } + + // Trim from end + std::string& RTrim(std::string& str) + { + str.erase(std::find_if(str.rbegin(), str.rend(), [](const unsigned char input) + { + return !std::isspace(input); + }).base(), str.end()); + + return str; + } + + // Trim from both ends + void Trim(std::string& str) + { + LTrim(RTrim(str)); + } + + std::string Convert(const std::wstring& wstr) + { + std::string result; + result.reserve(wstr.size()); + + for (const auto& chr : wstr) + { + result.push_back(static_cast(chr)); + } + + return result; + } + + std::wstring Convert(const std::string& str) + { + std::wstring result; + result.reserve(str.size()); + + for (const auto& chr : str) + { + result.push_back(static_cast(chr)); + } + + return result; + } + + std::string FormatTimeSpan(int milliseconds) + { + int secondsTotal = milliseconds / 1000; + int seconds = secondsTotal % 60; + int minutesTotal = secondsTotal / 60; + int minutes = minutesTotal % 60; + int hoursTotal = minutesTotal / 60; + + return VA("%02d:%02d:%02d", hoursTotal, minutes, seconds); + } + + std::string FormatBandwidth(std::size_t bytes, int milliseconds) + { + static const char* sizes[] = + { + "B", + "KB", + "MB", + "GB", + "TB", + }; + + if (!milliseconds) return "0.00 B/s"; + + double bytesPerSecond = (1000.0 / milliseconds) * bytes; + + std::size_t i; + for (i = 0; bytesPerSecond > 1000 && i < ARRAYSIZE(sizes); ++i) // 1024 or 1000? + { + bytesPerSecond /= 1000; + } + + return VA("%.2f %s/s", static_cast(bytesPerSecond), sizes[i]); + } + +#ifdef ENABLE_BASE64 + // Encodes a given string in Base64 + std::string EncodeBase64(const char* input, const unsigned long inputSize) + { + unsigned long outlen = long(inputSize + (inputSize / 3.0) + 16); + unsigned char* outbuf = new unsigned char[outlen]; //Reserve output memory + base64_encode(reinterpret_cast(const_cast(input)), inputSize, outbuf, &outlen); + std::string ret(reinterpret_cast(outbuf), outlen); + delete[] outbuf; + return ret; + } + + // Encodes a given string in Base64 + std::string EncodeBase64(const std::string& input) + { + return EncodeBase64(input.data(), input.size()); + } +#endif + +#ifdef ENABLE_BASE128 + // Encodes a given string in Base128 + std::string EncodeBase128(const std::string& input) + { + base128 encoder; + + void* inbuffer = const_cast(input.data()); + char* buffer = encoder.encode(inbuffer, input.size()); + /* + Interesting to see that the buffer returned by the encoder is not a standalone string copy + but instead is a pointer to the internal "encoded" field of the encoder. So if you deinitialize + the encoder that string will probably become garbage. + */ + std::string retval(buffer); + return retval; + } +#endif +} diff --git a/src/ZoneUtils/Utils/String.hpp b/src/ZoneUtils/Utils/String.hpp new file mode 100644 index 0000000..0868c0e --- /dev/null +++ b/src/ZoneUtils/Utils/String.hpp @@ -0,0 +1,153 @@ +#pragma once + +template +constexpr auto ARRAY_COUNT(Type(&)[n]) { return n; } + +namespace ZoneTool::Utils +{ + template + class VAProvider + { + public: + static_assert(Buffers != 0 && MinBufferSize != 0, "Buffers and MinBufferSize mustn't be 0"); + + VAProvider() : currentBuffer_(0) {} + + [[nodiscard]] const char* get(const char* format, va_list ap) + { + ++this->currentBuffer_ %= ARRAY_COUNT(this->stringPool_); + auto entry = &this->stringPool_[this->currentBuffer_]; + + if (!entry->size_ || !entry->buffer_) + { + throw std::runtime_error("String pool not initialized"); + } + + while (true) + { + const auto res = vsnprintf_s(entry->buffer_, entry->size_, _TRUNCATE, format, ap); + if (res > 0) break; // Success + if (res == 0) return ""; // Error + + entry->doubleSize(); + } + + return entry->buffer_; + } + + private: + class Entry + { + public: + Entry(std::size_t size = MinBufferSize) : size_(size), buffer_(nullptr) + { + if (this->size_ < MinBufferSize) this->size_ = MinBufferSize; + this->allocate(); + } + + ~Entry() + { + if (this->buffer_) std::free(this->buffer_); + this->size_ = 0; + this->buffer_ = nullptr; + } + + void allocate() + { + if (this->buffer_) std::free(this->buffer_); + this->buffer_ = (char*)std::calloc(this->size_ + 1, 1); + } + + void doubleSize() + { + this->size_ *= 2; + this->allocate(); + } + + std::size_t size_; + char* buffer_; + }; + + std::size_t currentBuffer_; + Entry stringPool_[Buffers]; + }; + + template // This should display a nice "nullptr" instead of a number + static void SanitizeFormatArgs(Arg& arg) + { + if constexpr (std::is_same_v || std::is_same_v) + { + if (arg == nullptr) + { + arg = const_cast("nullptr"); + } + } + } + + [[nodiscard]] const char* VA(const char* fmt, ...); + + template + [[nodiscard]] const char* Format(std::string_view fmt, Args&&... args) + { + static thread_local std::string vaBuffer; + (SanitizeFormatArgs(args), ...); + std::vformat(fmt, std::make_format_args(args...)).swap(vaBuffer); + return vaBuffer.data(); + } + + [[nodiscard]] std::string ToLower(const std::string& text); + [[nodiscard]] std::string ToUpper(const std::string& text); + + template + [[nodiscard]] OutputIter ApplyToLower(OutputIter container) + { + OutputIter result; + std::ranges::transform(container, std::back_inserter(result), [](const std::string& s) -> std::string + { + return ToLower(s); + }); + + return result; + } + + template + [[nodiscard]] OutputIter ApplyToUpper(OutputIter container) + { + OutputIter result; + std::ranges::transform(container, std::back_inserter(result), [](const std::string& s) -> std::string + { + return ToUpper(s); + }); + + return result; + } + + [[nodiscard]] bool Compare(const std::string& lhs, const std::string& rhs); + + [[nodiscard]] std::vector Split(const std::string& str, char delim); + void Replace(std::string& str, const std::string& from, const std::string& to); + + [[nodiscard]] bool StartsWith(const std::string& haystack, const std::string& needle); + [[nodiscard]] bool EndsWith(const std::string& haystack, const std::string& needle); + + [[nodiscard]] bool IsNumber(const std::string& str); + + std::string& LTrim(std::string& str); + std::string& RTrim(std::string& str); + void Trim(std::string& str); + + [[nodiscard]] std::string Convert(const std::wstring& wstr); + [[nodiscard]] std::wstring Convert(const std::string& str); + + [[nodiscard]] std::string FormatTimeSpan(int milliseconds); + [[nodiscard]] std::string FormatBandwidth(std::size_t bytes, int milliseconds); + + [[nodiscard]] std::string DumpHex(const std::string& data, const std::string& separator = " "); + + [[nodiscard]] std::string XOR(std::string str, char value); + + [[nodiscard]] std::string EncodeBase64(const char* input, unsigned long inputSize); + [[nodiscard]] std::string EncodeBase64(const std::string& input); + + [[nodiscard]] std::string EncodeBase128(const std::string& input); +} diff --git a/src/ZoneUtils/Zone/ZoneBuffer.cpp b/src/ZoneUtils/Zone/ZoneBuffer.cpp index 65eb07a..59f165e 100644 --- a/src/ZoneUtils/Zone/ZoneBuffer.cpp +++ b/src/ZoneUtils/Zone/ZoneBuffer.cpp @@ -339,29 +339,6 @@ namespace ZoneTool return ZoneBuffer::compress_zlib(data.data(), data.size(), compress_blocks); } - std::vector ZoneBuffer::compress_zstd() - { - // calculate buffer size needed for current zone - auto size = ZSTD_compressBound(this->m_pos); - - // alloc array for compressed data - std::vector compressed; - compressed.resize(size); - - // compress buffer - auto destsize = ZSTD_compress(compressed.data(), size, this->m_buf.data(), this->m_pos, 11); - compressed.resize(destsize); - - if (ZSTD_isError(destsize)) - { - ZONETOOL_ERROR("An error occured while compressing the fastfile: %s", ZSTD_getErrorName(destsize)); - return {}; - } - - // return compressed buffer - return compressed; - } - std::vector ZoneBuffer::compress_zlib(bool compress_blocks) { return ZoneBuffer::compress_zlib(this->m_buf.data(), this->m_pos, compress_blocks); diff --git a/src/ZoneUtils/Zone/ZoneBuffer.hpp b/src/ZoneUtils/Zone/ZoneBuffer.hpp index abfdb1a..3b3a95c 100644 --- a/src/ZoneUtils/Zone/ZoneBuffer.hpp +++ b/src/ZoneUtils/Zone/ZoneBuffer.hpp @@ -349,7 +349,6 @@ namespace ZoneTool static std::vector compress_zlib(const std::uint8_t* data, const std::size_t size, bool compress_blocks = false); static std::vector compress_zlib(const std::vector& data, bool compress_blocks = false); - std::vector compress_zstd(); std::vector compress_zlib(bool compress_blocks = false); void encrypt(); }; diff --git a/src/ZoneUtils/ZoneUtils.hpp b/src/ZoneUtils/ZoneUtils.hpp index 9e2ed0c..1261c68 100644 --- a/src/ZoneUtils/ZoneUtils.hpp +++ b/src/ZoneUtils/ZoneUtils.hpp @@ -148,6 +148,7 @@ namespace ZoneTool #include "Utils/FileSystem.hpp" #include "Utils/Function.hpp" #include "Utils/Memory.hpp" +#include "Utils/String.hpp" #include "Utils/BinaryDumper.hpp" #include "Utils/Expressions.hpp" #include "Linker.hpp"