fix some shit

generate.bat

@@ -2,4 +2,4 @@
 echo Updating submodules
 git submodule update --init --recursive
 
-tools\premake5.exe vs2019
+tools\premake5.exe vs2022

premake5.lua

@@ -44,7 +44,7 @@ workspace "zonetool"
 		defines "CPU_64BIT"
 	filter {}
 
-	buildoptions "/std:c++latest"
+	buildoptions "/std:c++17"
 	buildoptions "/Zc:strictStrings-"
 	systemversion "latest"
 	symbols "On"

src/IW4/Assets/GfxImage.cpp

@@ -8,7 +8,7 @@
 // ========================================================
 #include "stdafx.hpp"
 #include "IW5/Assets/GfxImage.hpp"
-
+#include "Utils/String.hpp"
 
 namespace ZoneTool
 {
@@ -249,14 +249,14 @@ namespace ZoneTool
 						
 						alpha_image.cached = false;
 						alpha_image.cardMemory.platform[0] = pixels.size();
-						if (this->name().starts_with("*refle"))
+						if (Utils::StartsWith(this->name(), "*refle"))
 						{
 							alpha_image.format = 0x18280186;
 							alpha_image.mapType = 5;
 							alpha_image.semantic = 1;
 							alpha_image.category = 1;
 						}
-						else if (this->name().starts_with("*light"))
+						else if (Utils::StartsWith(this->name(), "*light"))
 						{
 							alpha_image.format = 0x2800017A;
 							alpha_image.mapType = 3;

src/IW5/Assets/ClipMap.cpp

@@ -826,7 +826,7 @@ colmap->dynEntDefList[_num1][_num2]._item = newEntDef[_num2]._item;
 			END_LOG_STREAM;
 			buf->pop_stream();
 
-			encrypt_data(dest, sizeof clipMap_t);
+			//encrypt_data(dest, sizeof clipMap_t);
 
 #ifdef USE_VMPROTECT
 			VMProtectEnd();

src/IW5/Assets/ComWorld.cpp

@@ -142,7 +142,7 @@ namespace ZoneTool
 
 			buf->pop_stream();
 
-			encrypt_data(dest, sizeof ComWorld);
+			//encrypt_data(dest, sizeof ComWorld);
 		}
 
 		void IComWorld::dump(ComWorld* asset, bool fromIW5)

src/IW5/Assets/FxWorld.cpp

@@ -230,7 +230,7 @@ namespace ZoneTool
 			END_LOG_STREAM;
 			buf->pop_stream();
 
-			encrypt_data(dest, sizeof FxWorld);
+			//encrypt_data(dest, sizeof FxWorld);
 		}
 
 		void IFxWorld::dump(FxWorld* asset)

src/IW5/Assets/GfxWorld.cpp

@@ -1055,7 +1055,7 @@ namespace ZoneTool
 			END_LOG_STREAM;
 			buf->pop_stream();
 
-			encrypt_data(dest, sizeof GfxWorld);
+			//encrypt_data(dest, sizeof GfxWorld);
 #ifdef USE_VMPROTECT
 			VMProtectEnd();
 #endif

src/IW5/Assets/MapEnts.cpp

@@ -251,7 +251,7 @@ namespace ZoneTool
 			END_LOG_STREAM;
 			buf->pop_stream();
 
-			encrypt_data(dest, sizeof MapEnts);
+			//encrypt_data(dest, sizeof MapEnts);
 		}
 
 		void IMapEnts::dump(MapEnts* asset)

src/IW5/Assets/Techset.cpp

@@ -271,7 +271,7 @@ namespace ZoneTool
 			END_LOG_STREAM;
 			buf->pop_stream();
 
-			encrypt_data(dest, sizeof MaterialTechniqueSet);
+			//encrypt_data(dest, sizeof MaterialTechniqueSet);
 		}
 
 		void ITechset::dump_technique(MaterialTechnique* asset)

src/IW5/Assets/WeaponDef.cpp

@@ -843,7 +843,12 @@ namespace ZoneTool
 #define WEAPON_SUBASSET(__field__,__type__,__struct__) \
 			if (data->__field__) \
 			{ \
+				ZONETOOL_INFO("field %s was added", #__field__); \
 				zone->add_asset_of_type(__type__, data->__field__->name); \
+			} \
+			else \
+			{ \
+				ZONETOOL_INFO("field %s was null", #__field__); \
 			}
 
 			for (auto i = 0u; i < 16; i++)
@@ -1585,7 +1590,7 @@ namespace ZoneTool
 			END_LOG_STREAM;
 			buf->pop_stream();
 
-			encrypt_data(dest, sizeof WeaponCompleteDef);
+			//encrypt_data(dest, sizeof WeaponCompleteDef);
 		}
 
 #define WEAPON_DUMP_FIELD(__field__) \
@@ -2176,6 +2181,8 @@ namespace ZoneTool
 				}
 			}
 
+			ZONETOOL_INFO("hideTags done dumping for %s", asset->szInternalName);
+
 			for (int i = 0; i < 6; i++)
 			{
 				if (asset->scopes && asset->scopes[i])
@@ -2210,6 +2217,8 @@ namespace ZoneTool
 				}
 			}
 
+			ZONETOOL_INFO("scopes/others/underBarrels done dumping for %s", asset->szInternalName);
+
 			for (int i = 0; i < 42; i++)
 			{
 				if (asset->szXAnims && asset->szXAnims[i])
@@ -2222,6 +2231,8 @@ namespace ZoneTool
 				}
 			}
 
+			ZONETOOL_INFO("szXAnims done dumping for %s", asset->szInternalName);
+
 			for (int i = 0; i < asset->numAnimOverrides; i++)
 			{
 				data["animOverrides"][i]["altmodeAnim"] = (asset->animOverrides[i].altmodeAnim)
@@ -2237,6 +2248,8 @@ namespace ZoneTool
 				data["animOverrides"][i]["animTreeType"] = asset->animOverrides[i].animTreeType;
 			}
 
+			ZONETOOL_INFO("animOverrides done dumping for %s", asset->szInternalName);
+
 			for (int i = 0; i < asset->numSoundOverrides; i++)
 			{
 				data["soundOverrides"][i]["altmodeSound"] = (asset->soundOverrides[i].altmodeSound)
@@ -2250,6 +2263,8 @@ namespace ZoneTool
 				data["soundOverrides"][i]["soundType"] = asset->soundOverrides[i].soundType;
 			}
 
+			ZONETOOL_INFO("soundOverrides done dumping for %s", asset->szInternalName);
+
 			for (int i = 0; i < asset->numFXOverrides; i++)
 			{
 				data["fxOverrides"][i]["altmodeFX"] = (asset->fxOverrides[i].altmodeFX)
@@ -2263,6 +2278,8 @@ namespace ZoneTool
 					: "";
 			}
 
+			ZONETOOL_INFO("fxOverrides done dumping for %s", asset->szInternalName);
+
 			for (int i = 0; i < asset->numReloadStateTimerOverrides; i++)
 			{
 				data["reloadOverrides"][i]["attachment"] = asset->reloadOverrides[i].attachment;
@@ -2270,6 +2287,8 @@ namespace ZoneTool
 				data["reloadOverrides"][i]["reloadEmptyAddTime"] = asset->reloadOverrides[i].reloadEmptyAddTime;
 			}
 
+			ZONETOOL_INFO("reloadOverrides done dumping for %s", asset->szInternalName);
+
 			for (int i = 0; i < asset->numNotetrackOverrides; i++)
 			{
 				data["notetrackOverrides"][i]["attachment"] = asset->notetrackOverrides[i].attachment;
@@ -2283,6 +2302,8 @@ namespace ZoneTool
 				}
 			}
 
+			ZONETOOL_INFO("notetrackOverrides done dumping for %s", asset->szInternalName);
+
 			WEAPON_DUMP_FIELD(szInternalName);
 			WEAPON_DUMP_FIELD(szDisplayName);
 			WEAPON_DUMP_FIELD(numAnimOverrides);
@@ -2325,6 +2346,8 @@ namespace ZoneTool
 
 		void IWeaponDef::dump(WeaponCompleteDef* asset, const std::function<const char* (uint16_t)>& convertToString)
 		{
+			ZONETOOL_INFO("Dumping WeaponCompleteDef %s", asset->szInternalName);
+
 			std::string path = "weapons/mp/"s + asset->szInternalName;
 			std::string json = dump_complete(asset, convertToString).dump(4);
 

src/IW5/Zone.cpp

@@ -16,7 +16,7 @@ namespace ZoneTool
 		{
 			if (name.empty())
 			{
-				return nullptr;	
+				return nullptr;
 			}
 			
 			for (std::size_t idx = 0; idx < m_assets.size(); idx++)
@@ -245,12 +245,12 @@ namespace ZoneTool
 			buf->save("debug\\" + this->name_ + ".zone");
 
 			// Compress buffer
-			auto buf_compressed = buf->compress_zstd();
+			auto buf_compressed = buf->compress_zlib();
 
 			// Generate FF header
 			auto header = this->m_zonemem->Alloc<XFileHeader>();
 			strcpy(header->header, "IWffu100");
-			header->version = 2000;
+			header->version = 1;
 			header->allowOnlineUpdate = 0;
 
 			// Save fastfile
@@ -260,15 +260,16 @@ namespace ZoneTool
 
 			fastfile.write(buf_compressed.data(), buf_compressed.size());
 
-			std::string localappdata = getenv("LOCALAPPDATA");
+			//std::string localappdata = getenv("LOCALAPPDATA");
-			fastfile.save(localappdata + "\\Plutonium-staging\\storage\\iw5\\zone\\" + this->name_ + ".ff");
+			//fastfile.save(localappdata + "\\Plutonium-staging\\storage\\iw5\\zone\\" + this->name_ + ".ff");
 
 			// oxygen output paths
 			// fastfile.save("C:\\Users\\RektInator\\AppData\\Local\\Plutonium\\storage\\iw5\\zone\\" + this->name_ + ".ff");
-			fastfile.save("zone\\english\\" + this->name_ + ".ff");
+			CreateDirectoryA("zone_out", nullptr);
+			fastfile.save("zone_out\\" + this->name_ + ".ff");
 
 			ZONETOOL_INFO("Successfully compiled fastfile \"%s\"!", this->name_.data());
-			ZONETOOL_INFO("Compiling took %u msec.", GetTickCount64() - startTime);
+			ZONETOOL_INFO("Compiling took %llu msec.", GetTickCount64() - startTime);
 
 			// this->m_linker->UnloadZones();
 

src/ZoneUtils/Utils/String.cpp

@@ -0,0 +1,250 @@
+#include "stdafx.hpp"
+#include "String.hpp"
+
+#include <algorithm>
+#include <ranges>
+
+#ifdef ENABLE_BASE128
+#include "base128.h"
+#endif
+
+namespace ZoneTool::Utils
+{
+	const char* VA(const char* fmt, ...)
+	{
+		static thread_local VAProvider<8, 256> provider;
+
+		va_list ap;
+		va_start(ap, fmt);
+
+		const char* result;
+		result = provider.get(fmt, ap);
+
+		va_end(ap);
+		return result;
+	}
+
+	std::string ToLower(const std::string& text)
+	{
+		std::string result;
+		std::transform(text.begin(), text.end(), std::back_inserter(result), [](const unsigned char input) -> char
+		{
+			return static_cast<char>(std::tolower(input));
+		});
+
+		return result;
+	}
+
+	std::string ToUpper(const std::string& text)
+	{
+		std::string result;
+		std::transform(text.begin(), text.end(), std::back_inserter(result), [](const unsigned char input) -> char
+		{
+			return static_cast<char>(std::toupper(input));
+		});
+
+		return result;
+	}
+
+	bool Compare(const std::string& lhs, const std::string& rhs)
+	{
+		return false;
+	}
+
+	std::string DumpHex(const std::string& data, const std::string& separator)
+	{
+		std::string result;
+
+		for (std::size_t i = 0; i < data.size(); ++i)
+		{
+			if (i > 0)
+			{
+				result.append(separator);
+			}
+
+			result.append(VA("%02X", data[i] & 0xFF));
+		}
+
+		return result;
+	}
+
+	std::string XOR(std::string str, char value)
+	{
+		for (std::size_t i = 0; i < str.size(); ++i)
+		{
+			str[i] ^= static_cast<unsigned char>(value);
+		}
+
+		return str;
+	}
+
+	std::vector<std::string> Split(const std::string& str, const char delim)
+	{
+		std::stringstream ss(str);
+		std::string item;
+		std::vector<std::string> elems;
+
+		while (std::getline(ss, item, delim))
+		{
+			elems.push_back(item); // elems.push_back(std::move(item)); // if C++11 (based on comment from S1x)
+		}
+
+		return elems;
+	}
+
+	void Replace(std::string& str, const std::string& from, const std::string& to)
+	{
+		std::size_t nPos = 0;
+
+		while ((nPos = str.find(from, nPos)) != std::string::npos)
+		{
+			str = str.replace(nPos, from.length(), to);
+			nPos += to.length();
+		}
+	}
+
+	bool StartsWith(const std::string& haystack, const std::string& needle)
+	{
+		return haystack.find(needle) == 0; // If the pos of the first found char is 0, string starts with 'needle'
+	}
+
+	bool EndsWith(const std::string& haystack, const std::string& needle)
+	{
+		if (needle.size() > haystack.size()) return false;
+		return std::equal(needle.rbegin(), needle.rend(), haystack.rbegin());
+	}
+
+	bool IsNumber(const std::string& str)
+	{
+		return !str.empty() && std::find_if(str.begin(), str.end(), [](unsigned char input)
+		{
+			return !std::isdigit(input);
+		}) == str.end();
+	}
+
+	// Trim from start
+	std::string& LTrim(std::string& str)
+	{
+		str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](const unsigned char input)
+		{
+			return !std::isspace(input);
+		}));
+
+		return str;
+	}
+
+	// Trim from end
+	std::string& RTrim(std::string& str)
+	{
+		str.erase(std::find_if(str.rbegin(), str.rend(), [](const  unsigned char input)
+		{
+			return !std::isspace(input);
+		}).base(), str.end());
+
+		return str;
+	}
+
+	// Trim from both ends
+	void Trim(std::string& str)
+	{
+		LTrim(RTrim(str));
+	}
+
+	std::string Convert(const std::wstring& wstr)
+	{
+		std::string result;
+		result.reserve(wstr.size());
+
+		for (const auto& chr : wstr)
+		{
+			result.push_back(static_cast<char>(chr));
+		}
+
+		return result;
+	}
+
+	std::wstring Convert(const std::string& str)
+	{
+		std::wstring result;
+		result.reserve(str.size());
+
+		for (const auto& chr : str)
+		{
+			result.push_back(static_cast<wchar_t>(chr));
+		}
+
+		return result;
+	}
+
+	std::string FormatTimeSpan(int milliseconds)
+	{
+		int secondsTotal = milliseconds / 1000;
+		int seconds = secondsTotal % 60;
+		int minutesTotal = secondsTotal / 60;
+		int minutes = minutesTotal % 60;
+		int hoursTotal = minutesTotal / 60;
+
+		return VA("%02d:%02d:%02d", hoursTotal, minutes, seconds);
+	}
+
+	std::string FormatBandwidth(std::size_t bytes, int milliseconds)
+	{
+		static const char* sizes[] =
+		{
+			"B",
+			"KB",
+			"MB",
+			"GB",
+			"TB",
+		};
+
+		if (!milliseconds) return "0.00 B/s";
+
+		double bytesPerSecond = (1000.0 / milliseconds) * bytes;
+
+		std::size_t i;
+		for (i = 0; bytesPerSecond > 1000 && i < ARRAYSIZE(sizes); ++i) // 1024 or 1000?
+		{
+			bytesPerSecond /= 1000;
+		}
+
+		return VA("%.2f %s/s", static_cast<float>(bytesPerSecond), sizes[i]);
+	}
+
+#ifdef ENABLE_BASE64
+	// Encodes a given string in Base64
+	std::string EncodeBase64(const char* input, const unsigned long inputSize)
+	{
+		unsigned long outlen = long(inputSize + (inputSize / 3.0) + 16);
+		unsigned char* outbuf = new unsigned char[outlen]; //Reserve output memory
+		base64_encode(reinterpret_cast<unsigned char*>(const_cast<char*>(input)), inputSize, outbuf, &outlen);
+		std::string ret(reinterpret_cast<char*>(outbuf), outlen);
+		delete[] outbuf;
+		return ret;
+	}
+
+	// Encodes a given string in Base64
+	std::string EncodeBase64(const std::string& input)
+	{
+		return EncodeBase64(input.data(), input.size());
+	}
+#endif
+
+#ifdef ENABLE_BASE128
+	// Encodes a given string in Base128
+	std::string EncodeBase128(const std::string& input)
+	{
+		base128 encoder;
+
+		void* inbuffer = const_cast<char*>(input.data());
+		char* buffer = encoder.encode(inbuffer, input.size());
+		/*
+		Interesting to see that the buffer returned by the encoder is not a standalone string copy
+		but instead is a pointer to the internal "encoded" field of the encoder. So if you deinitialize
+		the encoder that string will probably become garbage.
+		*/
+		std::string retval(buffer);
+		return retval;
+	}
+#endif
+}

src/ZoneUtils/Utils/String.hpp

@@ -0,0 +1,153 @@
+#pragma once
+
+template <class Type, std::size_t n>
+constexpr auto ARRAY_COUNT(Type(&)[n]) { return n; }
+
+namespace ZoneTool::Utils
+{
+	template <std::size_t Buffers, std::size_t MinBufferSize>
+	class VAProvider
+	{
+	public:
+		static_assert(Buffers != 0 && MinBufferSize != 0, "Buffers and MinBufferSize mustn't be 0");
+
+		VAProvider() : currentBuffer_(0) {}
+
+		[[nodiscard]] const char* get(const char* format, va_list ap)
+		{
+			++this->currentBuffer_ %= ARRAY_COUNT(this->stringPool_);
+			auto entry = &this->stringPool_[this->currentBuffer_];
+
+			if (!entry->size_ || !entry->buffer_)
+			{
+				throw std::runtime_error("String pool not initialized");
+			}
+
+			while (true)
+			{
+				const auto res = vsnprintf_s(entry->buffer_, entry->size_, _TRUNCATE, format, ap);
+				if (res > 0) break; // Success
+				if (res == 0) return ""; // Error
+
+				entry->doubleSize();
+			}
+
+			return entry->buffer_;
+		}
+
+	private:
+		class Entry
+		{
+		public:
+			Entry(std::size_t size = MinBufferSize) : size_(size), buffer_(nullptr)
+			{
+				if (this->size_ < MinBufferSize) this->size_ = MinBufferSize;
+				this->allocate();
+			}
+
+			~Entry()
+			{
+				if (this->buffer_) std::free(this->buffer_);
+				this->size_ = 0;
+				this->buffer_ = nullptr;
+			}
+
+			void allocate()
+			{
+				if (this->buffer_) std::free(this->buffer_);
+				this->buffer_ = (char*)std::calloc(this->size_ + 1, 1);
+			}
+
+			void doubleSize()
+			{
+				this->size_ *= 2;
+				this->allocate();
+			}
+
+			std::size_t size_;
+			char* buffer_;
+		};
+
+		std::size_t currentBuffer_;
+		Entry stringPool_[Buffers];
+	};
+
+	template <typename Arg> // This should display a nice "nullptr" instead of a number
+	static void SanitizeFormatArgs(Arg& arg)
+	{
+		if constexpr (std::is_same_v<Arg, char*> || std::is_same_v<Arg, const char*>)
+		{
+			if (arg == nullptr)
+			{
+				arg = const_cast<char*>("nullptr");
+			}
+		}
+	}
+
+	[[nodiscard]] const char* VA(const char* fmt, ...);
+
+	template <typename... Args>
+	[[nodiscard]] const char* Format(std::string_view fmt, Args&&... args)
+	{
+		static thread_local std::string vaBuffer;
+		(SanitizeFormatArgs(args), ...);
+		std::vformat(fmt, std::make_format_args(args...)).swap(vaBuffer);
+		return vaBuffer.data();
+	}
+
+	[[nodiscard]] std::string ToLower(const std::string& text);
+	[[nodiscard]] std::string ToUpper(const std::string& text);
+
+	template <class OutputIter>
+	[[nodiscard]] OutputIter ApplyToLower(OutputIter container)
+	{
+		OutputIter result;
+		std::ranges::transform(container, std::back_inserter(result), [](const std::string& s) -> std::string
+		{
+			return ToLower(s);
+		});
+
+		return result;
+	}
+
+	template <class OutputIter>
+	[[nodiscard]] OutputIter ApplyToUpper(OutputIter container)
+	{
+		OutputIter result;
+		std::ranges::transform(container, std::back_inserter(result), [](const std::string& s) -> std::string
+		{
+			return ToUpper(s);
+		});
+
+		return result;
+	}
+
+	[[nodiscard]] bool Compare(const std::string& lhs, const std::string& rhs);
+
+	[[nodiscard]] std::vector<std::string> Split(const std::string& str, char delim);
+	void Replace(std::string& str, const std::string& from, const std::string& to);
+
+	[[nodiscard]] bool StartsWith(const std::string& haystack, const std::string& needle);
+	[[nodiscard]] bool EndsWith(const std::string& haystack, const std::string& needle);
+
+	[[nodiscard]] bool IsNumber(const std::string& str);
+
+	std::string& LTrim(std::string& str);
+	std::string& RTrim(std::string& str);
+	void Trim(std::string& str);
+
+	[[nodiscard]] std::string Convert(const std::wstring& wstr);
+	[[nodiscard]] std::wstring Convert(const std::string& str);
+
+	[[nodiscard]] std::string FormatTimeSpan(int milliseconds);
+	[[nodiscard]] std::string FormatBandwidth(std::size_t bytes, int milliseconds);
+
+	[[nodiscard]] std::string DumpHex(const std::string& data, const std::string& separator = " ");
+
+	[[nodiscard]] std::string XOR(std::string str, char value);
+
+	[[nodiscard]] std::string EncodeBase64(const char* input, unsigned long inputSize);
+	[[nodiscard]] std::string EncodeBase64(const std::string& input);
+
+	[[nodiscard]] std::string EncodeBase128(const std::string& input);
+}

src/ZoneUtils/Zone/ZoneBuffer.cpp

@@ -339,29 +339,6 @@ namespace ZoneTool
 		return ZoneBuffer::compress_zlib(data.data(), data.size(), compress_blocks);
 	}
 
-	std::vector<std::uint8_t> ZoneBuffer::compress_zstd()
-	{
-		// calculate buffer size needed for current zone
-		auto size = ZSTD_compressBound(this->m_pos);
-
-		// alloc array for compressed data
-		std::vector<std::uint8_t> compressed;
-		compressed.resize(size);
-
-		// compress buffer
-		auto destsize = ZSTD_compress(compressed.data(), size, this->m_buf.data(), this->m_pos, 11);
-		compressed.resize(destsize);
-
-		if (ZSTD_isError(destsize))
-		{
-			ZONETOOL_ERROR("An error occured while compressing the fastfile: %s", ZSTD_getErrorName(destsize));
-			return {};
-		}
-
-		// return compressed buffer
-		return compressed;
-	}
-
 	std::vector<std::uint8_t> ZoneBuffer::compress_zlib(bool compress_blocks)
 	{
 		return ZoneBuffer::compress_zlib(this->m_buf.data(), this->m_pos, compress_blocks);

src/ZoneUtils/Zone/ZoneBuffer.hpp

@@ -349,7 +349,6 @@ namespace ZoneTool
 		static std::vector<std::uint8_t> compress_zlib(const std::uint8_t* data, const std::size_t size, bool compress_blocks = false);
 		static std::vector<std::uint8_t> compress_zlib(const std::vector<std::uint8_t>& data, bool compress_blocks = false);
 		
-		std::vector<std::uint8_t> compress_zstd();
 		std::vector<std::uint8_t> compress_zlib(bool compress_blocks = false);
 		void encrypt();
 	};

src/ZoneUtils/ZoneUtils.hpp

@@ -148,6 +148,7 @@ namespace ZoneTool
 #include "Utils/FileSystem.hpp"
 #include "Utils/Function.hpp"
 #include "Utils/Memory.hpp"
+#include "Utils/String.hpp"
 #include "Utils/BinaryDumper.hpp"
 #include "Utils/Expressions.hpp"
 #include "Linker.hpp"