Add loading of iw3 fastfiles

src/Common/Game/IW3/IW3_Assets.h

@@ -55,12 +55,10 @@ namespace IW3
     enum XFileBlock
     {
         XFILE_BLOCK_TEMP,
-        XFILE_BLOCK_RUNTIME_BEGIN,
+        XFILE_BLOCK_RUNTIME,
-        XFILE_BLOCK_RUNTIME = XFILE_BLOCK_RUNTIME_BEGIN,
         XFILE_BLOCK_LARGE_RUNTIME,
         XFILE_BLOCK_PHYSICAL_RUNTIME,
-        XFILE_BLOCK_RUNTIME_END,
+        XFILE_BLOCK_VIRTUAL,
-        XFILE_BLOCK_VIRTUAL = XFILE_BLOCK_RUNTIME_END,
         XFILE_BLOCK_LARGE,
         XFILE_BLOCK_PHYSICAL,
         XFILE_BLOCK_VERTEX,
@@ -252,7 +250,7 @@ namespace IW3
         uint16_t numframes;
         bool bLoop;
         bool bDelta;
-        char boneCount[10];
+        unsigned char boneCount[10];
         char notifyCount;
         char assetType;
         bool isDefault;

src/ZoneCode/Game/IW3/XAssets/MaterialTechniqueSet.txt

@@ -18,6 +18,9 @@ reorder:
 // MaterialPass
 use MaterialPass;
 set count args perPrimArgCount + perObjArgCount + stableArgCount;
+set reusable vertexDecl;
+set reusable vertexShader;
+set reusable pixelShader;
 
 // MaterialShaderArgument
 use MaterialShaderArgument;

src/ZoneCommon/Game/IW3/GameAssetPoolIW3.cpp

@@ -10,14 +10,11 @@ using namespace IW3;
 
 const char* GameAssetPoolIW3::ASSET_TYPE_NAMES[]
 {
+    "xmodelpieces",
     "physpreset",
     "xanim",
-    "xmodelsurfs",
     "xmodel",
     "material",
-    "pixelshader",
-    "vertexshader",
-    "vertexdecl",
     "techniqueset",
     "image",
     "sound",
@@ -86,7 +83,7 @@ GameAssetPoolIW3::GameAssetPoolIW3(Zone* zone, const int priority)
     : ZoneAssetPools(zone),
     m_priority(priority)
 {
-    assert(std::extent<decltype(ASSET_TYPE_NAMES)>::value == ASSET_TYPE_COUNT);
+    static_assert(std::extent<decltype(ASSET_TYPE_NAMES)>::value == ASSET_TYPE_COUNT);
 
     m_phys_preset = nullptr;
     m_xanim_parts = nullptr;

src/ZoneCommon/Game/IW3/ZoneConstantsIW3.h

@@ -0,0 +1,26 @@
+#pragma once
+#include <cstdint>
+#include <string>
+
+#include "Zone/ZoneTypes.h"
+#include "Game/IW3/IW3.h"
+
+namespace IW3
+{
+    class ZoneConstants final
+    {
+        ZoneConstants() = default;
+
+    public:
+        static constexpr const char* MAGIC_UNSIGNED = "IWffu100";
+        static constexpr int ZONE_VERSION = 5;
+        
+        static_assert(std::char_traits<char>::length(MAGIC_UNSIGNED) == sizeof(ZoneHeader::m_magic));
+
+        static constexpr size_t AUTHED_CHUNK_SIZE = 0x2000;
+        static constexpr unsigned AUTHED_CHUNK_COUNT_PER_GROUP = 256;
+
+        static constexpr int OFFSET_BLOCK_BIT_COUNT = 4;
+        static constexpr block_t INSERT_BLOCK = XFILE_BLOCK_VIRTUAL;
+    };
+}

src/ZoneLoading/Game/IW3/ContentLoaderIW3.cpp

@@ -0,0 +1,175 @@
+#include "ContentLoaderIW3.h"
+#include "Game/IW3/IW3.h"
+#include "Loading/Exception/UnsupportedAssetTypeException.h"
+
+#include <cassert>
+
+#include "Game/IW3/XAssets/clipmap_t/clipmap_t_load_db.h"
+#include "Game/IW3/XAssets/comworld/comworld_load_db.h"
+#include "Game/IW3/XAssets/font_s/font_s_load_db.h"
+#include "Game/IW3/XAssets/fxeffectdef/fxeffectdef_load_db.h"
+#include "Game/IW3/XAssets/fximpacttable/fximpacttable_load_db.h"
+#include "Game/IW3/XAssets/gameworldmp/gameworldmp_load_db.h"
+#include "Game/IW3/XAssets/gameworldsp/gameworldsp_load_db.h"
+#include "Game/IW3/XAssets/gfximage/gfximage_load_db.h"
+#include "Game/IW3/XAssets/gfxlightdef/gfxlightdef_load_db.h"
+#include "Game/IW3/XAssets/gfxworld/gfxworld_load_db.h"
+#include "Game/IW3/XAssets/loadedsound/loadedsound_load_db.h"
+#include "Game/IW3/XAssets/localizeentry/localizeentry_load_db.h"
+#include "Game/IW3/XAssets/mapents/mapents_load_db.h"
+#include "Game/IW3/XAssets/material/material_load_db.h"
+#include "Game/IW3/XAssets/materialtechniqueset/materialtechniqueset_load_db.h"
+#include "Game/IW3/XAssets/menudef_t/menudef_t_load_db.h"
+#include "Game/IW3/XAssets/menulist/menulist_load_db.h"
+#include "Game/IW3/XAssets/physpreset/physpreset_load_db.h"
+#include "Game/IW3/XAssets/rawfile/rawfile_load_db.h"
+#include "Game/IW3/XAssets/snd_alias_list_t/snd_alias_list_t_load_db.h"
+#include "Game/IW3/XAssets/sndcurve/sndcurve_load_db.h"
+#include "Game/IW3/XAssets/stringtable/stringtable_load_db.h"
+#include "Game/IW3/XAssets/weapondef/weapondef_load_db.h"
+#include "Game/IW3/XAssets/xanimparts/xanimparts_load_db.h"
+#include "Game/IW3/XAssets/xmodel/xmodel_load_db.h"
+
+using namespace IW3;
+
+ContentLoader::ContentLoader()
+{
+    varXAsset = nullptr;
+    varScriptStringList = nullptr;
+}
+
+void ContentLoader::LoadScriptStringList(const bool atStreamStart)
+{
+    assert(m_zone->m_script_strings.Empty());
+
+    m_stream->PushBlock(XFILE_BLOCK_VIRTUAL);
+
+    if (atStreamStart)
+        m_stream->Load<ScriptStringList>(varScriptStringList);
+
+    if (varScriptStringList->strings != nullptr)
+    {
+        assert(varScriptStringList->strings == PTR_FOLLOWING);
+
+        varScriptStringList->strings = m_stream->Alloc<const char*>(alignof(const char*));
+        varXString = varScriptStringList->strings;
+        LoadXStringArray(true, varScriptStringList->count);
+
+        for (int i = 0; i < varScriptStringList->count; i++)
+        {
+            if (varScriptStringList->strings[i])
+            {
+                m_zone->m_script_strings.AddScriptString(varScriptStringList->strings[i]);
+            }
+            else
+            {
+                m_zone->m_script_strings.AddScriptString("");
+            }
+        }
+    }
+
+    m_stream->PopBlock();
+
+    assert(m_zone->m_script_strings.Count() <= SCR_STRING_MAX + 1);
+}
+
+void ContentLoader::LoadXAsset(const bool atStreamStart)
+{
+#define LOAD_ASSET(type_index, typeName, headerEntry) \
+    case type_index: \
+        { \
+            Loader_##typeName loader(m_zone, m_stream); \
+            loader.Load(&varXAsset->header.headerEntry); \
+            break; \
+        }
+#define SKIP_ASSET(type_index, typeName, headerEntry) \
+    case type_index: \
+        break;
+
+    assert(varXAsset != nullptr);
+
+    if (atStreamStart)
+        m_stream->Load<XAsset>(varXAsset);
+
+    switch (varXAsset->type)
+    {
+        LOAD_ASSET(ASSET_TYPE_PHYSPRESET, PhysPreset, physPreset)
+            LOAD_ASSET(ASSET_TYPE_XANIMPARTS, XAnimParts, parts)
+            LOAD_ASSET(ASSET_TYPE_XMODEL, XModel, model)
+            LOAD_ASSET(ASSET_TYPE_MATERIAL, Material, material)
+            LOAD_ASSET(ASSET_TYPE_TECHNIQUE_SET, MaterialTechniqueSet, techniqueSet)
+            LOAD_ASSET(ASSET_TYPE_IMAGE, GfxImage, image)
+            LOAD_ASSET(ASSET_TYPE_SOUND, snd_alias_list_t, sound)
+            LOAD_ASSET(ASSET_TYPE_SOUND_CURVE, SndCurve, sndCurve)
+            LOAD_ASSET(ASSET_TYPE_LOADED_SOUND, LoadedSound, loadSnd)
+            LOAD_ASSET(ASSET_TYPE_CLIPMAP, clipMap_t, clipMap)
+            LOAD_ASSET(ASSET_TYPE_CLIPMAP_PVS, clipMap_t, clipMap)
+            LOAD_ASSET(ASSET_TYPE_COMWORLD, ComWorld, comWorld)
+            LOAD_ASSET(ASSET_TYPE_GAMEWORLD_SP, GameWorldSp, gameWorldSp)
+            LOAD_ASSET(ASSET_TYPE_GAMEWORLD_MP, GameWorldMp, gameWorldMp)
+            LOAD_ASSET(ASSET_TYPE_MAP_ENTS, MapEnts, mapEnts)
+            LOAD_ASSET(ASSET_TYPE_GFXWORLD, GfxWorld, gfxWorld)
+            LOAD_ASSET(ASSET_TYPE_LIGHT_DEF, GfxLightDef, lightDef)
+            LOAD_ASSET(ASSET_TYPE_FONT, Font_s, font)
+            LOAD_ASSET(ASSET_TYPE_MENULIST, MenuList, menuList)
+            LOAD_ASSET(ASSET_TYPE_MENU, menuDef_t, menu)
+            LOAD_ASSET(ASSET_TYPE_LOCALIZE_ENTRY, LocalizeEntry, localize)
+            LOAD_ASSET(ASSET_TYPE_WEAPON, WeaponDef, weapon)
+            SKIP_ASSET(ASSET_TYPE_SNDDRIVER_GLOBALS, SndDriverGlobals, sndDriverGlobals)
+            LOAD_ASSET(ASSET_TYPE_FX, FxEffectDef, fx)
+            LOAD_ASSET(ASSET_TYPE_IMPACT_FX, FxImpactTable, impactFx)
+            LOAD_ASSET(ASSET_TYPE_RAWFILE, RawFile, rawfile)
+            LOAD_ASSET(ASSET_TYPE_STRINGTABLE, StringTable, stringTable)
+
+    default:
+        {
+            throw UnsupportedAssetTypeException(varXAsset->type);
+        }
+    }
+
+#undef LOAD_ASSET
+}
+
+void ContentLoader::LoadXAssetArray(const bool atStreamStart, const size_t count)
+{
+    assert(varXAsset != nullptr);
+
+    if (atStreamStart)
+        m_stream->Load<XAsset>(varXAsset, count);
+
+    for (asset_type_t assetType = 0; assetType < ASSET_TYPE_COUNT; assetType++)
+    {
+        m_zone->m_pools->InitPoolDynamic(assetType);
+    }
+
+    for (size_t index = 0; index < count; index++)
+    {
+        LoadXAsset(false);
+        varXAsset++;
+    }
+}
+
+void ContentLoader::Load(Zone* zone, IZoneInputStream* stream)
+{
+    m_zone = zone;
+    m_stream = stream;
+
+    m_stream->PushBlock(XFILE_BLOCK_VIRTUAL);
+
+    XAssetList assetList{};
+    m_stream->LoadDataRaw(&assetList, sizeof assetList);
+
+    varScriptStringList = &assetList.stringList;
+    LoadScriptStringList(false);
+
+    if (assetList.assets != nullptr)
+    {
+        assert(assetList.assets == PTR_FOLLOWING);
+
+        assetList.assets = m_stream->Alloc<XAsset>(alignof(XAsset));
+        varXAsset = assetList.assets;
+        LoadXAssetArray(true, assetList.assetCount);
+    }
+
+    m_stream->PopBlock();
+}

src/ZoneLoading/Game/IW3/ContentLoaderIW3.h

@@ -0,0 +1,23 @@
+#pragma once
+#include "Loading/ContentLoaderBase.h"
+#include "Loading/IContentLoadingEntryPoint.h"
+#include "Game/IW3/IW3.h"
+
+namespace IW3
+{
+    class ContentLoader final : public ContentLoaderBase, public IContentLoadingEntryPoint
+    {
+        XAsset* varXAsset;
+        ScriptStringList* varScriptStringList;
+
+        void LoadScriptStringList(bool atStreamStart);
+
+        void LoadXAsset(bool atStreamStart);
+        void LoadXAssetArray(bool atStreamStart, size_t count);
+
+    public:
+        ContentLoader();
+
+        void Load(Zone* zone, IZoneInputStream* stream) override;
+    };
+}

src/ZoneLoading/Game/IW3/ZoneLoaderFactoryIW3.cpp

@@ -0,0 +1,112 @@
+#include "ZoneLoaderFactoryIW3.h"
+
+#include <cassert>
+#include <cstring>
+#include <type_traits>
+
+#include "Game/IW3/IW3.h"
+
+#include "Utils/ClassUtils.h"
+#include "ContentLoaderIW3.h"
+#include "Game/IW3/GameAssetPoolIW3.h"
+#include "Game/IW3/GameIW3.h"
+#include "Game/GameLanguage.h"
+#include "Game/IW3/ZoneConstantsIW3.h"
+#include "Loading/Processor/ProcessorInflate.h"
+#include "Loading/Steps/StepSkipBytes.h"
+#include "Loading/Steps/StepAddProcessor.h"
+#include "Loading/Steps/StepAllocXBlocks.h"
+#include "Loading/Steps/StepLoadZoneContent.h"
+
+using namespace IW3;
+
+class ZoneLoaderFactory::Impl
+{
+    static GameLanguage GetZoneLanguage(std::string& zoneName)
+    {
+        return GameLanguage::LANGUAGE_NONE;
+    }
+
+    static bool CanLoad(ZoneHeader& header, bool* isSecure, bool* isOfficial)
+    {
+        assert(isSecure != nullptr);
+        assert(isOfficial != nullptr);
+
+        if (header.m_version != ZoneConstants::ZONE_VERSION)
+        {
+            return false;
+        }
+
+        if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED, std::char_traits<char>::length(ZoneConstants::MAGIC_UNSIGNED)))
+        {
+            *isSecure = false;
+            *isOfficial = true;
+            return true;
+        }
+
+        return false;
+    }
+
+    static void SetupBlock(ZoneLoader* zoneLoader)
+    {
+#define XBLOCK_DEF(name, type) std::make_unique<XBlock>(STR(name), name, type)
+
+        zoneLoader->AddXBlock(XBLOCK_DEF(IW3::XFILE_BLOCK_TEMP, XBlock::Type::BLOCK_TYPE_TEMP));
+        zoneLoader->AddXBlock(XBLOCK_DEF(IW3::XFILE_BLOCK_RUNTIME, XBlock::Type::BLOCK_TYPE_RUNTIME));
+        zoneLoader->AddXBlock(XBLOCK_DEF(IW3::XFILE_BLOCK_LARGE_RUNTIME, XBlock::Type::BLOCK_TYPE_RUNTIME));
+        zoneLoader->AddXBlock(XBLOCK_DEF(IW3::XFILE_BLOCK_PHYSICAL_RUNTIME, XBlock::Type::BLOCK_TYPE_RUNTIME));
+        zoneLoader->AddXBlock(XBLOCK_DEF(IW3::XFILE_BLOCK_VIRTUAL, XBlock::Type::BLOCK_TYPE_NORMAL));
+        zoneLoader->AddXBlock(XBLOCK_DEF(IW3::XFILE_BLOCK_LARGE, XBlock::Type::BLOCK_TYPE_NORMAL));
+        zoneLoader->AddXBlock(XBLOCK_DEF(IW3::XFILE_BLOCK_PHYSICAL, XBlock::Type::BLOCK_TYPE_NORMAL));
+        zoneLoader->AddXBlock(XBLOCK_DEF(IW3::XFILE_BLOCK_VERTEX, XBlock::Type::BLOCK_TYPE_NORMAL));
+        zoneLoader->AddXBlock(XBLOCK_DEF(IW3::XFILE_BLOCK_INDEX, XBlock::Type::BLOCK_TYPE_NORMAL));
+
+#undef XBLOCK_DEF
+    }
+
+public:
+    static ZoneLoader* CreateLoaderForHeader(ZoneHeader& header, std::string& fileName)
+    {
+        bool isSecure;
+        bool isOfficial;
+
+        // Check if this file is a supported IW4 zone.
+        if (!CanLoad(header, &isSecure, &isOfficial))
+            return nullptr;
+
+        // Create new zone
+        auto zone = std::make_unique<Zone>(fileName, 0, &g_GameIW3);
+        auto* zonePtr = zone.get();
+        zone->m_pools = std::make_unique<GameAssetPoolIW3>(zonePtr, 0);
+        zone->m_language = GetZoneLanguage(fileName);
+
+        // File is supported. Now setup all required steps for loading this file.
+        auto* zoneLoader = new ZoneLoader(std::move(zone));
+
+        SetupBlock(zoneLoader);
+
+        // Skip unknown 1 byte field that the game ignores as well
+        // zoneLoader->AddLoadingStep(std::make_unique<StepSkipBytes>(1));
+
+        // Skip timestamp
+        // zoneLoader->AddLoadingStep(std::make_unique<StepSkipBytes>(8));
+
+        zoneLoader->AddLoadingStep(std::make_unique<StepAddProcessor>(std::make_unique<ProcessorInflate>(ZoneConstants::AUTHED_CHUNK_SIZE)));
+
+        // Start of the XFile struct
+        zoneLoader->AddLoadingStep(std::make_unique<StepSkipBytes>(8));
+        // Skip size and externalSize fields since they are not interesting for us
+        zoneLoader->AddLoadingStep(std::make_unique<StepAllocXBlocks>());
+
+        // Start of the zone content
+        zoneLoader->AddLoadingStep(std::make_unique<StepLoadZoneContent>(std::make_unique<ContentLoader>(), zonePtr, ZoneConstants::OFFSET_BLOCK_BIT_COUNT, ZoneConstants::INSERT_BLOCK));
+
+        // Return the fully setup zoneloader
+        return zoneLoader;
+    }
+};
+
+ZoneLoader* ZoneLoaderFactory::CreateLoaderForHeader(ZoneHeader& header, std::string& fileName)
+{
+    return Impl::CreateLoaderForHeader(header, fileName);
+}

src/ZoneLoading/Game/IW3/ZoneLoaderFactoryIW3.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "Loading/IZoneLoaderFactory.h"
+#include <string>
+
+namespace IW3
+{
+    class ZoneLoaderFactory final : public IZoneLoaderFactory
+    {
+        class Impl;
+
+    public:
+        ZoneLoader* CreateLoaderForHeader(ZoneHeader& header, std::string& fileName) override;
+    };
+}

src/ZoneLoading/ZoneLoading.cpp

@@ -4,6 +4,7 @@
 #include <fstream>
 #include <iostream>
 
+#include "Game/IW3/ZoneLoaderFactoryIW3.h"
 #include "Game/IW4/ZoneLoaderFactoryIW4.h"
 #include "Game/T6/ZoneLoaderFactoryT6.h"
 #include "Utils/ObjFileStream.h"
@@ -12,6 +13,7 @@ namespace fs = std::filesystem;
 
 IZoneLoaderFactory* ZoneLoaderFactories[]
 {
+    new IW3::ZoneLoaderFactory(),
     new IW4::ZoneLoaderFactory(),
     new T6::ZoneLoaderFactory()
 };