Merge pull request #616 from michaeloliverx/iw4-xenon-zone

feat: dump iw4 xbox fastfile data

src/ZoneCommon/Game/IW4/ZoneConstantsIW4.h

@@ -16,8 +16,9 @@ namespace IW4
         static constexpr const char* MAGIC_SIGNED_OAT = "ABff0100";
         static constexpr const char* MAGIC_UNSIGNED = "IWffu100";
         static constexpr const char* MAGIC_IW4X = "IW4x";
-        static constexpr int ZONE_VERSION = 276;
+        static constexpr int ZONE_VERSION_PC = 276;
-        static constexpr int IW4X_ZONE_VERSION = 3;
+        static constexpr int ZONE_VERSION_IW4x = 3;
+        static constexpr int ZONE_VERSION_XENON = 269;
 
         static_assert(std::char_traits<char>::length(MAGIC_SIGNED_INFINITY_WARD) == sizeof(ZoneHeader::m_magic));
         static_assert(std::char_traits<char>::length(MAGIC_SIGNED_OAT) == sizeof(ZoneHeader::m_magic));
@@ -25,7 +26,7 @@ namespace IW4
         static_assert(std::char_traits<char>::length(MAGIC_IW4X) == sizeof(ZoneHeader::m_magic) - sizeof(uint32_t));
 
         static constexpr const char* MAGIC_AUTH_HEADER = "IWffs100";
-        inline static const uint8_t RSA_PUBLIC_KEY_INFINITY_WARD[]{
+        inline static const uint8_t RSA_PUBLIC_KEY_INFINITY_WARD_PC[]{
             0x30, 0x82, 0x01, 0x0A, 0x02, 0x82, 0x01, 0x01, 0x00, 0xA5, 0x86, 0xCC, 0x18, 0xA9, 0x12, 0x17, 0x4F, 0x3A, 0xC9, 0x0C, 0xD2, 0x38, 0x5D,
             0xDB, 0x67, 0x62, 0xA4, 0xE3, 0xD4, 0x42, 0x05, 0x8A, 0x57, 0x0C, 0x31, 0x4E, 0x19, 0xE4, 0xBA, 0x89, 0x73, 0x13, 0xDB, 0x72, 0x25, 0x63,
             0xB1, 0x2F, 0xD7, 0xF1, 0x08, 0x48, 0x34, 0x06, 0xD7, 0x84, 0x5F, 0xC8, 0xCF, 0x2F, 0xB6, 0xA3, 0x5A, 0x8F, 0x7E, 0xAA, 0x9D, 0x51, 0xE7,
@@ -40,6 +41,21 @@ namespace IW4
             0x77, 0xCD, 0x62, 0x7D, 0x9D, 0x40, 0x26, 0x44, 0x4B, 0x3B, 0x0A, 0x89, 0x02, 0x03, 0x01, 0x00, 0x01,
         };
 
+        inline static const uint8_t RSA_PUBLIC_KEY_INFINITY_WARD_XENON[270]{
+            0x30, 0x82, 0x01, 0x0A, 0x02, 0x82, 0x01, 0x01, 0x00, 0xCD, 0x96, 0x50, 0xC8, 0xB2, 0x4E, 0x10, 0xE4, 0x79, 0x05, 0x41, 0x6E, 0x9B, 0xEF,
+            0xCE, 0x51, 0xC4, 0x2D, 0x9E, 0xC9, 0xD1, 0x84, 0x23, 0xF0, 0xAC, 0x22, 0x24, 0x18, 0xE9, 0x9D, 0x28, 0xCB, 0xAF, 0xB4, 0x7F, 0x60, 0xB3,
+            0x67, 0x32, 0x43, 0xFC, 0x0F, 0xA9, 0x70, 0x78, 0x42, 0xCC, 0xA9, 0x86, 0x40, 0xCA, 0x32, 0xCA, 0x82, 0x9B, 0x0C, 0x63, 0xB0, 0x51, 0x89,
+            0x14, 0x29, 0xEA, 0x15, 0x33, 0x3F, 0x7B, 0xEB, 0x66, 0xED, 0xF7, 0x16, 0xF7, 0x45, 0x06, 0xC6, 0x41, 0x62, 0xDE, 0x00, 0x75, 0xFA, 0x8C,
+            0x8F, 0xE5, 0xBF, 0x73, 0xCB, 0x75, 0x2C, 0x44, 0x09, 0x50, 0xD8, 0x1E, 0x51, 0xE2, 0x58, 0xB2, 0x56, 0x6F, 0xE5, 0xF5, 0xC0, 0x20, 0x4F,
+            0x5F, 0x55, 0x8F, 0x8D, 0xD1, 0x37, 0xCC, 0xE8, 0x3B, 0x09, 0x06, 0x2C, 0x5D, 0xB9, 0x7B, 0x8E, 0xE3, 0xC7, 0x83, 0xCB, 0x8F, 0x40, 0x63,
+            0x89, 0xAE, 0x6B, 0x0F, 0xDE, 0x38, 0x5E, 0xD7, 0xF4, 0x65, 0x8C, 0x30, 0x5A, 0x8B, 0x9E, 0xA2, 0x42, 0x03, 0xF7, 0x80, 0xAB, 0xDF, 0xA0,
+            0x8F, 0x95, 0xA0, 0xBB, 0xB6, 0x1D, 0xA3, 0xBD, 0x3B, 0x6F, 0x42, 0x68, 0xDD, 0x42, 0xFB, 0xE6, 0x32, 0x02, 0x3B, 0x08, 0xEE, 0x34, 0x3C,
+            0x8F, 0x1E, 0xE8, 0x59, 0x11, 0x08, 0x09, 0x29, 0xBE, 0x69, 0xB1, 0xD4, 0x55, 0x10, 0x6F, 0xF3, 0x92, 0x17, 0x09, 0x3E, 0x70, 0xA0, 0x1A,
+            0xE6, 0x7D, 0x6D, 0x31, 0xD0, 0xF8, 0x33, 0xDF, 0xF0, 0x29, 0x55, 0x62, 0x1D, 0x5F, 0x3E, 0x8D, 0x3A, 0x1D, 0x5A, 0x13, 0xB0, 0xF9, 0x89,
+            0x89, 0x91, 0x8F, 0x06, 0xBA, 0x3A, 0x37, 0x16, 0xC5, 0x7D, 0x89, 0xEF, 0xEF, 0x63, 0x4E, 0xF3, 0x89, 0x19, 0x06, 0x89, 0xF1, 0x7B, 0xDC,
+            0xF5, 0x6E, 0x36, 0x0A, 0x8B, 0xC8, 0xC4, 0x5D, 0xC2, 0x7D, 0x13, 0x9D, 0x02, 0x03, 0x01, 0x00, 0x01,
+        };
+
         static constexpr size_t AUTHED_CHUNK_SIZE = 0x2000;
         static constexpr unsigned AUTHED_CHUNK_COUNT_PER_GROUP = 256;
 

src/ZoneLoading/Game/IW4/ZoneLoaderFactoryIW4.cpp

@@ -12,25 +12,30 @@
 #include "Loading/Processor/ProcessorInflate.h"
 #include "Loading/Steps/StepAddProcessor.h"
 #include "Loading/Steps/StepAllocXBlocks.h"
+#include "Loading/Steps/StepDumpData.h"
 #include "Loading/Steps/StepLoadHash.h"
 #include "Loading/Steps/StepLoadSignature.h"
 #include "Loading/Steps/StepLoadZoneContent.h"
 #include "Loading/Steps/StepLoadZoneSizes.h"
 #include "Loading/Steps/StepRemoveProcessor.h"
 #include "Loading/Steps/StepSkipBytes.h"
+#include "Loading/Steps/StepSkipZoneImageHeaders.h"
 #include "Loading/Steps/StepVerifyFileName.h"
 #include "Loading/Steps/StepVerifyHash.h"
 #include "Loading/Steps/StepVerifyMagic.h"
 #include "Loading/Steps/StepVerifySignature.h"
 #include "Utils/ClassUtils.h"
+#include "Utils/Endianness.h"
 #include "Utils/Logging/Log.h"
 
 #include <cassert>
 #include <cstring>
+#include <filesystem>
 #include <iostream>
 #include <type_traits>
 
 using namespace IW4;
+namespace fs = std::filesystem;
 
 namespace
 {
@@ -42,13 +47,49 @@ namespace
 
     std::optional<ZoneLoaderInspectionResultIW4> InspectZoneHeaderIw4(const ZoneHeader& header)
     {
-        if (header.m_version != ZoneConstants::ZONE_VERSION)
+        if (endianness::FromLittleEndian(header.m_version) == ZoneConstants::ZONE_VERSION_PC)
-            return std::nullopt;
-
-        if (!memcmp(header.m_magic, ZoneConstants::MAGIC_IW4X, std::char_traits<char>::length(ZoneConstants::MAGIC_IW4X)))
         {
-            if (*reinterpret_cast<const uint32_t*>(&header.m_magic[std::char_traits<char>::length(ZoneConstants::MAGIC_IW4X)])
+            if (!memcmp(header.m_magic, ZoneConstants::MAGIC_IW4X, std::char_traits<char>::length(ZoneConstants::MAGIC_IW4X)))
-                == ZoneConstants::IW4X_ZONE_VERSION)
+            {
+                if (*reinterpret_cast<const uint32_t*>(&header.m_magic[std::char_traits<char>::length(ZoneConstants::MAGIC_IW4X)])
+                    == ZoneConstants::ZONE_VERSION_IW4x)
+                {
+                    return ZoneLoaderInspectionResultIW4{
+                        .m_generic_result =
+                            ZoneLoaderInspectionResult{
+                                                       .m_game_id = GameId::IW4,
+                                                       .m_endianness = GameEndianness::LE,
+                                                       .m_word_size = GameWordSize::ARCH_32,
+                                                       .m_platform = GamePlatform::PC,
+                                                       .m_is_official = false,
+                                                       .m_is_signed = false,
+                                                       .m_is_encrypted = false,
+                                                       },
+                        .m_is_iw4x = true,
+                    };
+                }
+
+                return std::nullopt;
+            }
+
+            if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_INFINITY_WARD, std::char_traits<char>::length(ZoneConstants::MAGIC_SIGNED_INFINITY_WARD)))
+            {
+                return ZoneLoaderInspectionResultIW4{
+                    .m_generic_result =
+                        ZoneLoaderInspectionResult{
+                                                   .m_game_id = GameId::IW4,
+                                                   .m_endianness = GameEndianness::LE,
+                                                   .m_word_size = GameWordSize::ARCH_32,
+                                                   .m_platform = GamePlatform::PC,
+                                                   .m_is_official = true,
+                                                   .m_is_signed = true,
+                                                   .m_is_encrypted = false,
+                                                   },
+                    .m_is_iw4x = false,
+                };
+            }
+
+            if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED, std::char_traits<char>::length(ZoneConstants::MAGIC_UNSIGNED)))
             {
                 return ZoneLoaderInspectionResultIW4{
                     .m_generic_result =
@@ -61,45 +102,44 @@ namespace
                                                    .m_is_signed = false,
                                                    .m_is_encrypted = false,
                                                    },
-                    .m_is_iw4x = true,
+                    .m_is_iw4x = false,
                 };
             }
-
-            return std::nullopt;
         }
-
+        else if (endianness::FromBigEndian(header.m_version) == ZoneConstants::ZONE_VERSION_XENON)
-        if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_INFINITY_WARD, std::char_traits<char>::length(ZoneConstants::MAGIC_SIGNED_INFINITY_WARD)))
         {
-            return ZoneLoaderInspectionResultIW4{
+            if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED, std::char_traits<char>::length(ZoneConstants::MAGIC_UNSIGNED)))
-                .m_generic_result =
+            {
-                    ZoneLoaderInspectionResult{
+                return ZoneLoaderInspectionResultIW4{
-                                               .m_game_id = GameId::IW4,
+                    .m_generic_result =
-                                               .m_endianness = GameEndianness::LE,
+                        ZoneLoaderInspectionResult{
-                                               .m_word_size = GameWordSize::ARCH_32,
+                                                   .m_game_id = GameId::IW4,
-                                               .m_platform = GamePlatform::PC,
+                                                   .m_endianness = GameEndianness::BE,
-                                               .m_is_official = true,
+                                                   .m_word_size = GameWordSize::ARCH_32,
-                                               .m_is_signed = true,
+                                                   .m_platform = GamePlatform::XBOX,
-                                               .m_is_encrypted = false,
+                                                   .m_is_official = false,
-                                               },
+                                                   .m_is_signed = false,
-                .m_is_iw4x = false,
+                                                   .m_is_encrypted = false,
-            };
+                                                   },
-        }
+                    .m_is_iw4x = false,
-
+                };
-        if (!memcmp(header.m_magic, ZoneConstants::MAGIC_UNSIGNED, std::char_traits<char>::length(ZoneConstants::MAGIC_UNSIGNED)))
+            }
-        {
+            if (!memcmp(header.m_magic, ZoneConstants::MAGIC_SIGNED_INFINITY_WARD, std::char_traits<char>::length(ZoneConstants::MAGIC_SIGNED_INFINITY_WARD)))
-            return ZoneLoaderInspectionResultIW4{
+            {
-                .m_generic_result =
+                return ZoneLoaderInspectionResultIW4{
-                    ZoneLoaderInspectionResult{
+                    .m_generic_result =
-                                               .m_game_id = GameId::IW4,
+                        ZoneLoaderInspectionResult{
-                                               .m_endianness = GameEndianness::LE,
+                                                   .m_game_id = GameId::IW4,
-                                               .m_word_size = GameWordSize::ARCH_32,
+                                                   .m_endianness = GameEndianness::BE,
-                                               .m_platform = GamePlatform::PC,
+                                                   .m_word_size = GameWordSize::ARCH_32,
-                                               .m_is_official = false,
+                                                   .m_platform = GamePlatform::XBOX,
-                                               .m_is_signed = false,
+                                                   .m_is_official = true,
-                                               .m_is_encrypted = false,
+                                                   .m_is_signed = true,
-                                               },
+                                                   .m_is_encrypted = false,
-                .m_is_iw4x = false,
+                                                   },
-            };
+                    .m_is_iw4x = false,
+                };
+            }
         }
 
         return std::nullopt;
@@ -121,13 +161,28 @@ namespace
 #undef XBLOCK_DEF
     }
 
-    std::unique_ptr<cryptography::IPublicKeyAlgorithm> SetupRsa(const bool isOfficial)
+    std::unique_ptr<cryptography::IPublicKeyAlgorithm> SetupRsa(const bool isOfficial, const GamePlatform platform)
     {
         if (isOfficial)
         {
             auto rsa = cryptography::CreateRsa(cryptography::HashingAlgorithm::RSA_HASH_SHA256, cryptography::RsaPaddingMode::RSA_PADDING_PSS);
 
-            if (!rsa->SetKey(ZoneConstants::RSA_PUBLIC_KEY_INFINITY_WARD, sizeof(ZoneConstants::RSA_PUBLIC_KEY_INFINITY_WARD)))
+            bool keySetSuccessful;
+            if (platform == GamePlatform::PC)
+            {
+                keySetSuccessful = rsa->SetKey(ZoneConstants::RSA_PUBLIC_KEY_INFINITY_WARD_PC, sizeof(ZoneConstants::RSA_PUBLIC_KEY_INFINITY_WARD_PC));
+            }
+            else if (platform == GamePlatform::XBOX)
+            {
+                keySetSuccessful = rsa->SetKey(ZoneConstants::RSA_PUBLIC_KEY_INFINITY_WARD_XENON, sizeof(ZoneConstants::RSA_PUBLIC_KEY_INFINITY_WARD_XENON));
+            }
+            else
+            {
+                con::error("No public key for platform");
+                return nullptr;
+            }
+
+            if (!keySetSuccessful)
             {
                 con::error("Invalid public key for signature checking");
                 return nullptr;
@@ -151,7 +206,7 @@ namespace
             return;
 
         // If file is signed setup a RSA instance.
-        auto rsa = SetupRsa(inspectResult.m_generic_result.m_is_official);
+        auto rsa = SetupRsa(inspectResult.m_generic_result.m_is_official, inspectResult.m_generic_result.m_platform);
 
         zoneLoader.AddLoadingStep(step::CreateStepVerifyMagic(ZoneConstants::MAGIC_AUTH_HEADER));
         zoneLoader.AddLoadingStep(step::CreateStepSkipBytes(4)); // Skip reserved
@@ -227,6 +282,10 @@ std::unique_ptr<ZoneLoader> ZoneLoaderFactory::CreateLoaderForHeader(const ZoneH
     // Skip timestamp
     zoneLoader->AddLoadingStep(step::CreateStepSkipBytes(8));
 
+    // Xbox fastfiles have an additional header of all included images outside the zone data
+    if (inspectResult->m_generic_result.m_platform == GamePlatform::XBOX)
+        zoneLoader->AddLoadingStep(step::CreateStepSkipZoneImageHeaders());
+
     // Add steps for loading the auth header which also contain the signature of the zone if it is signed.
     AddAuthHeaderSteps(*inspectResult, *zoneLoader, fileName);
 
@@ -238,21 +297,32 @@ std::unique_ptr<ZoneLoader> ZoneLoaderFactory::CreateLoaderForHeader(const ZoneH
         zoneLoader->AddLoadingStep(step::CreateStepSkipBytes(1));
     }
 
-    // Start of the XFile struct
+    if (inspectResult->m_generic_result.m_endianness == GameEndianness::LE)
-    zoneLoader->AddLoadingStep(step::CreateStepLoadZoneSizes());
+    {
-    zoneLoader->AddLoadingStep(step::CreateStepAllocXBlocks());
+        // Start of the XFile struct
+        zoneLoader->AddLoadingStep(step::CreateStepLoadZoneSizes());
+        zoneLoader->AddLoadingStep(step::CreateStepAllocXBlocks());
 
-    // Start of the zone content
+        // Start of the zone content
-    zoneLoader->AddLoadingStep(step::CreateStepLoadZoneContent(
+        zoneLoader->AddLoadingStep(step::CreateStepLoadZoneContent(
-        [zonePtr](ZoneInputStream& stream)
+            [zonePtr](ZoneInputStream& stream)
-        {
+            {
-            return std::make_unique<ContentLoader>(*zonePtr, stream);
+                return std::make_unique<ContentLoader>(*zonePtr, stream);
-        },
+            },
-        32u,
+            32u,
-        ZoneConstants::OFFSET_BLOCK_BIT_COUNT,
+            ZoneConstants::OFFSET_BLOCK_BIT_COUNT,
-        ZoneConstants::INSERT_BLOCK,
+            ZoneConstants::INSERT_BLOCK,
-        zonePtr->Memory(),
+            zonePtr->Memory(),
-        std::move(progressCallback)));
+            std::move(progressCallback)));
+    }
+    else
+    {
+        fs::path dumpFileNamePath = fs::path(fileName).filename();
+        dumpFileNamePath.replace_extension(".dat");
+        std::string dumpFileName = dumpFileNamePath.string();
+        con::warn("Dumping xbox assets is not supported, making a full fastfile data dump to {}", dumpFileName);
+        zoneLoader->AddLoadingStep(step::CreateStepDumpData(dumpFileName, 0xFFFFFFFF));
+    }
 
     return zoneLoader;
 }

src/ZoneLoading/Loading/Steps/StepSkipZoneImageHeaders.cpp

@@ -0,0 +1,75 @@
+#include "StepSkipZoneImageHeaders.h"
+
+#include "Utils/Endianness.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <vector>
+
+namespace
+{
+    class StepSkipZoneImageHeaders final : public ILoadingStep
+    {
+    public:
+        void PerformStep(ZoneLoader& zoneLoader, ILoadingStream& stream) override
+        {
+
+            // Xbox fastfiles have additional header data before the auth header:
+            // - 4 bytes: language flags bitmask
+            // - 4 bytes: image count
+            // - sizeof(XAssetStreamFile) * image count (for each language in the bitmask)
+            // - 4 bytes: unknown
+            // - 4 bytes: unknown
+            // struct XAssetStreamFile // sizeof=0xC
+            // {
+            //     unsigned int fileIndex;
+            //     unsigned int offset;
+            //     unsigned int offsetEnd;
+            // };
+
+            uint32_t languageFlags;
+            stream.Load(&languageFlags, sizeof(languageFlags));
+            languageFlags = endianness::FromBigEndian(languageFlags);
+
+            uint32_t imageCount;
+            stream.Load(&imageCount, sizeof(imageCount));
+            imageCount = endianness::FromBigEndian(imageCount);
+
+            // Count how many languages are set in the bitmask
+            uint32_t languageCount = 0;
+            for (int i = 0; i < 15; i++)
+            {
+                if (languageFlags & (1 << i))
+                {
+                    languageCount++;
+                }
+            }
+
+            // Skip image stream file data (12 bytes per image per language)
+            const size_t imageDataSize = 12 * imageCount * languageCount;
+            if (imageDataSize > 0)
+            {
+                std::vector<uint8_t> tempBuffer(std::min(imageDataSize, size_t(8192)));
+                size_t skipped = 0;
+                while (skipped < imageDataSize)
+                {
+                    const size_t toSkip = std::min(imageDataSize - skipped, tempBuffer.size());
+                    stream.Load(tempBuffer.data(), toSkip);
+                    skipped += toSkip;
+                }
+            }
+
+            // Skip the final 8 bytes (2 unknown 4-byte values)
+            uint8_t finalBytes[8];
+            stream.Load(finalBytes, sizeof(finalBytes));
+        }
+    };
+} // namespace
+
+namespace step
+{
+    std::unique_ptr<ILoadingStep> CreateStepSkipZoneImageHeaders()
+    {
+        return std::make_unique<StepSkipZoneImageHeaders>();
+    }
+} // namespace step

src/ZoneLoading/Loading/Steps/StepSkipZoneImageHeaders.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "Loading/ILoadingStep.h"
+
+#include <memory>
+
+namespace step
+{
+    std::unique_ptr<ILoadingStep> CreateStepSkipZoneImageHeaders();
+}

src/ZoneWriting/Game/IW4/ZoneWriterFactoryIW4.cpp

@@ -39,7 +39,7 @@ namespace
     ZoneHeader CreateHeaderForParams(const bool isSecure, const bool isOfficial)
     {
         ZoneHeader header{};
-        header.m_version = ZoneConstants::ZONE_VERSION;
+        header.m_version = ZoneConstants::ZONE_VERSION_PC;
 
         if (isSecure)
         {