chore: fix annoying IW4 cross-type pointer reusage with dirty hack

The game reuses pointer across different types as long as bytes and size matches
This leads to non-pointer and pointer types being reused
To fix this loading code now handles block memory offsets by nulling
and block offsets to non-block data with pointing to raw block data
The behaviour only seems to realistically happen on nulled memory

src/ZoneCodeGeneratorLib/Generating/Templates/ZoneLoadTemplate.cpp

@@ -993,7 +993,7 @@ namespace
 
                     if (info && !info->m_has_matching_cross_platform_structure)
                     {
-                        LINEF("*{0} = m_stream.ConvertOffsetToPointerLookup(*{0});", MakeTypePtrVarName(def))
+                        LINEF("*{0} = m_stream.ConvertOffsetToPointerLookup(*{0}).Expect();", MakeTypePtrVarName(def))
                     }
                     else
                     {
@@ -1659,7 +1659,21 @@ namespace
 
                 if (ShouldAllocOutOfBlock(*member, loadType))
                 {
-                    LINEF("{0} = m_stream.ConvertOffsetToPointerLookup({0});", MakeMemberAccess(info, member, modifier))
+                    LINE_STARTF("{0} = m_stream.ConvertOffsetToPointerLookup({0})", MakeMemberAccess(info, member, modifier))
+                    if (loadType == MemberLoadType::POINTER_ARRAY)
+                    {
+                        LINE_MIDDLEF(".OrNulled({0}uz * ({1}), sizeof({2}{3}) * ({1}), m_memory)",
+                                     member->m_member->m_type_declaration->GetSize(),
+                                     MakeEvaluation(modifier.GetPointerArrayCountEvaluation()),
+                                     MakeTypeDecl(member->m_member->m_type_declaration.get()),
+                                     MakeFollowingReferences(modifier.GetFollowingDeclarationModifiers()))
+                    }
+                    else
+                    {
+                        LINE_MIDDLE(".Expect()")
+                    }
+
+                    LINE_END(";")
                 }
                 else
                 {

src/ZoneLoading/Loading/Exception/InvalidLookupPositionException.cpp

@@ -0,0 +1,19 @@
+#include "InvalidLookupPositionException.h"
+
+#include <format>
+
+InvalidLookupPositionException::InvalidLookupPositionException(block_t block, size_t offset)
+    : m_block(block),
+      m_offset(offset)
+{
+}
+
+std::string InvalidLookupPositionException::DetailedMessage()
+{
+    return std::format("Zone tried to lookup at block {}, offset {} that was not recorded", m_block, m_offset);
+}
+
+char const* InvalidLookupPositionException::what() const noexcept
+{
+    return "Zone tried to lookup at zone offset that is not recorded";
+}

src/ZoneLoading/Loading/Exception/InvalidLookupPositionException.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "LoadingException.h"
+#include "Zone/ZoneTypes.h"
+
+#include <cstdlib>
+
+class InvalidLookupPositionException final : public LoadingException
+{
+public:
+    InvalidLookupPositionException(block_t block, size_t offset);
+
+    std::string DetailedMessage() override;
+    char const* what() const noexcept override;
+
+private:
+    block_t m_block;
+    size_t m_offset;
+};

src/ZoneLoading/Zone/Stream/ZoneInputStream.cpp

@@ -349,7 +349,7 @@ namespace
             m_pointer_redirect_lookup.emplace(zonePtr, alias);
         }
 
-        void* ConvertOffsetToPointerLookup(const void* offset) override
+        MaybePointerFromLookup<void> ConvertOffsetToPointerLookup(const void* offset) override
         {
             // For details see ConvertOffsetToPointer
             const auto offsetInt = reinterpret_cast<uintptr_t>(offset) - 1u;
@@ -367,10 +367,9 @@ namespace
 
             const auto foundPointerLookup = m_pointer_redirect_lookup.find(offsetInt);
             if (foundPointerLookup != m_pointer_redirect_lookup.end())
-                return foundPointerLookup->second;
+                return MaybePointerFromLookup<void>(foundPointerLookup->second);
 
-            assert(false);
+            return MaybePointerFromLookup<void>(&block->m_buffer[blockOffset], blockNum, blockOffset);
-            return &block->m_buffer[blockOffset];
         }
 
         void* ConvertOffsetToAliasLookup(const void* offset) override

src/ZoneLoading/Zone/Stream/ZoneInputStream.h

@@ -1,11 +1,13 @@
 #pragma once
 
+#include "Loading/Exception/InvalidLookupPositionException.h"
 #include "Loading/ILoadingStream.h"
 #include "Utils/MemoryManager.h"
 #include "Zone/Stream/IZoneStream.h"
 #include "Zone/XBlock.h"
 
 #include <cassert>
+#include <cstdlib>
 #include <cstring>
 #include <memory>
 #include <type_traits>
@@ -52,6 +54,69 @@ private:
     size_t m_offset;
 };
 
+template<typename T> class MaybePointerFromLookup
+{
+public:
+    explicit MaybePointerFromLookup(void* ptr)
+        : m_valid(true),
+          m_ptr(ptr),
+          m_block(0),
+          m_offset(0)
+    {
+    }
+
+    MaybePointerFromLookup(void* ptr, const block_t block, const size_t offset)
+        : m_valid(false),
+          m_ptr(ptr),
+          m_block(block),
+          m_offset(offset)
+    {
+    }
+
+    explicit MaybePointerFromLookup(const MaybePointerFromLookup<void>& other)
+        : m_valid(other.m_valid),
+          m_ptr(static_cast<T*>(other.m_ptr)),
+          m_block(other.m_block),
+          m_offset(other.m_offset)
+    {
+    }
+
+    [[nodiscard]] T* Expect() const
+    {
+        if (!m_valid)
+            throw InvalidLookupPositionException(m_block, m_offset);
+
+        return static_cast<T*>(m_ptr);
+    }
+
+    /**
+     * The original linker does an annoying optimization where ConvertOffsetToPointer makes structs
+     * reuse data across non-matching types.
+     * E.g. a pointer array reuses memory of a scriptstring array.
+     * Since cross-platform the sizes of the types do not match anymore, this has to be handled differently.
+     * The scenario seems to realistically only happen when the data is nulled so just alloc a nulled memory block.
+     * If this strategy fails, in the future it might need to realloc and load existing block data with fill.
+     */
+    [[nodiscard]] T* OrNulled(const size_t gameSize, const size_t size, MemoryManager& memory) const
+    {
+        if (m_valid)
+            return static_cast<T*>(m_ptr);
+
+        auto* result = static_cast<T*>(memory.AllocRaw(size));
+
+        // We expect the original game buffer to also have been nulled
+        assert(gameSize < size);
+        assert(memcmp(result, m_ptr, gameSize) == 0);
+
+        return result;
+    }
+
+    bool m_valid;
+    void* m_ptr;
+    block_t m_block;
+    size_t m_offset;
+};
+
 class ZoneInputStream : public IZoneStream
 {
 public:
@@ -155,11 +220,11 @@ public:
      */
     virtual void AddPointerLookup(void* redirectTo, const void* redirectFrom) = 0;
 
-    virtual void* ConvertOffsetToPointerLookup(const void* offset) = 0;
+    virtual MaybePointerFromLookup<void> ConvertOffsetToPointerLookup(const void* offset) = 0;
 
-    template<typename T> T* ConvertOffsetToPointerLookup(T* offset)
+    template<typename T> MaybePointerFromLookup<T> ConvertOffsetToPointerLookup(T* offset)
     {
-        return static_cast<T*>(ConvertOffsetToPointerLookup(static_cast<const void*>(offset)));
+        return MaybePointerFromLookup<T>(ConvertOffsetToPointerLookup(static_cast<const void*>(offset)));
     }
 
     virtual void* ConvertOffsetToAliasLookup(const void* offset) = 0;