Calculate StructuredDataDef sizes and offsets

src/ObjLoading/Game/IW4/AssetLoaders/AssetLoaderStructuredDataDefSet.cpp

@@ -80,7 +80,7 @@ void AssetLoaderStructuredDataDefSet::ConvertEnum(StructuredDataEnum* outputEnum
 
 void AssetLoaderStructuredDataDefSet::ConvertStruct(StructuredDataStruct* outputStruct, const CommonStructuredDataDefStruct* inputStruct, MemoryManager* memory)
 {
-    outputStruct->size = static_cast<int>(inputStruct->m_size);
+    outputStruct->size = static_cast<int>(inputStruct->m_size_in_byte);
     outputStruct->bitOffset = inputStruct->m_bit_offset;
 
     outputStruct->propertyCount = static_cast<int>(inputStruct->m_properties.size());
@@ -93,8 +93,15 @@ void AssetLoaderStructuredDataDefSet::ConvertStruct(StructuredDataStruct* output
             const auto& inputProperty = inputStruct->m_properties[propertyIndex];
 
             outputProperty.name = memory->Dup(inputProperty.m_name.c_str());
-            outputProperty.offset = inputProperty.m_offset;
             outputProperty.type = ConvertType(inputProperty.m_type);
+
+            if(outputProperty.type.type != DATA_BOOL)
+            {
+                assert(inputProperty.m_offset_in_bits % 8 == 0);
+                outputProperty.offset = inputProperty.m_offset_in_bits / 8;
+            }
+            else
+                outputProperty.offset = inputProperty.m_offset_in_bits;
         }
     }
     else
@@ -103,16 +110,16 @@ void AssetLoaderStructuredDataDefSet::ConvertStruct(StructuredDataStruct* output
 
 void AssetLoaderStructuredDataDefSet::ConvertIndexedArray(StructuredDataIndexedArray* outputIndexedArray, const CommonStructuredDataDefIndexedArray* inputIndexedArray, MemoryManager* memory)
 {
-    outputIndexedArray->arraySize = static_cast<int>(inputIndexedArray->m_array_size);
+    outputIndexedArray->arraySize = static_cast<int>(inputIndexedArray->m_element_count);
     outputIndexedArray->elementType = ConvertType(inputIndexedArray->m_array_type);
-    outputIndexedArray->elementSize = inputIndexedArray->m_element_size;
+    outputIndexedArray->elementSize = inputIndexedArray->m_element_size_in_bits;
 }
 
 void AssetLoaderStructuredDataDefSet::ConvertEnumedArray(StructuredDataEnumedArray* outputEnumedArray, const CommonStructuredDataDefEnumedArray* inputEnumedArray, MemoryManager* memory)
 {
     outputEnumedArray->enumIndex = static_cast<int>(inputEnumedArray->m_enum_index);
     outputEnumedArray->elementType = ConvertType(inputEnumedArray->m_array_type);
-    outputEnumedArray->elementSize = inputEnumedArray->m_element_size;
+    outputEnumedArray->elementSize = inputEnumedArray->m_element_size_in_bits;
 }
 
 void AssetLoaderStructuredDataDefSet::ConvertDef(StructuredDataDef* outputDef, const CommonStructuredDataDef* inputDef, MemoryManager* memory)
@@ -161,7 +168,7 @@ void AssetLoaderStructuredDataDefSet::ConvertDef(StructuredDataDef* outputDef, c
         outputDef->enumedArrays = nullptr;
 
     outputDef->rootType = ConvertType(inputDef->m_root_type);
-    outputDef->size = inputDef->m_size;
+    outputDef->size = inputDef->m_size_in_byte;
 }
 
 StructuredDataDefSet* AssetLoaderStructuredDataDefSet::ConvertSet(const std::string& assetName, const std::vector<std::unique_ptr<CommonStructuredDataDef>>& defs, MemoryManager* memory)

src/ObjLoading/Parsing/StructuredDataDef/Domain/CommonStructuredDataDef.cpp

@@ -8,6 +8,6 @@ CommonStructuredDataDef::CommonStructuredDataDef()
 CommonStructuredDataDef::CommonStructuredDataDef(const int version)
     : m_version(version),
       m_checksum(0u),
-      m_size(0u)
+      m_size_in_byte(0u)
 {
 }

src/ObjLoading/Parsing/StructuredDataDef/Domain/CommonStructuredDataDef.h

@@ -17,7 +17,7 @@ public:
     int m_version;
     size_t m_checksum;
     CommonStructuredDataDefType m_root_type;
-    size_t m_size;
+    size_t m_size_in_byte;
 
     CommonStructuredDataDef();
     explicit CommonStructuredDataDef(int version);

src/ObjLoading/Parsing/StructuredDataDef/Domain/CommonStructuredDataDefEnum.cpp

@@ -27,3 +27,8 @@ CommonStructuredDataDefEnum::CommonStructuredDataDefEnum(std::string name, const
       m_reserved_entry_count(reservedEntryCount)
 {
 }
+
+size_t CommonStructuredDataDefEnum::ElementCount() const
+{
+    return m_reserved_entry_count > 0 ? static_cast<size_t>(m_reserved_entry_count) : m_entries.size();
+}

src/ObjLoading/Parsing/StructuredDataDef/Domain/CommonStructuredDataDefEnum.h

@@ -1,7 +1,10 @@
 #pragma once
+
 #include <string>
 #include <vector>
 
+#include "Utils/ClassUtils.h"
+
 struct CommonStructuredDataDefEnumEntry
 {
     std::string m_name;
@@ -20,4 +23,6 @@ struct CommonStructuredDataDefEnum
     CommonStructuredDataDefEnum();
     explicit CommonStructuredDataDefEnum(std::string name);
     CommonStructuredDataDefEnum(std::string name, int reservedEntryCount);
+
+    _NODISCARD size_t ElementCount() const;
 };

src/ObjLoading/Parsing/StructuredDataDef/Domain/CommonStructuredDataDefStruct.cpp

@@ -1,32 +1,32 @@
 #include "CommonStructuredDataDefStruct.h"
 
 CommonStructuredDataDefStructEntry::CommonStructuredDataDefStructEntry()
-    : m_offset(0u)
+    : m_offset_in_bits(0u)
 {
 }
 
 CommonStructuredDataDefStructEntry::CommonStructuredDataDefStructEntry(std::string name)
     : m_name(std::move(name)),
-      m_offset(0u)
+      m_offset_in_bits(0u)
 {
 }
 
-CommonStructuredDataDefStructEntry::CommonStructuredDataDefStructEntry(std::string name, const CommonStructuredDataDefType type, const size_t offset)
+CommonStructuredDataDefStructEntry::CommonStructuredDataDefStructEntry(std::string name, const CommonStructuredDataDefType type, const size_t offsetInBits)
     : m_name(std::move(name)),
       m_type(type),
-      m_offset(offset)
+      m_offset_in_bits(offsetInBits)
 {
 }
 
 CommonStructuredDataDefStruct::CommonStructuredDataDefStruct()
     : m_bit_offset(0u),
-      m_size(0u)
+      m_size_in_byte(0u)
 {
 }
 
 CommonStructuredDataDefStruct::CommonStructuredDataDefStruct(std::string name)
     : m_name(std::move(name)),
       m_bit_offset(0u),
-      m_size(0u)
+      m_size_in_byte(0u)
 {
 }

src/ObjLoading/Parsing/StructuredDataDef/Domain/CommonStructuredDataDefStruct.h

@@ -9,11 +9,11 @@ struct CommonStructuredDataDefStructEntry
 {
     std::string m_name;
     CommonStructuredDataDefType m_type;
-    size_t m_offset;
+    size_t m_offset_in_bits;
 
     CommonStructuredDataDefStructEntry();
     explicit CommonStructuredDataDefStructEntry(std::string name);
-    CommonStructuredDataDefStructEntry(std::string name, CommonStructuredDataDefType type, size_t offset);
+    CommonStructuredDataDefStructEntry(std::string name, CommonStructuredDataDefType type, size_t offsetInBits);
 };
 
 struct CommonStructuredDataDefStruct
@@ -21,7 +21,7 @@ struct CommonStructuredDataDefStruct
     std::string m_name;
     std::vector<CommonStructuredDataDefStructEntry> m_properties;
     size_t m_bit_offset;
-    size_t m_size;
+    size_t m_size_in_byte;
 
     CommonStructuredDataDefStruct();
     explicit CommonStructuredDataDefStruct(std::string name);

src/ObjLoading/Parsing/StructuredDataDef/Domain/CommonStructuredDataDefTypes.cpp

@@ -43,15 +43,19 @@ bool operator>=(const CommonStructuredDataDefType& lhs, const CommonStructuredDa
 }
 
 CommonStructuredDataDefIndexedArray::CommonStructuredDataDefIndexedArray()
-    : m_array_size(0u),
+    : CommonStructuredDataDefIndexedArray({}, 0u)
-      m_element_size(0u)
 {
 }
 
-CommonStructuredDataDefIndexedArray::CommonStructuredDataDefIndexedArray(const CommonStructuredDataDefType type, const size_t arraySize)
+CommonStructuredDataDefIndexedArray::CommonStructuredDataDefIndexedArray(const CommonStructuredDataDefType type, const size_t elementCount)
+    : CommonStructuredDataDefIndexedArray(type, elementCount, 0u)
+{
+}
+
+CommonStructuredDataDefIndexedArray::CommonStructuredDataDefIndexedArray(const CommonStructuredDataDefType type, const size_t elementCount, const size_t elementSize)
     : m_array_type(type),
-      m_array_size(arraySize),
+      m_element_count(elementCount),
-      m_element_size(0u)
+      m_element_size_in_bits(elementSize)
 {
 }
 
@@ -61,7 +65,7 @@ bool operator<(const CommonStructuredDataDefIndexedArray& lhs, const CommonStruc
         return true;
     if (rhs.m_array_type < lhs.m_array_type)
         return false;
-    return lhs.m_array_size < rhs.m_array_size;
+    return lhs.m_element_count < rhs.m_element_count;
 }
 
 bool operator<=(const CommonStructuredDataDefIndexedArray& lhs, const CommonStructuredDataDefIndexedArray& rhs)
@@ -80,15 +84,20 @@ bool operator>=(const CommonStructuredDataDefIndexedArray& lhs, const CommonStru
 }
 
 CommonStructuredDataDefEnumedArray::CommonStructuredDataDefEnumedArray()
-    : m_enum_index(0u),
+    : CommonStructuredDataDefEnumedArray({}, 0u)
-      m_element_size(0u)
 {
 }
 
 CommonStructuredDataDefEnumedArray::CommonStructuredDataDefEnumedArray(const CommonStructuredDataDefType type, const size_t enumIndex)
+    : CommonStructuredDataDefEnumedArray(type, enumIndex, 0u, 0u)
+{
+}
+
+CommonStructuredDataDefEnumedArray::CommonStructuredDataDefEnumedArray(const CommonStructuredDataDefType type, const size_t enumIndex, const size_t elementCount, const size_t elementSizeInBits)
     : m_array_type(type),
       m_enum_index(enumIndex),
-      m_element_size(0u)
+      m_element_count(elementCount),
+      m_element_size_in_bits(elementSizeInBits)
 {
 }
 

src/ObjLoading/Parsing/StructuredDataDef/Domain/CommonStructuredDataDefTypes.h

@@ -41,11 +41,12 @@ struct CommonStructuredDataDefType
 struct CommonStructuredDataDefIndexedArray
 {
     CommonStructuredDataDefType m_array_type;
-    size_t m_array_size;
+    size_t m_element_count;
-    size_t m_element_size;
+    size_t m_element_size_in_bits;
 
     CommonStructuredDataDefIndexedArray();
-    CommonStructuredDataDefIndexedArray(CommonStructuredDataDefType type, size_t arraySize);
+    CommonStructuredDataDefIndexedArray(CommonStructuredDataDefType type, size_t elementCount);
+    CommonStructuredDataDefIndexedArray(CommonStructuredDataDefType type, size_t elementCount, size_t elementSize);
 
     friend bool operator<(const CommonStructuredDataDefIndexedArray& lhs, const CommonStructuredDataDefIndexedArray& rhs);
     friend bool operator<=(const CommonStructuredDataDefIndexedArray& lhs, const CommonStructuredDataDefIndexedArray& rhs);
@@ -57,10 +58,12 @@ struct CommonStructuredDataDefEnumedArray
 {
     CommonStructuredDataDefType m_array_type;
     size_t m_enum_index;
-    size_t m_element_size;
+    size_t m_element_count;
+    size_t m_element_size_in_bits;
 
     CommonStructuredDataDefEnumedArray();
     CommonStructuredDataDefEnumedArray(CommonStructuredDataDefType type, size_t enumIndex);
+    CommonStructuredDataDefEnumedArray(CommonStructuredDataDefType type, size_t enumIndex, size_t elementCount, size_t elementSizeInBits);
 
     friend bool operator<(const CommonStructuredDataDefEnumedArray& lhs, const CommonStructuredDataDefEnumedArray& rhs);
     friend bool operator<=(const CommonStructuredDataDefEnumedArray& lhs, const CommonStructuredDataDefEnumedArray& rhs);

src/ObjLoading/Parsing/StructuredDataDef/Sequence/StructuredDataDefDefScopeSequences.cpp

@@ -75,9 +75,15 @@ namespace sdd::def_scope_sequences
             state->m_current_struct = newStructPtr;
             state->m_def_types_by_name.emplace(newStruct->m_name, CommonStructuredDataDefType(CommonStructuredDataDefTypeCategory::STRUCT, newStructIndex));
             state->m_current_def->m_structs.emplace_back(std::move(newStruct));
+            state->m_current_struct_offset_in_bits = 0;
 
             if(newStructPtr->m_name == "root")
+            {
+                state->m_current_struct_offset_in_bits = 64u;
                 state->m_current_def->m_root_type = CommonStructuredDataDefType(CommonStructuredDataDefTypeCategory::STRUCT, newStructIndex);
+            }
+            else
+                state->m_current_struct_offset_in_bits = 0;
         }
     };
 
@@ -101,6 +107,13 @@ namespace sdd::def_scope_sequences
             assert(state->m_current_enum == nullptr);
             assert(state->m_current_struct == nullptr);
 
+            if(state->m_current_def->m_root_type.m_category == CommonStructuredDataDefTypeCategory::STRUCT
+                && state->m_current_def->m_root_type.m_info.type_index < state->m_current_def->m_structs.size())
+            {
+                const auto* _struct = state->m_current_def->m_structs[state->m_current_def->m_root_type.m_info.type_index].get();
+                state->m_current_def->m_size_in_byte = _struct->m_size_in_byte;
+            }
+
             state->m_current_def = nullptr;
             state->m_def_types_by_name.clear();
             state->m_def_indexed_arrays.clear();

src/ObjLoading/Parsing/StructuredDataDef/Sequence/StructuredDataDefStructScopeSequences.cpp

@@ -69,34 +69,47 @@ namespace sdd::struct_scope_sequences
         }
 
     private:
-        static CommonStructuredDataDefType ProcessType(StructuredDataDefParserState* state, SequenceResult<SimpleParserValue>& result)
+        static CommonStructuredDataDefType ProcessType(StructuredDataDefParserState* state, SequenceResult<SimpleParserValue>& result, size_t& currentSize, size_t& currentBitAlign)
         {
             const auto typeTag = result.NextTag();
 
             switch (typeTag)
             {
             case TAG_TYPE_INT:
+                currentSize = 32;
+                currentBitAlign = 8;
                 return CommonStructuredDataDefType(CommonStructuredDataDefTypeCategory::INT);
             case TAG_TYPE_BYTE:
+                currentSize = 8;
+                currentBitAlign = 8;
                 return CommonStructuredDataDefType(CommonStructuredDataDefTypeCategory::BYTE);
             case TAG_TYPE_BOOL:
+                currentSize = 1;
+                currentBitAlign = 0;
                 return CommonStructuredDataDefType(CommonStructuredDataDefTypeCategory::BOOL);
             case TAG_TYPE_FLOAT:
+                currentSize = 32;
+                currentBitAlign = 8;
                 return CommonStructuredDataDefType(CommonStructuredDataDefTypeCategory::FLOAT);
             case TAG_TYPE_SHORT:
+                currentSize = 16;
+                currentBitAlign = 8;
                 return CommonStructuredDataDefType(CommonStructuredDataDefTypeCategory::SHORT);
             case TAG_TYPE_STRING:
                 {
+                    currentBitAlign = 8;
                     const auto& stringLengthToken = result.NextCapture(CAPTURE_STRING_LENGTH);
                     const auto stringLength = stringLengthToken.IntegerValue();
 
                     if (stringLength <= 0)
                         throw ParsingException(stringLengthToken.GetPos(), "String length must be greater than zero");
 
+                    currentSize = stringLength * 8;
                     return {CommonStructuredDataDefTypeCategory::STRING, static_cast<size_t>(stringLength)};
                 }
             case TAG_TYPE_NAMED:
                 {
+                    currentBitAlign = 8;
                     const auto& typeNameToken = result.NextCapture(CAPTURE_TYPE_NAME);
                     const auto typeName = typeNameToken.IdentifierValue();
 
@@ -104,6 +117,23 @@ namespace sdd::struct_scope_sequences
                     if (existingType == state->m_def_types_by_name.end())
                         throw ParsingException(typeNameToken.GetPos(), "No type defined under this name");
 
+                    if (existingType->second.m_category == CommonStructuredDataDefTypeCategory::STRUCT)
+                    {
+                        assert(existingType->second.m_info.type_index < state->m_current_def->m_structs.size());
+                        const auto* _struct = state->m_current_def->m_structs[existingType->second.m_info.type_index].get();
+                        currentSize = _struct->m_size_in_byte * 8;
+                    }
+                    else if (existingType->second.m_category == CommonStructuredDataDefTypeCategory::ENUM)
+                    {
+                        assert(existingType->second.m_info.type_index < state->m_current_def->m_enums.size());
+                        currentSize = 16;
+                    }
+                    else
+                    {
+                        assert(false);
+                        currentSize = 0;
+                    }
+
                     return existingType->second;
                 }
             default:
@@ -111,19 +141,23 @@ namespace sdd::struct_scope_sequences
             }
         }
 
-        static CommonStructuredDataDefType ProcessArray(StructuredDataDefParserState* state, SequenceResult<SimpleParserValue>& result, CommonStructuredDataDefType currentType)
+        static CommonStructuredDataDefType ProcessArray(StructuredDataDefParserState* state, SequenceResult<SimpleParserValue>& result, const CommonStructuredDataDefType currentType,
+                                                        size_t& currentSize, size_t& currentBitAlign)
         {
+            currentBitAlign = 8;
             const auto arrayTag = result.NextTag();
 
             if (arrayTag == TAG_ARRAY_INDEX)
             {
                 const auto& arraySizeToken = result.NextCapture(CAPTURE_ARRAY_SIZE);
-                const auto arraySize = arraySizeToken.IntegerValue();
+                const auto arrayElementCount = arraySizeToken.IntegerValue();
 
-                if (arraySize <= 0)
+                if (arrayElementCount <= 0)
                     throw ParsingException(arraySizeToken.GetPos(), "Array size must be greater than zero");
 
-                const CommonStructuredDataDefIndexedArray indexedArray(currentType, arraySize);
+                currentSize *= arrayElementCount;
+
+                const CommonStructuredDataDefIndexedArray indexedArray(currentType, arrayElementCount);
 
                 const auto existingIndexedArray = state->m_def_indexed_arrays.find(indexedArray);
                 if (existingIndexedArray != state->m_def_indexed_arrays.end())
@@ -145,6 +179,13 @@ namespace sdd::struct_scope_sequences
                 if (existingType->second.m_category != CommonStructuredDataDefTypeCategory::ENUM)
                     throw ParsingException(enumNameToken.GetPos(), "Type for enumed array must be an enum");
 
+                assert(existingType->second.m_info.type_index < state->m_current_def->m_enums.size());
+                const auto* _enum = state->m_current_def->m_enums[existingType->second.m_info.type_index].get();
+
+                const auto enumElementCount = _enum->ElementCount();
+                assert(enumElementCount > 0);
+                currentSize *= enumElementCount;
+
                 const CommonStructuredDataDefEnumedArray enumedArray(currentType, existingType->second.m_info.type_index);
 
                 const auto existingEnumedArray = state->m_def_enumed_arrays.find(enumedArray);
@@ -165,13 +206,18 @@ namespace sdd::struct_scope_sequences
             assert(state->m_current_def != nullptr);
             assert(state->m_current_struct != nullptr);
 
-            auto currentType = ProcessType(state, result);
+            size_t currentSize = 0;
+            size_t currentAlign = 0;
+            auto currentType = ProcessType(state, result, currentSize, currentAlign);
 
             while (result.PeekAndRemoveIfTag(TAG_ARRAY) == TAG_ARRAY)
-                currentType = ProcessArray(state, result, currentType);
+                currentType = ProcessArray(state, result, currentType, currentSize, currentAlign);
 
-            // TODO: Calculate offset
+            if (currentAlign > 0)
-            state->m_current_struct->m_properties.emplace_back(result.NextCapture(CAPTURE_ENTRY_NAME).IdentifierValue(), currentType, 0);
+                state->m_current_struct_offset_in_bits = (state->m_current_struct_offset_in_bits + currentAlign - 1) / currentAlign * currentAlign;
+
+            state->m_current_struct->m_properties.emplace_back(result.NextCapture(CAPTURE_ENTRY_NAME).IdentifierValue(), currentType, state->m_current_struct_offset_in_bits);
+            state->m_current_struct_offset_in_bits += currentSize;
         }
     };
 
@@ -193,6 +239,9 @@ namespace sdd::struct_scope_sequences
         {
             assert(state->m_current_struct != nullptr);
 
+            state->m_current_struct->m_size_in_byte = (state->m_current_struct_offset_in_bits + 7) / 8;
+            state->m_current_struct_offset_in_bits = 0u;
+
             std::sort(state->m_current_struct->m_properties.begin(), state->m_current_struct->m_properties.end(),
                       [](const CommonStructuredDataDefStructEntry& e1, const CommonStructuredDataDefStructEntry& e2)
                       {

src/ObjLoading/Parsing/StructuredDataDef/StructuredDataDefParserState.cpp

@@ -5,6 +5,7 @@ using namespace sdd;
 StructuredDataDefParserState::StructuredDataDefParserState()
     : m_current_def(nullptr),
       m_current_enum(nullptr),
-      m_current_struct(nullptr)
+      m_current_struct(nullptr),
+      m_current_struct_offset_in_bits(0u)
 {
 }

src/ObjLoading/Parsing/StructuredDataDef/StructuredDataDefParserState.h

@@ -1,7 +1,9 @@
 #pragma once
+
 #include <map>
 #include <memory>
 #include <vector>
+#include <cstddef>
 
 #include "Domain/CommonStructuredDataDef.h"
 
@@ -15,6 +17,7 @@ namespace sdd
         CommonStructuredDataDef* m_current_def;
         CommonStructuredDataDefEnum* m_current_enum;
         CommonStructuredDataDefStruct* m_current_struct;
+        size_t m_current_struct_offset_in_bits;
 
         std::map<std::string, CommonStructuredDataDefType> m_def_types_by_name;
         std::map<CommonStructuredDataDefIndexedArray, size_t> m_def_indexed_arrays;