When parsing StructuredDataDefs accept undefined types

src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.cpp

@@ -96,7 +96,7 @@ uint32_t CommonStructuredDataStruct::CalculateChecksum(const CommonStructuredDat
                 {
                     const auto& enumedArray = def.m_enumed_arrays[currentType.m_info.type_index];
 
-                    if(enumedArray.m_enum_index < def.m_enums.size())
+                    if (enumedArray.m_enum_index < def.m_enums.size())
                     {
                         const auto& _enum = *def.m_enums[enumedArray.m_enum_index];
                         checksum = crc32(checksum, reinterpret_cast<const Bytef*>(_enum.m_name.c_str()), _enum.m_name.size() + 1);
@@ -116,10 +116,18 @@ uint32_t CommonStructuredDataStruct::CalculateChecksum(const CommonStructuredDat
     return checksum;
 }
 
-void CommonStructuredDataStruct::SortProperties()
+void CommonStructuredDataStruct::SortPropertiesByOffset()
 {
     std::sort(m_properties.begin(), m_properties.end(), [](const CommonStructuredDataStructProperty& e1, const CommonStructuredDataStructProperty& e2)
     {
         return e1.m_offset_in_bits < e2.m_offset_in_bits;
     });
 }
+
+void CommonStructuredDataStruct::SortPropertiesByName()
+{
+    std::sort(m_properties.begin(), m_properties.end(), [](const CommonStructuredDataStructProperty& e1, const CommonStructuredDataStructProperty& e2)
+    {
+        return e1.m_name < e2.m_name;
+    });
+}

src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.h

@@ -30,5 +30,6 @@ struct CommonStructuredDataStruct
 
     _NODISCARD uint32_t CalculateChecksum(const CommonStructuredDataDef& def, uint32_t initialValue) const;
 
-    void SortProperties();
+    void SortPropertiesByOffset();
+    void SortPropertiesByName();
 };

src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataDefScopeSequences.cpp

@@ -31,7 +31,9 @@ namespace sdd::def_scope_sequences
         {
             assert(state->m_current_def);
 
-            auto newEnum = std::make_unique<CommonStructuredDataEnum>(result.NextCapture(CAPTURE_NAME).IdentifierValue());
+            const auto& nameToken = result.NextCapture(CAPTURE_NAME);
+            auto newEnum = std::make_unique<CommonStructuredDataEnum>(nameToken.IdentifierValue());
+            const auto newEnumIndex = state->m_current_def->m_enums.size();
             if (result.HasNextCapture(CAPTURE_RESERVED_COUNT))
             {
                 const auto& reservedCountToken = result.NextCapture(CAPTURE_RESERVED_COUNT);
@@ -41,8 +43,18 @@ namespace sdd::def_scope_sequences
                     throw ParsingException(reservedCountToken.GetPos(), "Reserved enum entry count must be greater than zero");
             }
 
+            const auto existingType = state->m_def_types_by_name.find(newEnum->m_name);
+            if (existingType != state->m_def_types_by_name.end())
+            {
+                if (existingType->second.m_category == CommonStructuredDataTypeCategory::UNKNOWN)
+                    existingType->second = CommonStructuredDataType(CommonStructuredDataTypeCategory::ENUM, newEnumIndex);
+                else
+                    throw ParsingException(nameToken.GetPos(), "Type with this name has already been defined");
+            }
+            else
+                state->m_def_types_by_name.emplace(newEnum->m_name, CommonStructuredDataType(CommonStructuredDataTypeCategory::ENUM, newEnumIndex));
+
             state->m_current_enum = newEnum.get();
-            state->m_def_types_by_name.emplace(newEnum->m_name, CommonStructuredDataType(CommonStructuredDataTypeCategory::ENUM, state->m_current_def->m_enums.size()));
             state->m_current_def->m_enums.emplace_back(std::move(newEnum));
         }
     };
@@ -68,22 +80,33 @@ namespace sdd::def_scope_sequences
         {
             assert(state->m_current_def);
 
-            auto newStruct = std::make_unique<CommonStructuredDataStruct>(result.NextCapture(CAPTURE_NAME).IdentifierValue());
+            const auto& nameToken = result.NextCapture(CAPTURE_NAME);
+            auto newStruct = std::make_unique<CommonStructuredDataStruct>(nameToken.IdentifierValue());
             auto* newStructPtr = newStruct.get();
             const auto newStructIndex = state->m_current_def->m_structs.size();
 
+            const auto existingType = state->m_def_types_by_name.find(newStruct->m_name);
+            if (existingType != state->m_def_types_by_name.end())
+            {
+                if (existingType->second.m_category == CommonStructuredDataTypeCategory::UNKNOWN)
+                    existingType->second = CommonStructuredDataType(CommonStructuredDataTypeCategory::STRUCT, newStructIndex);
+                else
+                    throw ParsingException(nameToken.GetPos(), "Type with this name has already been defined");
+            }
+            else
+                state->m_def_types_by_name.emplace(newStruct->m_name, CommonStructuredDataType(CommonStructuredDataTypeCategory::STRUCT, newStructIndex));
+
             state->m_current_struct = newStructPtr;
-            state->m_def_types_by_name.emplace(newStruct->m_name, CommonStructuredDataType(CommonStructuredDataTypeCategory::STRUCT, newStructIndex));
             state->m_current_def->m_structs.emplace_back(std::move(newStruct));
-            state->m_current_struct_offset_in_bits = 0;
+            state->m_current_struct_padding_offset = 0;
 
             if (newStructPtr->m_name == "root")
             {
-                state->m_current_struct_offset_in_bits = 64u;
+                state->m_current_struct_padding_offset = 64u;
                 state->m_current_def->m_root_type = CommonStructuredDataType(CommonStructuredDataTypeCategory::STRUCT, newStructIndex);
             }
             else
-                state->m_current_struct_offset_in_bits = 0;
+                state->m_current_struct_padding_offset = 0;
         }
     };
 
@@ -154,9 +177,11 @@ namespace sdd::def_scope_sequences
             assert(state->m_current_struct == nullptr);
 
             CreateDefaultStructWhenNoStructsSpecified(state);
+            // TODO: Replace all unknown type references
+            // TODO: Calculate struct sizes and property offsets
             SetDefSizeFromRootStruct(state);
 
-            if(!state->m_checksum_overriden)
+            if (!state->m_checksum_overriden)
                 state->m_current_def->m_checksum = state->m_current_def->CalculateChecksum();
             else
                 state->m_current_def->m_checksum = state->m_checksum_override_value;
@@ -175,7 +200,7 @@ using namespace sdd;
 using namespace def_scope_sequences;
 
 StructuredDataDefScopeSequences::StructuredDataDefScopeSequences(std::vector<std::unique_ptr<StructuredDataDefParser::sequence_t>>& allSequences,
-                                                                       std::vector<StructuredDataDefParser::sequence_t*>& scopeSequences)
+                                                                 std::vector<StructuredDataDefParser::sequence_t*>& scopeSequences)
     : AbstractScopeSequenceHolder(allSequences, scopeSequences)
 {
 }

src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataStructScopeSequences.cpp

@@ -3,6 +3,7 @@
 #include <algorithm>
 
 #include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+#include "Utils/Alignment.h"
 
 namespace sdd::struct_scope_sequences
 {
@@ -65,69 +66,45 @@ namespace sdd::struct_scope_sequences
         }
 
     private:
-        static CommonStructuredDataType ProcessType(StructuredDataDefParserState* state, SequenceResult<SimpleParserValue>& result, size_t& currentSize, size_t& currentBitAlign)
+        static CommonStructuredDataType ProcessType(StructuredDataDefParserState* state, SequenceResult<SimpleParserValue>& result)
         {
             const auto typeTag = result.NextTag();
 
             switch (typeTag)
             {
             case TAG_TYPE_INT:
-                currentSize = 32;
-                currentBitAlign = 8;
                 return CommonStructuredDataType(CommonStructuredDataTypeCategory::INT);
             case TAG_TYPE_BYTE:
-                currentSize = 8;
-                currentBitAlign = 8;
                 return CommonStructuredDataType(CommonStructuredDataTypeCategory::BYTE);
             case TAG_TYPE_BOOL:
-                currentSize = 1;
-                currentBitAlign = 0;
                 return CommonStructuredDataType(CommonStructuredDataTypeCategory::BOOL);
             case TAG_TYPE_FLOAT:
-                currentSize = 32;
-                currentBitAlign = 8;
                 return CommonStructuredDataType(CommonStructuredDataTypeCategory::FLOAT);
             case TAG_TYPE_SHORT:
-                currentSize = 16;
-                currentBitAlign = 8;
                 return CommonStructuredDataType(CommonStructuredDataTypeCategory::SHORT);
             case TAG_TYPE_STRING:
                 {
-                    currentBitAlign = 8;
                     const auto& stringLengthToken = result.NextCapture(CAPTURE_STRING_LENGTH);
                     const auto stringLength = stringLengthToken.IntegerValue();
 
                     if (stringLength <= 0)
                         throw ParsingException(stringLengthToken.GetPos(), "String length must be greater than zero");
 
-                    currentSize = stringLength * 8;
                     return {CommonStructuredDataTypeCategory::STRING, static_cast<size_t>(stringLength)};
                 }
             case TAG_TYPE_NAMED:
                 {
-                    currentBitAlign = 8;
                     const auto& typeNameToken = result.NextCapture(CAPTURE_TYPE_NAME);
                     const auto typeName = typeNameToken.IdentifierValue();
 
                     const auto existingType = state->m_def_types_by_name.find(typeName);
                     if (existingType == state->m_def_types_by_name.end())
-                        throw ParsingException(typeNameToken.GetPos(), "No type defined under this name");
-
-                    if (existingType->second.m_category == CommonStructuredDataTypeCategory::STRUCT)
                     {
-                        assert(existingType->second.m_info.type_index < state->m_current_def->m_structs.size());
+                        const auto undefinedTypeIndex = state->m_undefined_types.size();
-                        const auto* _struct = state->m_current_def->m_structs[existingType->second.m_info.type_index].get();
+                        const CommonStructuredDataType undefinedType(CommonStructuredDataTypeCategory::UNKNOWN, undefinedTypeIndex);
-                        currentSize = _struct->m_size_in_byte * 8;
+                        state->m_undefined_types.emplace_back(typeName, typeNameToken.GetPos());
-                    }
+                        state->m_def_types_by_name.emplace(std::make_pair(typeName, undefinedType));
-                    else if (existingType->second.m_category == CommonStructuredDataTypeCategory::ENUM)
+                        return undefinedType;
-                    {
-                        assert(existingType->second.m_info.type_index < state->m_current_def->m_enums.size());
-                        currentSize = 16;
-                    }
-                    else
-                    {
-                        assert(false);
-                        currentSize = 0;
                     }
 
                     return existingType->second;
@@ -137,11 +114,8 @@ namespace sdd::struct_scope_sequences
             }
         }
 
-        static CommonStructuredDataType ProcessArray(StructuredDataDefParserState* state, const SimpleParserValue& arrayToken, const CommonStructuredDataType currentType,
+        static CommonStructuredDataType ProcessArray(StructuredDataDefParserState* state, const SimpleParserValue& arrayToken, const CommonStructuredDataType currentType)
-                                                     size_t& currentSize, size_t& currentBitAlign)
         {
-            currentBitAlign = 8;
-
             if (arrayToken.m_type == SimpleParserValueType::INTEGER)
             {
                 const auto arrayElementCount = arrayToken.IntegerValue();
@@ -149,8 +123,6 @@ namespace sdd::struct_scope_sequences
                 if (arrayElementCount <= 0)
                     throw ParsingException(arrayToken.GetPos(), "Array size must be greater than zero");
 
-                currentSize *= arrayElementCount;
-
                 const CommonStructuredDataIndexedArray indexedArray(currentType, arrayElementCount);
 
                 const auto existingIndexedArray = state->m_def_indexed_arrays.find(indexedArray);
@@ -177,7 +149,6 @@ namespace sdd::struct_scope_sequences
 
                 const auto enumElementCount = _enum->ElementCount();
                 assert(enumElementCount > 0);
-                currentSize *= enumElementCount;
 
                 const CommonStructuredDataEnumedArray enumedArray(currentType, existingType->second.m_info.type_index);
 
@@ -199,22 +170,16 @@ namespace sdd::struct_scope_sequences
             assert(state->m_current_def != nullptr);
             assert(state->m_current_struct != nullptr);
 
-            size_t currentSize = 0;
+            auto currentType = ProcessType(state, result);
-            size_t currentAlign = 0;
-            auto currentType = ProcessType(state, result, currentSize, currentAlign);
 
             std::vector<std::reference_wrapper<const SimpleParserValue>> arrayTokens;
             while (result.HasNextCapture(CAPTURE_ARRAY_SIZE))
                 arrayTokens.emplace_back(result.NextCapture(CAPTURE_ARRAY_SIZE));
 
             for (auto i = arrayTokens.rbegin(); i != arrayTokens.rend(); ++i)
-                currentType = ProcessArray(state, i->get(), currentType, currentSize, currentAlign);
+                currentType = ProcessArray(state, i->get(), currentType);
 
-            if (currentAlign > 0)
+            state->m_current_struct->m_properties.emplace_back(result.NextCapture(CAPTURE_ENTRY_NAME).IdentifierValue(), currentType, state->m_current_struct_padding_offset);
-                state->m_current_struct_offset_in_bits = (state->m_current_struct_offset_in_bits + currentAlign - 1) / currentAlign * currentAlign;
-
-            state->m_current_struct->m_properties.emplace_back(result.NextCapture(CAPTURE_ENTRY_NAME).IdentifierValue(), currentType, state->m_current_struct_offset_in_bits);
-            state->m_current_struct_offset_in_bits += currentSize;
         }
     };
 
@@ -245,11 +210,7 @@ namespace sdd::struct_scope_sequences
             if (paddingValue <= 0)
                 throw ParsingException(paddingValueToken.GetPos(), "Padding value must be greater than 0");
 
-            // Align to next byte
+            state->m_current_struct_padding_offset += paddingValue * 8;
-            state->m_current_struct_offset_in_bits = (state->m_current_struct_offset_in_bits + 7) / 8 * 8;
-
-            // Add padding value to current size
-            state->m_current_struct_offset_in_bits += static_cast<size_t>(paddingValue);
         }
     };
 
@@ -271,20 +232,11 @@ namespace sdd::struct_scope_sequences
         {
             assert(state->m_current_struct != nullptr);
 
-            // Set the size of the finalized struct
+            state->m_current_struct->m_size_in_byte = utils::Align(state->m_current_struct_padding_offset, 8u) / 8;
-            if (!state->m_current_struct->m_properties.empty())
+            state->m_current_struct_padding_offset = 0u;
-                state->m_current_struct->m_size_in_byte = (state->m_current_struct_offset_in_bits + 7) / 8;
-            else
-                state->m_current_struct->m_size_in_byte = 0u;
-
-            state->m_current_struct_offset_in_bits = 0u;
 
             // Sort the entries of the struct alphabetically
-            std::sort(state->m_current_struct->m_properties.begin(), state->m_current_struct->m_properties.end(),
+            state->m_current_struct->SortPropertiesByName();
-                      [](const CommonStructuredDataStructProperty& e1, const CommonStructuredDataStructProperty& e2)
-                      {
-                          return e1.m_name < e2.m_name;
-                      });
             state->m_current_struct = nullptr;
         }
     };

src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.cpp

@@ -2,11 +2,20 @@
 
 using namespace sdd;
 
+UndefinedType::UndefinedType()
+= default;
+
+UndefinedType::UndefinedType(std::string name, const TokenPos firstUsagePos)
+    : m_name(std::move(name)),
+      m_first_usage_pos(firstUsagePos)
+{
+}
+
 StructuredDataDefParserState::StructuredDataDefParserState()
     : m_current_def(nullptr),
       m_current_enum(nullptr),
       m_current_struct(nullptr),
-      m_current_struct_offset_in_bits(0u),
+      m_current_struct_padding_offset(0u),
       m_checksum_override_value(0u),
       m_checksum_overriden(false)
 {

src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.h

@@ -4,11 +4,24 @@
 #include <memory>
 #include <vector>
 #include <cstddef>
+#include <string>
 
+#include "Parsing/TokenPos.h"
 #include "StructuredDataDef/CommonStructuredDataDef.h"
 
 namespace sdd
 {
+    class UndefinedType
+    {
+    public:
+        std::string m_name;
+        TokenPos m_first_usage_pos;
+        CommonStructuredDataType m_mapped_type;
+
+        UndefinedType();
+        UndefinedType(std::string name, TokenPos firstUsagePos);
+    };
+
     class StructuredDataDefParserState
     {
     public:
@@ -17,7 +30,8 @@ namespace sdd
         CommonStructuredDataDef* m_current_def;
         CommonStructuredDataEnum* m_current_enum;
         CommonStructuredDataStruct* m_current_struct;
-        size_t m_current_struct_offset_in_bits;
+
+        size_t m_current_struct_padding_offset;
 
         size_t m_checksum_override_value;
         bool m_checksum_overriden;
@@ -25,6 +39,7 @@ namespace sdd
         std::map<std::string, CommonStructuredDataType> m_def_types_by_name;
         std::map<CommonStructuredDataIndexedArray, size_t> m_def_indexed_arrays;
         std::map<CommonStructuredDataEnumedArray, size_t> m_def_enumed_arrays;
+        std::vector<UndefinedType> m_undefined_types;
 
         StructuredDataDefParserState();
     };

src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperStructuredDataDefSet.cpp

@@ -115,7 +115,7 @@ void AssetDumperStructuredDataDefSet::ConvertStruct(const CommonStructuredDataDe
             outProperty.m_offset_in_bits = inProperty.offset * 8;
     }
 
-    out->SortProperties();
+    out->SortPropertiesByOffset();
 }
 
 void AssetDumperStructuredDataDefSet::ConvertIndexedArray(const CommonStructuredDataDef* def, CommonStructuredDataIndexedArray* out, const StructuredDataIndexedArray* in)