diff --git a/src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.cpp b/src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.cpp index 1c513697..5e0e32d6 100644 --- a/src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.cpp +++ b/src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.cpp @@ -96,7 +96,7 @@ uint32_t CommonStructuredDataStruct::CalculateChecksum(const CommonStructuredDat { const auto& enumedArray = def.m_enumed_arrays[currentType.m_info.type_index]; - if(enumedArray.m_enum_index < def.m_enums.size()) + if (enumedArray.m_enum_index < def.m_enums.size()) { const auto& _enum = *def.m_enums[enumedArray.m_enum_index]; checksum = crc32(checksum, reinterpret_cast(_enum.m_name.c_str()), _enum.m_name.size() + 1); @@ -116,10 +116,18 @@ uint32_t CommonStructuredDataStruct::CalculateChecksum(const CommonStructuredDat return checksum; } -void CommonStructuredDataStruct::SortProperties() +void CommonStructuredDataStruct::SortPropertiesByOffset() { std::sort(m_properties.begin(), m_properties.end(), [](const CommonStructuredDataStructProperty& e1, const CommonStructuredDataStructProperty& e2) { return e1.m_offset_in_bits < e2.m_offset_in_bits; }); } + +void CommonStructuredDataStruct::SortPropertiesByName() +{ + std::sort(m_properties.begin(), m_properties.end(), [](const CommonStructuredDataStructProperty& e1, const CommonStructuredDataStructProperty& e2) + { + return e1.m_name < e2.m_name; + }); +} diff --git a/src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.h b/src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.h index 8d93f18e..1f3cf770 100644 --- a/src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.h +++ b/src/ObjCommon/StructuredDataDef/CommonStructuredDataStruct.h @@ -30,5 +30,6 @@ struct CommonStructuredDataStruct _NODISCARD uint32_t CalculateChecksum(const CommonStructuredDataDef& def, uint32_t initialValue) const; - void SortProperties(); + void SortPropertiesByOffset(); + void SortPropertiesByName(); }; diff --git a/src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataDefScopeSequences.cpp b/src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataDefScopeSequences.cpp index fcda5ca3..abd90b4a 100644 --- a/src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataDefScopeSequences.cpp +++ b/src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataDefScopeSequences.cpp @@ -31,7 +31,9 @@ namespace sdd::def_scope_sequences { assert(state->m_current_def); - auto newEnum = std::make_unique(result.NextCapture(CAPTURE_NAME).IdentifierValue()); + const auto& nameToken = result.NextCapture(CAPTURE_NAME); + auto newEnum = std::make_unique(nameToken.IdentifierValue()); + const auto newEnumIndex = state->m_current_def->m_enums.size(); if (result.HasNextCapture(CAPTURE_RESERVED_COUNT)) { const auto& reservedCountToken = result.NextCapture(CAPTURE_RESERVED_COUNT); @@ -41,8 +43,18 @@ namespace sdd::def_scope_sequences throw ParsingException(reservedCountToken.GetPos(), "Reserved enum entry count must be greater than zero"); } + const auto existingType = state->m_def_types_by_name.find(newEnum->m_name); + if (existingType != state->m_def_types_by_name.end()) + { + if (existingType->second.m_category == CommonStructuredDataTypeCategory::UNKNOWN) + existingType->second = CommonStructuredDataType(CommonStructuredDataTypeCategory::ENUM, newEnumIndex); + else + throw ParsingException(nameToken.GetPos(), "Type with this name has already been defined"); + } + else + state->m_def_types_by_name.emplace(newEnum->m_name, CommonStructuredDataType(CommonStructuredDataTypeCategory::ENUM, newEnumIndex)); + state->m_current_enum = newEnum.get(); - state->m_def_types_by_name.emplace(newEnum->m_name, CommonStructuredDataType(CommonStructuredDataTypeCategory::ENUM, state->m_current_def->m_enums.size())); state->m_current_def->m_enums.emplace_back(std::move(newEnum)); } }; @@ -68,22 +80,33 @@ namespace sdd::def_scope_sequences { assert(state->m_current_def); - auto newStruct = std::make_unique(result.NextCapture(CAPTURE_NAME).IdentifierValue()); + const auto& nameToken = result.NextCapture(CAPTURE_NAME); + auto newStruct = std::make_unique(nameToken.IdentifierValue()); auto* newStructPtr = newStruct.get(); const auto newStructIndex = state->m_current_def->m_structs.size(); + const auto existingType = state->m_def_types_by_name.find(newStruct->m_name); + if (existingType != state->m_def_types_by_name.end()) + { + if (existingType->second.m_category == CommonStructuredDataTypeCategory::UNKNOWN) + existingType->second = CommonStructuredDataType(CommonStructuredDataTypeCategory::STRUCT, newStructIndex); + else + throw ParsingException(nameToken.GetPos(), "Type with this name has already been defined"); + } + else + state->m_def_types_by_name.emplace(newStruct->m_name, CommonStructuredDataType(CommonStructuredDataTypeCategory::STRUCT, newStructIndex)); + state->m_current_struct = newStructPtr; - state->m_def_types_by_name.emplace(newStruct->m_name, CommonStructuredDataType(CommonStructuredDataTypeCategory::STRUCT, newStructIndex)); state->m_current_def->m_structs.emplace_back(std::move(newStruct)); - state->m_current_struct_offset_in_bits = 0; + state->m_current_struct_padding_offset = 0; if (newStructPtr->m_name == "root") { - state->m_current_struct_offset_in_bits = 64u; + state->m_current_struct_padding_offset = 64u; state->m_current_def->m_root_type = CommonStructuredDataType(CommonStructuredDataTypeCategory::STRUCT, newStructIndex); } else - state->m_current_struct_offset_in_bits = 0; + state->m_current_struct_padding_offset = 0; } }; @@ -154,9 +177,11 @@ namespace sdd::def_scope_sequences assert(state->m_current_struct == nullptr); CreateDefaultStructWhenNoStructsSpecified(state); + // TODO: Replace all unknown type references + // TODO: Calculate struct sizes and property offsets SetDefSizeFromRootStruct(state); - if(!state->m_checksum_overriden) + if (!state->m_checksum_overriden) state->m_current_def->m_checksum = state->m_current_def->CalculateChecksum(); else state->m_current_def->m_checksum = state->m_checksum_override_value; @@ -175,7 +200,7 @@ using namespace sdd; using namespace def_scope_sequences; StructuredDataDefScopeSequences::StructuredDataDefScopeSequences(std::vector>& allSequences, - std::vector& scopeSequences) + std::vector& scopeSequences) : AbstractScopeSequenceHolder(allSequences, scopeSequences) { } diff --git a/src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataStructScopeSequences.cpp b/src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataStructScopeSequences.cpp index a3edd81a..9512dea9 100644 --- a/src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataStructScopeSequences.cpp +++ b/src/ObjLoading/StructuredDataDef/Parsing/Sequence/StructuredDataStructScopeSequences.cpp @@ -3,6 +3,7 @@ #include #include "Parsing/Simple/Matcher/SimpleMatcherFactory.h" +#include "Utils/Alignment.h" namespace sdd::struct_scope_sequences { @@ -65,69 +66,45 @@ namespace sdd::struct_scope_sequences } private: - static CommonStructuredDataType ProcessType(StructuredDataDefParserState* state, SequenceResult& result, size_t& currentSize, size_t& currentBitAlign) + static CommonStructuredDataType ProcessType(StructuredDataDefParserState* state, SequenceResult& result) { const auto typeTag = result.NextTag(); switch (typeTag) { case TAG_TYPE_INT: - currentSize = 32; - currentBitAlign = 8; return CommonStructuredDataType(CommonStructuredDataTypeCategory::INT); case TAG_TYPE_BYTE: - currentSize = 8; - currentBitAlign = 8; return CommonStructuredDataType(CommonStructuredDataTypeCategory::BYTE); case TAG_TYPE_BOOL: - currentSize = 1; - currentBitAlign = 0; return CommonStructuredDataType(CommonStructuredDataTypeCategory::BOOL); case TAG_TYPE_FLOAT: - currentSize = 32; - currentBitAlign = 8; return CommonStructuredDataType(CommonStructuredDataTypeCategory::FLOAT); case TAG_TYPE_SHORT: - currentSize = 16; - currentBitAlign = 8; return CommonStructuredDataType(CommonStructuredDataTypeCategory::SHORT); case TAG_TYPE_STRING: { - currentBitAlign = 8; const auto& stringLengthToken = result.NextCapture(CAPTURE_STRING_LENGTH); const auto stringLength = stringLengthToken.IntegerValue(); if (stringLength <= 0) throw ParsingException(stringLengthToken.GetPos(), "String length must be greater than zero"); - currentSize = stringLength * 8; return {CommonStructuredDataTypeCategory::STRING, static_cast(stringLength)}; } case TAG_TYPE_NAMED: { - currentBitAlign = 8; const auto& typeNameToken = result.NextCapture(CAPTURE_TYPE_NAME); const auto typeName = typeNameToken.IdentifierValue(); const auto existingType = state->m_def_types_by_name.find(typeName); if (existingType == state->m_def_types_by_name.end()) - throw ParsingException(typeNameToken.GetPos(), "No type defined under this name"); - - if (existingType->second.m_category == CommonStructuredDataTypeCategory::STRUCT) { - assert(existingType->second.m_info.type_index < state->m_current_def->m_structs.size()); - const auto* _struct = state->m_current_def->m_structs[existingType->second.m_info.type_index].get(); - currentSize = _struct->m_size_in_byte * 8; - } - else if (existingType->second.m_category == CommonStructuredDataTypeCategory::ENUM) - { - assert(existingType->second.m_info.type_index < state->m_current_def->m_enums.size()); - currentSize = 16; - } - else - { - assert(false); - currentSize = 0; + const auto undefinedTypeIndex = state->m_undefined_types.size(); + const CommonStructuredDataType undefinedType(CommonStructuredDataTypeCategory::UNKNOWN, undefinedTypeIndex); + state->m_undefined_types.emplace_back(typeName, typeNameToken.GetPos()); + state->m_def_types_by_name.emplace(std::make_pair(typeName, undefinedType)); + return undefinedType; } return existingType->second; @@ -137,11 +114,8 @@ namespace sdd::struct_scope_sequences } } - static CommonStructuredDataType ProcessArray(StructuredDataDefParserState* state, const SimpleParserValue& arrayToken, const CommonStructuredDataType currentType, - size_t& currentSize, size_t& currentBitAlign) + static CommonStructuredDataType ProcessArray(StructuredDataDefParserState* state, const SimpleParserValue& arrayToken, const CommonStructuredDataType currentType) { - currentBitAlign = 8; - if (arrayToken.m_type == SimpleParserValueType::INTEGER) { const auto arrayElementCount = arrayToken.IntegerValue(); @@ -149,8 +123,6 @@ namespace sdd::struct_scope_sequences if (arrayElementCount <= 0) throw ParsingException(arrayToken.GetPos(), "Array size must be greater than zero"); - currentSize *= arrayElementCount; - const CommonStructuredDataIndexedArray indexedArray(currentType, arrayElementCount); const auto existingIndexedArray = state->m_def_indexed_arrays.find(indexedArray); @@ -177,7 +149,6 @@ namespace sdd::struct_scope_sequences const auto enumElementCount = _enum->ElementCount(); assert(enumElementCount > 0); - currentSize *= enumElementCount; const CommonStructuredDataEnumedArray enumedArray(currentType, existingType->second.m_info.type_index); @@ -199,22 +170,16 @@ namespace sdd::struct_scope_sequences assert(state->m_current_def != nullptr); assert(state->m_current_struct != nullptr); - size_t currentSize = 0; - size_t currentAlign = 0; - auto currentType = ProcessType(state, result, currentSize, currentAlign); + auto currentType = ProcessType(state, result); std::vector> arrayTokens; while (result.HasNextCapture(CAPTURE_ARRAY_SIZE)) arrayTokens.emplace_back(result.NextCapture(CAPTURE_ARRAY_SIZE)); for (auto i = arrayTokens.rbegin(); i != arrayTokens.rend(); ++i) - currentType = ProcessArray(state, i->get(), currentType, currentSize, currentAlign); + currentType = ProcessArray(state, i->get(), currentType); - if (currentAlign > 0) - state->m_current_struct_offset_in_bits = (state->m_current_struct_offset_in_bits + currentAlign - 1) / currentAlign * currentAlign; - - state->m_current_struct->m_properties.emplace_back(result.NextCapture(CAPTURE_ENTRY_NAME).IdentifierValue(), currentType, state->m_current_struct_offset_in_bits); - state->m_current_struct_offset_in_bits += currentSize; + state->m_current_struct->m_properties.emplace_back(result.NextCapture(CAPTURE_ENTRY_NAME).IdentifierValue(), currentType, state->m_current_struct_padding_offset); } }; @@ -245,11 +210,7 @@ namespace sdd::struct_scope_sequences if (paddingValue <= 0) throw ParsingException(paddingValueToken.GetPos(), "Padding value must be greater than 0"); - // Align to next byte - state->m_current_struct_offset_in_bits = (state->m_current_struct_offset_in_bits + 7) / 8 * 8; - - // Add padding value to current size - state->m_current_struct_offset_in_bits += static_cast(paddingValue); + state->m_current_struct_padding_offset += paddingValue * 8; } }; @@ -271,20 +232,11 @@ namespace sdd::struct_scope_sequences { assert(state->m_current_struct != nullptr); - // Set the size of the finalized struct - if (!state->m_current_struct->m_properties.empty()) - state->m_current_struct->m_size_in_byte = (state->m_current_struct_offset_in_bits + 7) / 8; - else - state->m_current_struct->m_size_in_byte = 0u; - - state->m_current_struct_offset_in_bits = 0u; + state->m_current_struct->m_size_in_byte = utils::Align(state->m_current_struct_padding_offset, 8u) / 8; + state->m_current_struct_padding_offset = 0u; // Sort the entries of the struct alphabetically - std::sort(state->m_current_struct->m_properties.begin(), state->m_current_struct->m_properties.end(), - [](const CommonStructuredDataStructProperty& e1, const CommonStructuredDataStructProperty& e2) - { - return e1.m_name < e2.m_name; - }); + state->m_current_struct->SortPropertiesByName(); state->m_current_struct = nullptr; } }; diff --git a/src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.cpp b/src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.cpp index 272fed68..72c16d50 100644 --- a/src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.cpp +++ b/src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.cpp @@ -2,11 +2,20 @@ using namespace sdd; +UndefinedType::UndefinedType() += default; + +UndefinedType::UndefinedType(std::string name, const TokenPos firstUsagePos) + : m_name(std::move(name)), + m_first_usage_pos(firstUsagePos) +{ +} + StructuredDataDefParserState::StructuredDataDefParserState() : m_current_def(nullptr), m_current_enum(nullptr), m_current_struct(nullptr), - m_current_struct_offset_in_bits(0u), + m_current_struct_padding_offset(0u), m_checksum_override_value(0u), m_checksum_overriden(false) { diff --git a/src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.h b/src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.h index 71afb02e..2788775a 100644 --- a/src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.h +++ b/src/ObjLoading/StructuredDataDef/Parsing/StructuredDataDefParserState.h @@ -4,11 +4,24 @@ #include #include #include +#include +#include "Parsing/TokenPos.h" #include "StructuredDataDef/CommonStructuredDataDef.h" namespace sdd { + class UndefinedType + { + public: + std::string m_name; + TokenPos m_first_usage_pos; + CommonStructuredDataType m_mapped_type; + + UndefinedType(); + UndefinedType(std::string name, TokenPos firstUsagePos); + }; + class StructuredDataDefParserState { public: @@ -17,7 +30,8 @@ namespace sdd CommonStructuredDataDef* m_current_def; CommonStructuredDataEnum* m_current_enum; CommonStructuredDataStruct* m_current_struct; - size_t m_current_struct_offset_in_bits; + + size_t m_current_struct_padding_offset; size_t m_checksum_override_value; bool m_checksum_overriden; @@ -25,6 +39,7 @@ namespace sdd std::map m_def_types_by_name; std::map m_def_indexed_arrays; std::map m_def_enumed_arrays; + std::vector m_undefined_types; StructuredDataDefParserState(); }; diff --git a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperStructuredDataDefSet.cpp b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperStructuredDataDefSet.cpp index fca40a49..ebe7ea20 100644 --- a/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperStructuredDataDefSet.cpp +++ b/src/ObjWriting/Game/IW4/AssetDumpers/AssetDumperStructuredDataDefSet.cpp @@ -115,7 +115,7 @@ void AssetDumperStructuredDataDefSet::ConvertStruct(const CommonStructuredDataDe outProperty.m_offset_in_bits = inProperty.offset * 8; } - out->SortProperties(); + out->SortPropertiesByOffset(); } void AssetDumperStructuredDataDefSet::ConvertIndexedArray(const CommonStructuredDataDef* def, CommonStructuredDataIndexedArray* out, const StructuredDataIndexedArray* in)