diff --git a/src/Common/Game/T6/CommonT6.cpp b/src/Common/Game/T6/CommonT6.cpp index 3d181616..d2ea3577 100644 --- a/src/Common/Game/T6/CommonT6.cpp +++ b/src/Common/Game/T6/CommonT6.cpp @@ -58,6 +58,16 @@ int Common::Com_HashString(const char* str, const int len) return result; } +uint32_t Common::R_HashString(const char* str, uint32_t hash) +{ + for (const auto* pos = str; *pos; pos++) + { + hash = 33 * hash ^ (*pos | 0x20); + } + + return hash; +} + PackedTexCoords Common::Vec2PackTexCoords(const vec2_t* in) { return PackedTexCoords{ Pack32::Vec2PackTexCoords(in->v) }; diff --git a/src/Common/Game/T6/CommonT6.h b/src/Common/Game/T6/CommonT6.h index ad8f38c0..2c63c62e 100644 --- a/src/Common/Game/T6/CommonT6.h +++ b/src/Common/Game/T6/CommonT6.h @@ -10,6 +10,7 @@ namespace T6 static int Com_HashKey(const char* str, int maxLen); static int Com_HashString(const char* str); static int Com_HashString(const char* str, int len); + static uint32_t R_HashString(const char* str, uint32_t hash); static PackedTexCoords Vec2PackTexCoords(const vec2_t* in); static PackedUnitVec Vec3PackUnitVec(const vec3_t* in); diff --git a/src/Common/Game/T6/T6_Assets.h b/src/Common/Game/T6/T6_Assets.h index e41a5702..0f839ca8 100644 --- a/src/Common/Game/T6/T6_Assets.h +++ b/src/Common/Game/T6/T6_Assets.h @@ -765,10 +765,10 @@ namespace T6 int platform[2]; }; - struct GfxStreamedPartInfo { - unsigned int levelCountAndSize; + uint32_t levelCount : 4; + uint32_t levelSize : 28; unsigned int hash; uint16_t width; uint16_t height; diff --git a/src/Linker/Game/IW3/ZoneCreatorIW3.cpp b/src/Linker/Game/IW3/ZoneCreatorIW3.cpp index 23d230c2..9c6f5012 100644 --- a/src/Linker/Game/IW3/ZoneCreatorIW3.cpp +++ b/src/Linker/Game/IW3/ZoneCreatorIW3.cpp @@ -22,7 +22,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name) m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); } -std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) +std::vector ZoneCreator::CreateGdtList(const ZoneCreationContext& context) { std::vector gdtList; gdtList.reserve(context.m_gdt_files.size()); @@ -32,9 +32,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -72,7 +72,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW3/ZoneCreatorIW3.h b/src/Linker/Game/IW3/ZoneCreatorIW3.h index 37c77262..b1ae2b3d 100644 --- a/src/Linker/Game/IW3/ZoneCreatorIW3.h +++ b/src/Linker/Game/IW3/ZoneCreatorIW3.h @@ -12,8 +12,8 @@ namespace IW3 std::unordered_map m_asset_types_by_name; void AddAssetTypeName(asset_type_t assetType, std::string name); - static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + static std::vector CreateGdtList(const ZoneCreationContext& context); + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; public: diff --git a/src/Linker/Game/IW4/ZoneCreatorIW4.cpp b/src/Linker/Game/IW4/ZoneCreatorIW4.cpp index 3f175168..f0946826 100644 --- a/src/Linker/Game/IW4/ZoneCreatorIW4.cpp +++ b/src/Linker/Game/IW4/ZoneCreatorIW4.cpp @@ -21,7 +21,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name) m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); } -std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) +std::vector ZoneCreator::CreateGdtList(const ZoneCreationContext& context) { std::vector gdtList; gdtList.reserve(context.m_gdt_files.size()); @@ -31,9 +31,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -71,7 +71,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW4/ZoneCreatorIW4.h b/src/Linker/Game/IW4/ZoneCreatorIW4.h index 2906af3d..ef0cbc54 100644 --- a/src/Linker/Game/IW4/ZoneCreatorIW4.h +++ b/src/Linker/Game/IW4/ZoneCreatorIW4.h @@ -12,8 +12,8 @@ namespace IW4 std::unordered_map m_asset_types_by_name; void AddAssetTypeName(asset_type_t assetType, std::string name); - static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + static std::vector CreateGdtList(const ZoneCreationContext& context); + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; public: diff --git a/src/Linker/Game/IW5/ZoneCreatorIW5.cpp b/src/Linker/Game/IW5/ZoneCreatorIW5.cpp index 9a1b1ac2..27d2dfb1 100644 --- a/src/Linker/Game/IW5/ZoneCreatorIW5.cpp +++ b/src/Linker/Game/IW5/ZoneCreatorIW5.cpp @@ -21,7 +21,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name) m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); } -std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) +std::vector ZoneCreator::CreateGdtList(const ZoneCreationContext& context) { std::vector gdtList; gdtList.reserve(context.m_gdt_files.size()); @@ -31,9 +31,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -71,7 +71,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW5/ZoneCreatorIW5.h b/src/Linker/Game/IW5/ZoneCreatorIW5.h index 7584ebfc..61f8475d 100644 --- a/src/Linker/Game/IW5/ZoneCreatorIW5.h +++ b/src/Linker/Game/IW5/ZoneCreatorIW5.h @@ -12,8 +12,8 @@ namespace IW5 std::unordered_map m_asset_types_by_name; void AddAssetTypeName(asset_type_t assetType, std::string name); - static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + static std::vector CreateGdtList(const ZoneCreationContext& context); + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; public: diff --git a/src/Linker/Game/T5/ZoneCreatorT5.cpp b/src/Linker/Game/T5/ZoneCreatorT5.cpp index 7bd31478..72e682e8 100644 --- a/src/Linker/Game/T5/ZoneCreatorT5.cpp +++ b/src/Linker/Game/T5/ZoneCreatorT5.cpp @@ -32,9 +32,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -72,7 +72,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/T5/ZoneCreatorT5.h b/src/Linker/Game/T5/ZoneCreatorT5.h index 0c676019..986ab855 100644 --- a/src/Linker/Game/T5/ZoneCreatorT5.h +++ b/src/Linker/Game/T5/ZoneCreatorT5.h @@ -13,7 +13,7 @@ namespace T5 void AddAssetTypeName(asset_type_t assetType, std::string name); static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; public: diff --git a/src/Linker/Game/T6/ZoneCreatorT6.cpp b/src/Linker/Game/T6/ZoneCreatorT6.cpp index 90f5f14d..a2885949 100644 --- a/src/Linker/Game/T6/ZoneCreatorT6.cpp +++ b/src/Linker/Game/T6/ZoneCreatorT6.cpp @@ -23,7 +23,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name) m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); } -std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) +std::vector ZoneCreator::CreateGdtList(const ZoneCreationContext& context) { std::vector gdtList; gdtList.reserve(context.m_gdt_files.size()); @@ -33,9 +33,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -58,7 +58,7 @@ void ZoneCreator::CreateZoneAssetPools(Zone* zone) const zone->m_pools->InitPoolDynamic(assetType); } -void ZoneCreator::HandleMetadata(Zone* zone, ZoneCreationContext& context) const +void ZoneCreator::HandleMetadata(Zone* zone, const ZoneCreationContext& context) const { std::vector kvpList; @@ -126,7 +126,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/T6/ZoneCreatorT6.h b/src/Linker/Game/T6/ZoneCreatorT6.h index ff0bcc7d..8e04d448 100644 --- a/src/Linker/Game/T6/ZoneCreatorT6.h +++ b/src/Linker/Game/T6/ZoneCreatorT6.h @@ -12,10 +12,10 @@ namespace T6 std::unordered_map m_asset_types_by_name; void AddAssetTypeName(asset_type_t assetType, std::string name); - static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + static std::vector CreateGdtList(const ZoneCreationContext& context); + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; - void HandleMetadata(Zone* zone, ZoneCreationContext& context) const; + void HandleMetadata(Zone* zone, const ZoneCreationContext& context) const; public: ZoneCreator(); diff --git a/src/Linker/Linker.cpp b/src/Linker/Linker.cpp index 87d25422..d626364c 100644 --- a/src/Linker/Linker.cpp +++ b/src/Linker/Linker.cpp @@ -12,9 +12,9 @@ #include "ObjWriting.h" #include "ObjLoading.h" #include "SearchPath/SearchPaths.h" -#include "SearchPath/SearchPathFilesystem.h" #include "ObjContainer/IWD/IWD.h" #include "LinkerArgs.h" +#include "LinkerSearchPaths.h" #include "ZoneWriting.h" #include "Game/IW3/ZoneCreatorIW3.h" #include "ZoneCreation/ZoneCreationContext.h" @@ -23,8 +23,10 @@ #include "Game/IW5/ZoneCreatorIW5.h" #include "Game/T5/ZoneCreatorT5.h" #include "Game/T6/ZoneCreatorT6.h" +#include "ObjContainer/IPak/IPakWriter.h" #include "Utils/ObjFileStream.h" +#include "Utils/StringUtils.h" #include "Zone/AssetList/AssetList.h" #include "Zone/AssetList/AssetListStream.h" #include "Zone/Definition/ZoneDefinitionStream.h" @@ -40,195 +42,31 @@ const IZoneCreator* const ZONE_CREATORS[] new T6::ZoneCreator() }; -class Linker::Impl +enum class ProjectType +{ + FASTFILE, + IPAK, + + MAX +}; + +constexpr const char* PROJECT_TYPE_NAMES[static_cast(ProjectType::MAX)] +{ + "fastfile", + "ipak" +}; + +class LinkerImpl final : public Linker { - static constexpr const char* METADATA_NAME = "name"; static constexpr const char* METADATA_GAME = "game"; static constexpr const char* METADATA_GDT = "gdt"; + static constexpr const char* METADATA_NAME = "name"; + static constexpr const char* METADATA_TYPE = "type"; LinkerArgs m_args; - std::vector> m_loaded_project_search_paths; - SearchPaths m_asset_search_paths; - SearchPaths m_gdt_search_paths; - SearchPaths m_source_search_paths; + LinkerSearchPaths m_search_paths; std::vector> m_loaded_zones; - /** - * \brief Loads a search path. - * \param searchPath The search path to load. - */ - void LoadSearchPath(ISearchPath* searchPath) const - { - if (m_args.m_verbose) - { - printf("Loading search path: \"%s\"\n", searchPath->GetPath().c_str()); - } - - ObjLoading::LoadIWDsInSearchPath(searchPath); - } - - /** - * \brief Unloads a search path. - * \param searchPath The search path to unload. - */ - void UnloadSearchPath(ISearchPath* searchPath) const - { - if (m_args.m_verbose) - { - printf("Unloading search path: \"%s\"\n", searchPath->GetPath().c_str()); - } - - ObjLoading::UnloadIWDsInSearchPath(searchPath); - } - - SearchPaths GetAssetSearchPathsForProject(const std::string& gameName, const std::string& projectName) - { - SearchPaths searchPathsForProject; - - for (const auto& searchPathStr : m_args.GetAssetSearchPathsForProject(gameName, projectName)) - { - auto absolutePath = fs::absolute(searchPathStr); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding asset search path: " << absolutePath.string() << std::endl; - - auto searchPath = std::make_unique(searchPathStr); - LoadSearchPath(searchPath.get()); - searchPathsForProject.IncludeSearchPath(searchPath.get()); - m_loaded_project_search_paths.emplace_back(std::move(searchPath)); - } - - searchPathsForProject.IncludeSearchPath(&m_asset_search_paths); - - for (auto* iwd : IWD::Repository) - { - searchPathsForProject.IncludeSearchPath(iwd); - } - - return searchPathsForProject; - } - - SearchPaths GetGdtSearchPathsForProject(const std::string& gameName, const std::string& projectName) - { - SearchPaths searchPathsForProject; - - for (const auto& searchPathStr : m_args.GetGdtSearchPathsForProject(gameName, projectName)) - { - auto absolutePath = fs::absolute(searchPathStr); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Adding gdt search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl; - - searchPathsForProject.CommitSearchPath(std::make_unique(searchPathStr)); - } - - searchPathsForProject.IncludeSearchPath(&m_gdt_search_paths); - - return searchPathsForProject; - } - - SearchPaths GetSourceSearchPathsForProject(const std::string& projectName) - { - SearchPaths searchPathsForProject; - - for (const auto& searchPathStr : m_args.GetSourceSearchPathsForProject(projectName)) - { - auto absolutePath = fs::absolute(searchPathStr); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Adding source search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding source search path: " << absolutePath.string() << std::endl; - - searchPathsForProject.CommitSearchPath(std::make_unique(searchPathStr)); - } - - searchPathsForProject.IncludeSearchPath(&m_source_search_paths); - - return searchPathsForProject; - } - - /** - * \brief Initializes the Linker object's search paths based on the user's input. - * \return \c true if building the search paths was successful, otherwise \c false. - */ - bool BuildProjectIndependentSearchPaths() - { - for (const auto& path : m_args.GetProjectIndependentAssetSearchPaths()) - { - auto absolutePath = fs::absolute(path); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding asset search path: " << absolutePath.string() << std::endl; - - auto searchPath = std::make_unique(absolutePath.string()); - LoadSearchPath(searchPath.get()); - m_asset_search_paths.CommitSearchPath(std::move(searchPath)); - } - - for (const auto& path : m_args.GetProjectIndependentGdtSearchPaths()) - { - auto absolutePath = fs::absolute(path); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Loading gdt search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl; - - m_gdt_search_paths.CommitSearchPath(std::make_unique(absolutePath.string())); - } - - for (const auto& path : m_args.GetProjectIndependentSourceSearchPaths()) - { - auto absolutePath = fs::absolute(path); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Loading source search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding source search path: " << absolutePath.string() << std::endl; - - m_source_search_paths.CommitSearchPath(std::make_unique(absolutePath.string())); - } - - return true; - } - bool IncludeAdditionalZoneDefinitions(const std::string& initialFileName, ZoneDefinition& zoneDefinition, ISearchPath* sourceSearchPath) const { std::set sourceNames; @@ -278,6 +116,58 @@ class Linker::Impl return true; } + bool ReadAssetList(const std::string& zoneName, AssetList& assetList, ISearchPath* sourceSearchPath) const + { + { + const auto assetListFileName = "assetlist/" + zoneName + ".csv"; + const auto assetListStream = sourceSearchPath->Open(assetListFileName); + + if (assetListStream.IsOpen()) + { + const AssetListInputStream stream(*assetListStream.m_stream); + AssetListEntry entry; + + while (stream.NextEntry(entry)) + { + assetList.m_entries.emplace_back(std::move(entry)); + } + return true; + } + } + + { + const auto zoneDefinition = ReadZoneDefinition(zoneName, sourceSearchPath); + + if (zoneDefinition) + { + for (const auto& entry : zoneDefinition->m_assets) + { + assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name, entry.m_is_reference); + } + return true; + } + } + + return false; + } + + bool IncludeAssetLists(ZoneDefinition& zoneDefinition, ISearchPath* sourceSearchPath) const + { + for (const auto& assetListName : zoneDefinition.m_asset_lists) + { + AssetList assetList; + if (!ReadAssetList(assetListName, assetList, sourceSearchPath)) + { + std::cerr << "Failed to read asset list \"" << assetListName << "\"\n"; + return false; + } + + zoneDefinition.Include(assetList); + } + + return true; + } + static bool GetNameFromZoneDefinition(std::string& name, const std::string& projectName, const ZoneDefinition& zoneDefinition) { auto firstNameEntry = true; @@ -333,44 +223,12 @@ class Linker::Impl if (!IncludeAdditionalZoneDefinitions(projectName, *zoneDefinition, sourceSearchPath)) return nullptr; + if (!IncludeAssetLists(*zoneDefinition, sourceSearchPath)) + return nullptr; + return zoneDefinition; } - bool ReadAssetList(const std::string& zoneName, std::vector& assetList, ISearchPath* sourceSearchPath) const - { - { - const auto assetListFileName = "assetlist/" + zoneName + ".csv"; - const auto assetListStream = sourceSearchPath->Open(assetListFileName); - - if (assetListStream.IsOpen()) - { - const AssetListInputStream stream(*assetListStream.m_stream); - AssetListEntry entry; - - while (stream.NextEntry(entry)) - { - assetList.emplace_back(std::move(entry)); - } - return true; - } - } - - { - const auto zoneDefinition = ReadZoneDefinition(zoneName, sourceSearchPath); - - if (zoneDefinition) - { - for (const auto& entry : zoneDefinition->m_assets) - { - assetList.emplace_back(entry.m_asset_type, entry.m_asset_name); - } - return true; - } - } - - return false; - } - bool ProcessZoneDefinitionIgnores(const std::string& projectName, ZoneCreationContext& context, ISearchPath* sourceSearchPath) const { if (context.m_definition->m_ignores.empty()) @@ -397,6 +255,55 @@ class Linker::Impl return true; } + static bool ProjectTypeByName(ProjectType& projectType, const std::string& projectTypeName) + { + for (auto i = 0u; i < static_cast(ProjectType::MAX); i++) + { + if (projectTypeName == PROJECT_TYPE_NAMES[i]) + { + projectType = static_cast(i); + return true; + } + } + + return false; + } + + static bool GetProjectTypeFromZoneDefinition(ProjectType& projectType, const std::string& projectName, const ZoneDefinition& zoneDefinition) + { + auto firstGameEntry = true; + const auto [rangeBegin, rangeEnd] = zoneDefinition.m_metadata_lookup.equal_range(METADATA_TYPE); + for (auto i = rangeBegin; i != rangeEnd; ++i) + { + ProjectType parsedProjectType; + if (!ProjectTypeByName(parsedProjectType, i->second->m_value)) + { + std::cerr << "Not a valid project type: \"" << i->second->m_value << "\"\n"; + return false; + } + + if (firstGameEntry) + { + projectType = parsedProjectType; + firstGameEntry = false; + } + else + { + if (projectType != parsedProjectType) + { + std::cerr << "Conflicting types in project \"" << projectName << "\": " << PROJECT_TYPE_NAMES[static_cast(projectType)] + << " != " << PROJECT_TYPE_NAMES[static_cast(parsedProjectType)] << std::endl; + return false; + } + } + } + + if (firstGameEntry) + projectType = ProjectType::FASTFILE; + + return true; + } + static bool GetGameNameFromZoneDefinition(std::string& gameName, const std::string& projectName, const ZoneDefinition& zoneDefinition) { auto firstGameEntry = true; @@ -498,34 +405,88 @@ class Linker::Impl return true; } + bool BuildFastFile(const std::string& projectName, ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& gdtSearchPaths, SearchPaths& sourceSearchPaths) const + { + const auto zone = CreateZoneForDefinition(projectName, zoneDefinition, &assetSearchPaths, &gdtSearchPaths, &sourceSearchPaths); + auto result = zone != nullptr; + if (zone) + result = WriteZoneToFile(projectName, zone.get()); + + return result; + } + + bool BuildIPak(const std::string& projectName, const ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& sourceSearchPaths) const + { + const fs::path ipakFolderPath(m_args.GetOutputFolderPathForProject(projectName)); + auto ipakFilePath(ipakFolderPath); + ipakFilePath.append(zoneDefinition.m_name + ".ipak"); + + fs::create_directories(ipakFolderPath); + + std::ofstream stream(ipakFilePath, std::fstream::out | std::fstream::binary); + if (!stream.is_open()) + return false; + + const auto ipakWriter = IPakWriter::Create(stream, &assetSearchPaths); + for (const auto& assetEntry : zoneDefinition.m_assets) + { + if (assetEntry.m_is_reference) + continue; + + if (assetEntry.m_asset_type == "image") + ipakWriter->AddImage(assetEntry.m_asset_name); + } + + if (!ipakWriter->Write()) + { + std::cout << "Writing ipak failed." << std::endl; + stream.close(); + return false; + } + + std::cout << "Created ipak \"" << ipakFilePath.string() << "\"\n"; + + stream.close(); + return true; + } + bool BuildProject(const std::string& projectName) { - auto sourceSearchPaths = GetSourceSearchPathsForProject(projectName); + auto sourceSearchPaths = m_search_paths.GetSourceSearchPathsForProject(projectName); const auto zoneDefinition = ReadZoneDefinition(projectName, &sourceSearchPaths); if (!zoneDefinition) return false; + ProjectType projectType; + if (!GetProjectTypeFromZoneDefinition(projectType, projectName, *zoneDefinition)) + return false; + std::string gameName; if (!GetGameNameFromZoneDefinition(gameName, projectName, *zoneDefinition)) return false; + utils::MakeStringLowerCase(gameName); - for (auto& c : gameName) - c = static_cast(std::tolower(c)); + auto assetSearchPaths = m_search_paths.GetAssetSearchPathsForProject(gameName, projectName); + auto gdtSearchPaths = m_search_paths.GetGdtSearchPathsForProject(gameName, projectName); - auto assetSearchPaths = GetAssetSearchPathsForProject(gameName, projectName); - auto gdtSearchPaths = GetGdtSearchPathsForProject(gameName, projectName); - - const auto zone = CreateZoneForDefinition(projectName, *zoneDefinition, &assetSearchPaths, &gdtSearchPaths, &sourceSearchPaths); - auto result = zone != nullptr; - if (zone) - result = WriteZoneToFile(projectName, zone.get()); - - for (const auto& loadedSearchPath : m_loaded_project_search_paths) + auto result = false; + switch (projectType) { - UnloadSearchPath(loadedSearchPath.get()); + case ProjectType::FASTFILE: + result = BuildFastFile(projectName, *zoneDefinition, assetSearchPaths, gdtSearchPaths, sourceSearchPaths); + break; + + case ProjectType::IPAK: + result = BuildIPak(projectName, *zoneDefinition, assetSearchPaths, sourceSearchPaths); + break; + + default: + assert(false); + break; } - m_loaded_project_search_paths.clear(); + + m_search_paths.UnloadProjectSpecificSearchPaths(); return result; } @@ -576,18 +537,17 @@ class Linker::Impl } public: - Impl() - = default; + LinkerImpl() + : m_search_paths(m_args) + { + } - /** - * \copydoc Linker::Start - */ - bool Start(const int argc, const char** argv) + bool Start(const int argc, const char** argv) override { if (!m_args.ParseArgs(argc, argv)) return false; - if (!BuildProjectIndependentSearchPaths()) + if (!m_search_paths.BuildProjectIndependentSearchPaths()) return false; if (!LoadZones()) @@ -609,18 +569,7 @@ public: } }; -Linker::Linker() +std::unique_ptr Linker::Create() { - m_impl = new Impl(); -} - -Linker::~Linker() -{ - delete m_impl; - m_impl = nullptr; -} - -bool Linker::Start(const int argc, const char** argv) const -{ - return m_impl->Start(argc, argv); + return std::make_unique(); } diff --git a/src/Linker/Linker.h b/src/Linker/Linker.h index 3e452c90..06bb0cb0 100644 --- a/src/Linker/Linker.h +++ b/src/Linker/Linker.h @@ -1,13 +1,11 @@ #pragma once +#include class Linker { - class Impl; - Impl* m_impl; - public: - Linker(); - ~Linker(); + Linker() = default; + virtual ~Linker() = default; Linker(const Linker& other) = delete; Linker(Linker&& other) noexcept = delete; @@ -20,5 +18,7 @@ public: * \param argv The command line arguments. * \return \c true if the application was successful or \c false if an error occurred. */ - bool Start(int argc, const char** argv) const; -}; \ No newline at end of file + virtual bool Start(int argc, const char** argv) = 0; + + static std::unique_ptr Create(); +}; diff --git a/src/Linker/LinkerSearchPaths.cpp b/src/Linker/LinkerSearchPaths.cpp new file mode 100644 index 00000000..48ce337d --- /dev/null +++ b/src/Linker/LinkerSearchPaths.cpp @@ -0,0 +1,190 @@ +#include "LinkerSearchPaths.h" + +#include +#include + +#include "ObjLoading.h" +#include "ObjContainer/IWD/IWD.h" +#include "SearchPath/SearchPathFilesystem.h" + +namespace fs = std::filesystem; + +LinkerSearchPaths::LinkerSearchPaths(const LinkerArgs& args) + : m_args(args) +{ +} + +void LinkerSearchPaths::LoadSearchPath(ISearchPath* searchPath) const +{ + if (m_args.m_verbose) + { + printf("Loading search path: \"%s\"\n", searchPath->GetPath().c_str()); + } + + ObjLoading::LoadIWDsInSearchPath(searchPath); +} + +void LinkerSearchPaths::UnloadSearchPath(ISearchPath* searchPath) const +{ + if (m_args.m_verbose) + { + printf("Unloading search path: \"%s\"\n", searchPath->GetPath().c_str()); + } + + ObjLoading::UnloadIWDsInSearchPath(searchPath); +} + +SearchPaths LinkerSearchPaths::GetAssetSearchPathsForProject(const std::string& gameName, const std::string& projectName) +{ + SearchPaths searchPathsForProject; + + for (const auto& searchPathStr : m_args.GetAssetSearchPathsForProject(gameName, projectName)) + { + auto absolutePath = fs::absolute(searchPathStr); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding asset search path: " << absolutePath.string() << std::endl; + + auto searchPath = std::make_unique(searchPathStr); + LoadSearchPath(searchPath.get()); + searchPathsForProject.IncludeSearchPath(searchPath.get()); + m_loaded_project_search_paths.emplace_back(std::move(searchPath)); + } + + searchPathsForProject.IncludeSearchPath(&m_asset_search_paths); + + for (auto* iwd : IWD::Repository) + { + searchPathsForProject.IncludeSearchPath(iwd); + } + + return searchPathsForProject; +} + +SearchPaths LinkerSearchPaths::GetGdtSearchPathsForProject(const std::string& gameName, const std::string& projectName) +{ + SearchPaths searchPathsForProject; + + for (const auto& searchPathStr : m_args.GetGdtSearchPathsForProject(gameName, projectName)) + { + auto absolutePath = fs::absolute(searchPathStr); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Adding gdt search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl; + + searchPathsForProject.CommitSearchPath(std::make_unique(searchPathStr)); + } + + searchPathsForProject.IncludeSearchPath(&m_gdt_search_paths); + + return searchPathsForProject; +} + +SearchPaths LinkerSearchPaths::GetSourceSearchPathsForProject(const std::string& projectName) +{ + SearchPaths searchPathsForProject; + + for (const auto& searchPathStr : m_args.GetSourceSearchPathsForProject(projectName)) + { + auto absolutePath = fs::absolute(searchPathStr); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Adding source search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding source search path: " << absolutePath.string() << std::endl; + + searchPathsForProject.CommitSearchPath(std::make_unique(searchPathStr)); + } + + searchPathsForProject.IncludeSearchPath(&m_source_search_paths); + + return searchPathsForProject; +} + +bool LinkerSearchPaths::BuildProjectIndependentSearchPaths() +{ + for (const auto& path : m_args.GetProjectIndependentAssetSearchPaths()) + { + auto absolutePath = fs::absolute(path); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding asset search path: " << absolutePath.string() << std::endl; + + auto searchPath = std::make_unique(absolutePath.string()); + LoadSearchPath(searchPath.get()); + m_asset_search_paths.CommitSearchPath(std::move(searchPath)); + } + + for (const auto& path : m_args.GetProjectIndependentGdtSearchPaths()) + { + auto absolutePath = fs::absolute(path); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Loading gdt search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl; + + m_gdt_search_paths.CommitSearchPath(std::make_unique(absolutePath.string())); + } + + for (const auto& path : m_args.GetProjectIndependentSourceSearchPaths()) + { + auto absolutePath = fs::absolute(path); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Loading source search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding source search path: " << absolutePath.string() << std::endl; + + m_source_search_paths.CommitSearchPath(std::make_unique(absolutePath.string())); + } + + return true; +} + + +void LinkerSearchPaths::UnloadProjectSpecificSearchPaths() +{ + for (const auto& loadedSearchPath : m_loaded_project_search_paths) + { + UnloadSearchPath(loadedSearchPath.get()); + } + + m_loaded_project_search_paths.clear(); +} diff --git a/src/Linker/LinkerSearchPaths.h b/src/Linker/LinkerSearchPaths.h new file mode 100644 index 00000000..ea5825d2 --- /dev/null +++ b/src/Linker/LinkerSearchPaths.h @@ -0,0 +1,46 @@ +#pragma once +#include "SearchPath/SearchPaths.h" + +#include +#include + +#include "LinkerArgs.h" + +class LinkerSearchPaths +{ +public: + explicit LinkerSearchPaths(const LinkerArgs& args); + + /** + * \brief Loads a search path. + * \param searchPath The search path to load. + */ + void LoadSearchPath(ISearchPath* searchPath) const; + + /** + * \brief Unloads a search path. + * \param searchPath The search path to unload. + */ + void UnloadSearchPath(ISearchPath* searchPath) const; + + SearchPaths GetAssetSearchPathsForProject(const std::string& gameName, const std::string& projectName); + + SearchPaths GetGdtSearchPathsForProject(const std::string& gameName, const std::string& projectName); + + SearchPaths GetSourceSearchPathsForProject(const std::string& projectName); + + /** + * \brief Initializes the Linker object's search paths based on the user's input. + * \return \c true if building the search paths was successful, otherwise \c false. + */ + bool BuildProjectIndependentSearchPaths(); + + void UnloadProjectSpecificSearchPaths(); + +private: + const LinkerArgs& m_args; + std::vector> m_loaded_project_search_paths; + SearchPaths m_asset_search_paths; + SearchPaths m_gdt_search_paths; + SearchPaths m_source_search_paths; +}; diff --git a/src/Linker/ZoneCreation/ZoneCreationContext.h b/src/Linker/ZoneCreation/ZoneCreationContext.h index 2db20e1d..a1c7800c 100644 --- a/src/Linker/ZoneCreation/ZoneCreationContext.h +++ b/src/Linker/ZoneCreation/ZoneCreationContext.h @@ -2,7 +2,6 @@ #include #include #include -#include #include "SearchPath/ISearchPath.h" #include "Obj/Gdt/Gdt.h" @@ -16,7 +15,7 @@ public: ISearchPath* m_asset_search_path; ZoneDefinition* m_definition; std::vector> m_gdt_files; - std::vector m_ignored_assets; + AssetList m_ignored_assets; ZoneCreationContext(); ZoneCreationContext(ISearchPath* assetSearchPath, ZoneDefinition* definition); diff --git a/src/Linker/main.cpp b/src/Linker/main.cpp index 6e78bec1..36f84947 100644 --- a/src/Linker/main.cpp +++ b/src/Linker/main.cpp @@ -2,7 +2,7 @@ int main(const int argc, const char** argv) { - Linker linker; + const auto linker = Linker::Create(); - return linker.Start(argc, argv) ? 0 : 1; + return linker->Start(argc, argv) ? 0 : 1; } diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index ee259f1d..3d516821 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -2,65 +2,84 @@ #include -typedef uint32_t IPakHash; +#include "Utils/FileUtils.h" namespace ipak_consts { - static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; - static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; + static constexpr uint32_t IPAK_MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I'); + static constexpr uint32_t IPAK_VERSION = 0x50000; + + static constexpr uint32_t IPAK_INDEX_SECTION = 1; + static constexpr uint32_t IPAK_DATA_SECTION = 2; + static constexpr uint32_t IPAK_BRANDING_SECTION = FileUtils::MakeMagic32('M', 'E', 'T', 'A'); + + static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; + static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; + + static constexpr uint32_t IPAK_COMMAND_DEFAULT_SIZE = 0x7F00; + static constexpr uint32_t IPAK_COMMAND_UNCOMPRESSED = 0; + static constexpr uint32_t IPAK_COMMAND_COMPRESSED = 1; + static constexpr uint32_t IPAK_COMMAND_SKIP = 0xCF; + + static_assert(IPAK_COMMAND_DEFAULT_SIZE <= IPAK_CHUNK_SIZE); } +typedef uint32_t IPakHash; + struct IPakHeader { - uint32_t magic; - uint32_t version; - uint32_t size; - uint32_t sectionCount; + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t sectionCount; }; struct IPakSection { - uint32_t type; - uint32_t offset; - uint32_t size; - uint32_t itemCount; + uint32_t type; + uint32_t offset; + uint32_t size; + uint32_t itemCount; }; union IPakIndexEntryKey { - struct - { - IPakHash dataHash; - IPakHash nameHash; - }; - uint64_t combinedKey; + struct + { + IPakHash dataHash; + IPakHash nameHash; + }; + + uint64_t combinedKey; }; struct IPakIndexEntry { - IPakIndexEntryKey key; - uint32_t offset; - uint32_t size; + IPakIndexEntryKey key; + uint32_t offset; + uint32_t size; }; +struct IPakDataBlockCountAndOffset +{ + uint32_t offset : 24; + uint32_t count : 8; +}; + +static_assert(sizeof(IPakDataBlockCountAndOffset) == 4); + +struct IPakDataBlockCommand +{ + uint32_t size : 24; + uint32_t compressed : 8; +}; + +static_assert(sizeof(IPakDataBlockCommand) == 4); + struct IPakDataBlockHeader { - union - { - uint32_t countAndOffset; - struct - { - uint32_t offset : 24; - uint32_t count : 8; - }; - }; - union - { - uint32_t commands[31]; - struct - { - uint32_t size : 24; - uint32_t compressed : 8; - }_commands[31]; - }; -}; \ No newline at end of file + IPakDataBlockCountAndOffset countAndOffset; + IPakDataBlockCommand commands[31]; +}; + +static_assert(sizeof(IPakDataBlockHeader) == 128); diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.cpp b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.cpp new file mode 100644 index 00000000..dcd4ff82 --- /dev/null +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.cpp @@ -0,0 +1,71 @@ +#include "AssetLoaderGfxImage.h" + +#include +#include +#include +#include + +#include "Game/T6/CommonT6.h" +#include "Game/T6/T6.h" +#include "Image/IwiLoader.h" +#include "Pool/GlobalAssetPool.h" + +using namespace T6; + +void* AssetLoaderGfxImage::CreateEmptyAsset(const std::string& assetName, MemoryManager* memory) +{ + auto* image = memory->Create(); + memset(image, 0, sizeof(GfxImage)); + image->name = memory->Dup(assetName.c_str()); + return image; +} + +bool AssetLoaderGfxImage::CanLoadFromRaw() const +{ + return true; +} + +bool AssetLoaderGfxImage::LoadFromRaw(const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const +{ + const auto fileName = "images/" + assetName + ".iwi"; + const auto file = searchPath->Open(fileName); + if (!file.IsOpen()) + return false; + + const auto fileSize = static_cast(file.m_length); + const auto fileData = std::make_unique(fileSize); + file.m_stream->read(fileData.get(), fileSize); + const auto dataHash = static_cast(crc32(0u, reinterpret_cast(fileData.get()), fileSize)); + + MemoryManager tempMemory; + IwiLoader iwiLoader(&tempMemory); + std::istringstream ss(std::string(fileData.get(), fileSize)); + const auto texture = iwiLoader.LoadIwi(ss); + if (!texture) + { + std::cerr << "Failed to load texture from: " << fileName << "\n"; + return false; + } + + auto* image = memory->Create(); + memset(image, 0, sizeof(GfxImage)); + + image->name = memory->Dup(assetName.c_str()); + image->hash = Common::R_HashString(image->name, 0); + image->delayLoadPixels = true; + + image->noPicmip = !texture->HasMipMaps(); + image->width = static_cast(texture->GetWidth()); + image->height = static_cast(texture->GetHeight()); + image->depth = static_cast(texture->GetDepth()); + + image->streaming = 1; + image->streamedParts[0].levelCount = 1; + image->streamedParts[0].levelSize = static_cast(fileSize); + image->streamedParts[0].hash = dataHash & 0x1FFFFFFF; + image->streamedPartCount = 1; + + manager->AddAsset(ASSET_TYPE_IMAGE, assetName, image); + + return true; +} diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.h b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.h new file mode 100644 index 00000000..f88f7685 --- /dev/null +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.h @@ -0,0 +1,16 @@ +#pragma once +#include "Game/T6/T6.h" +#include "AssetLoading/BasicAssetLoader.h" +#include "AssetLoading/IAssetLoadingManager.h" +#include "SearchPath/ISearchPath.h" + +namespace T6 +{ + class AssetLoaderGfxImage final : public BasicAssetLoader + { + public: + _NODISCARD void* CreateEmptyAsset(const std::string& assetName, MemoryManager* memory) override; + _NODISCARD bool CanLoadFromRaw() const override; + bool LoadFromRaw(const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const override; + }; +} diff --git a/src/ObjLoading/Game/T6/ObjLoaderT6.cpp b/src/ObjLoading/Game/T6/ObjLoaderT6.cpp index 13b385e7..bafa90bb 100644 --- a/src/ObjLoading/Game/T6/ObjLoaderT6.cpp +++ b/src/ObjLoading/Game/T6/ObjLoaderT6.cpp @@ -7,6 +7,7 @@ #include "ObjContainer/IPak/IPak.h" #include "ObjLoading.h" #include "AssetLoaders/AssetLoaderFontIcon.h" +#include "AssetLoaders/AssetLoaderGfxImage.h" #include "AssetLoaders/AssetLoaderLocalizeEntry.h" #include "AssetLoaders/AssetLoaderPhysConstraints.h" #include "AssetLoaders/AssetLoaderPhysPreset.h" @@ -45,7 +46,7 @@ namespace T6 REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_XMODEL, XModel)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_MATERIAL, Material)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_TECHNIQUE_SET, MaterialTechniqueSet)) - REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_IMAGE, GfxImage)) + REGISTER_ASSET_LOADER(AssetLoaderGfxImage) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_SOUND, SndBank)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_SOUND_PATCH, SndPatch)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_CLIPMAP, clipMap_t)) diff --git a/src/ObjLoading/ObjContainer/IPak/IPak.cpp b/src/ObjLoading/ObjContainer/IPak/IPak.cpp index b02f1c4e..c8d6fff1 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPak.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPak.cpp @@ -18,9 +18,6 @@ ObjContainerRepository IPak::Repository; class IPak::Impl : public ObjContainerReferenceable { - static const uint32_t MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I'); - static const uint32_t VERSION = 0x50000; - std::string m_path; std::unique_ptr m_stream; @@ -73,8 +70,8 @@ class IPak::Impl : public ObjContainerReferenceable { IPakSection section{}; - m_stream->read(reinterpret_cast(§ion), sizeof section); - if (m_stream->gcount() != sizeof section) + m_stream->read(reinterpret_cast(§ion), sizeof(section)); + if (m_stream->gcount() != sizeof(section)) { printf("Unexpected eof when trying to load section.\n"); return false; @@ -82,11 +79,11 @@ class IPak::Impl : public ObjContainerReferenceable switch (section.type) { - case 1: + case ipak_consts::IPAK_INDEX_SECTION: m_index_section = std::make_unique(section); break; - case 2: + case ipak_consts::IPAK_DATA_SECTION: m_data_section = std::make_unique(section); break; @@ -101,20 +98,20 @@ class IPak::Impl : public ObjContainerReferenceable { IPakHeader header{}; - m_stream->read(reinterpret_cast(&header), sizeof header); + m_stream->read(reinterpret_cast(&header), sizeof(header)); if (m_stream->gcount() != sizeof header) { printf("Unexpected eof when trying to load header.\n"); return false; } - if (header.magic != MAGIC) + if (header.magic != ipak_consts::IPAK_MAGIC) { printf("Invalid ipak magic '0x%x'.\n", header.magic); return false; } - if (header.version != VERSION) + if (header.version != ipak_consts::IPAK_VERSION) { printf("Unsupported ipak version '%u'.\n", header.version); return false; diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index f98f100e..f3b5a1ed 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -40,9 +40,11 @@ IPakEntryReadStream::~IPakEntryReadStream() size_t IPakEntryReadStream::ReadChunks(uint8_t* buffer, const int64_t startPos, const size_t chunkCount) const { m_stream_manager_actions->StartReading(); + m_stream.seekg(startPos); m_stream.read(reinterpret_cast(buffer), static_cast(chunkCount) * IPAK_CHUNK_SIZE); const auto readSize = static_cast(m_stream.gcount()); + m_stream_manager_actions->StopReading(); return readSize / IPAK_CHUNK_SIZE; @@ -52,7 +54,6 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch { // Cannot load more than IPAK_CHUNK_COUNT_PER_READ chunks without overflowing the buffer assert(chunkCount <= IPAK_CHUNK_COUNT_PER_READ); - if (chunkCount > IPAK_CHUNK_COUNT_PER_READ) chunkCount = IPAK_CHUNK_COUNT_PER_READ; @@ -68,8 +69,11 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch const auto endPos = startPos + static_cast(chunkCount) * IPAK_CHUNK_SIZE; + // Check whether the start position is already part of the loaded data + // We might be able to reuse previously loaded data if (startPos >= m_buffer_start_pos && startPos < m_buffer_end_pos) { + // Check whether we need to move data from inside the buffer to the start to account for new start if (m_buffer_start_pos != startPos) { const auto moveEnd = endPos < m_buffer_end_pos ? endPos : m_buffer_end_pos; @@ -78,6 +82,7 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch m_buffer_start_pos = startPos; } + // Check whether we need to load additional data that was not previously loaded if (endPos > m_buffer_end_pos) { const auto readChunkCount = ReadChunks(&m_chunk_buffer[m_buffer_end_pos - startPos], m_buffer_end_pos, @@ -92,11 +97,15 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch return true; } + // Check whether the end position is already part of the loaded data if (endPos > m_buffer_start_pos && endPos <= m_buffer_end_pos) { assert(IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ - static_cast(m_buffer_start_pos - startPos) >= static_cast(endPos - m_buffer_start_pos)); + + // Move data to make sure the end is at the appropriate position to be able to load the missing data in the front memmove(&m_chunk_buffer[m_buffer_start_pos - startPos], m_chunk_buffer, static_cast(endPos - m_buffer_start_pos)); + // We already established that the start of the buffer is not already loaded so we will need to load additional data nonetheless const auto readChunkCount = ReadChunks(m_chunk_buffer, startPos, static_cast(m_buffer_start_pos - startPos) / IPAK_CHUNK_SIZE); @@ -109,6 +118,7 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch return m_buffer_end_pos == endPos; } + // None of the data needed is already loaded -> Load everything and do not reuse any previously loaded data const auto readChunkCount = ReadChunks(m_chunk_buffer, startPos, chunkCount); m_buffer_start_pos = startPos; @@ -117,22 +127,26 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch return chunkCount == readChunkCount; } -bool IPakEntryReadStream::ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const +bool IPakEntryReadStream::ValidateBlockHeader(const IPakDataBlockHeader* blockHeader) const { - if (blockHeader->count > 31) + if (blockHeader->countAndOffset.count > 31) { - printf("IPak block has more than 31 commands: %u -> Invalid\n", blockHeader->count); + std::cerr << "IPak block has more than 31 commands: " << blockHeader->countAndOffset.count << " -> Invalid\n"; return false; } - if (blockHeader->offset != m_file_head) + + // We expect the current file to be continued where we left off + if (blockHeader->countAndOffset.offset != m_file_head) { - // A matching offset is only relevant if a command contains data. - for (unsigned currentCommand = 0; currentCommand < blockHeader->count; currentCommand++) + // A matching offset is only relevant if a command contains data + for (unsigned currentCommand = 0; currentCommand < blockHeader->countAndOffset.count; currentCommand++) { - if (blockHeader->_commands[currentCommand].compressed == 0 - || blockHeader->_commands[currentCommand].compressed == 1) + // If compressed is not 0 or 1 it will not be read and therefore it is okay when the offset does not match + // The game uses IPAK_COMMAND_SKIP as value for compressed when it intends to skip the specified amount of data + if (blockHeader->commands[currentCommand].compressed == 0 + || blockHeader->commands[currentCommand].compressed == 1) { - printf("IPak block offset is not the file head: %u != %lld -> Invalid\n", blockHeader->offset, m_file_head); + std::cerr << "IPak block offset (" << blockHeader->countAndOffset.offset << ") is not the file head (" << m_file_head << ") -> Invalid\n"; return false; } } @@ -141,13 +155,12 @@ bool IPakEntryReadStream::ValidateBlockHeader(IPakDataBlockHeader* blockHeader) return true; } -bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, - const size_t blockOffsetInChunk) +bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(const IPakDataBlockHeader* blockHeader, const size_t blockOffsetInChunk) { size_t commandsSize = 0; - for (unsigned commandIndex = 0; commandIndex < blockHeader->count; commandIndex++) + for (unsigned commandIndex = 0; commandIndex < blockHeader->countAndOffset.count; commandIndex++) { - commandsSize += blockHeader->_commands[commandIndex].size; + commandsSize += blockHeader->commands[commandIndex].size; } const size_t requiredChunkCount = AlignForward(blockOffsetInChunk + sizeof(IPakDataBlockHeader) + commandsSize, IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE; @@ -158,8 +171,7 @@ bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHea { if (requiredChunkCount > IPAK_CHUNK_COUNT_PER_READ) { - printf("IPak block spans over more than %u blocks (%u), which is not supported.\n", - IPAK_CHUNK_COUNT_PER_READ, requiredChunkCount); + std::cerr << "IPak block spans over more than " << IPAK_CHUNK_COUNT_PER_READ << " chunks (" << requiredChunkCount << "), which is not supported.\n"; return false; } @@ -180,7 +192,6 @@ bool IPakEntryReadStream::NextBlock() const auto chunkStartPos = AlignBackwards(m_pos, IPAK_CHUNK_SIZE); const auto blockOffsetInChunk = static_cast(m_pos - chunkStartPos); - const auto sizeLeftToRead = m_entry_size - m_file_head; auto estimatedChunksToRead = AlignForward(m_entry_size - static_cast(m_pos - m_base_pos), IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE; @@ -216,7 +227,7 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com if (result != LZO_E_OK) { - printf("Decompressing block with lzo failed: %i!\n", result); + std::cerr << "Decompressing block with lzo failed: " << result << "!\n"; return false; } @@ -225,6 +236,10 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com m_current_command_offset = 0; m_file_head += outputSize; } + else + { + // Do not process data but instead skip specified commandSize + } } else { @@ -240,14 +255,14 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com bool IPakEntryReadStream::AdvanceStream() { - if (m_current_block == nullptr || m_next_command >= m_current_block->count) + if (m_current_block == nullptr || m_next_command >= m_current_block->countAndOffset.count) { if (!NextBlock()) return false; } - ProcessCommand(m_current_block->_commands[m_next_command].size, - m_current_block->_commands[m_next_command].compressed); + ProcessCommand(m_current_block->commands[m_next_command].size, + m_current_block->commands[m_next_command].compressed); m_next_command++; return true; diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h index f6b13a1c..c8be8e79 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h @@ -33,24 +33,68 @@ class IPakEntryReadStream final : public objbuf int64_t m_buffer_start_pos; int64_t m_buffer_end_pos; - template + template static T AlignForward(const T num, const T alignTo) { return (num + alignTo - 1) / alignTo * alignTo; } - template + template static T AlignBackwards(const T num, const T alignTo) { return num / alignTo * alignTo; } + /** + * \brief Reads the specified chunks from disk. + * \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data. + * \param startPos The file offset at which the data to be loaded starts at. + * \param chunkCount The amount of chunks to be loaded. + * \return The amount of chunks that could be successfully loaded. + */ size_t ReadChunks(uint8_t* buffer, int64_t startPos, size_t chunkCount) const; + + /** + * \brief Make sure the loaded chunk buffer window corresponds to the specified parameters and loads additional data if necessary. + * \param startPos The offset in the file that should be the start of the chunk buffer. + * \param chunkCount The amount of chunks that the buffer should hold. Can not exceed IPAK_CHUNK_COUNT_PER_READ. + * \return \c true if the chunk buffer window could successfully be adjusted, \c false otherwise. + */ bool SetChunkBufferWindow(int64_t startPos, size_t chunkCount); - bool ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const; - bool AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, size_t blockOffsetInChunk); + + /** + * \brief Makes sure the specified block can be safely loaded. + * \param blockHeader The block header to check. + * \return \c true if the block can be safely loaded, \c false otherwise. + */ + bool ValidateBlockHeader(const IPakDataBlockHeader* blockHeader) const; + + /** + * \brief Makes sure that the specified block fits inside the loaded chunk buffer window and adjusts the chunk buffer window if necessary. + * \param blockHeader The header of the block that needs to fit inside the loaded chunk buffer window. + * \param blockOffsetInChunk The offset of the block inside the current chunk. + * \return \c true if the chunk buffer window was either already valid or was successfully adjusted to have all block data loaded. \c false otherwise. + */ + bool AdjustChunkBufferWindowForBlockHeader(const IPakDataBlockHeader* blockHeader, size_t blockOffsetInChunk); + + /** + * \brief Ensures the next valid block is loaded. + * \return \c true if a new block was loaded or \c false if no further valid block could be loaded. + */ bool NextBlock(); + + /** + * \brief Processes a command with the specified parameters at the current position. + * \param commandSize The size of the command data + * \param compressed The compression value of the command. Can be \c 0 for uncompressed or \c 1 for lzo compression. Any other value skips the specified size of data. + * \return \c true if the specified command could be correctly processed or \c otherwise. + */ bool ProcessCommand(size_t commandSize, int compressed); + + /** + * \brief Ensures that the next valid command is loaded. + * \return \c true if a new command was loaded or \c false if no further valid command could be loaded. + */ bool AdvanceStream(); public: @@ -67,4 +111,4 @@ protected: std::streamsize xsgetn(char* ptr, std::streamsize count) override; pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode mode) override; pos_type seekpos(pos_type pos, std::ios_base::openmode mode) override; -}; \ No newline at end of file +}; diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp index e69de29b..6767a64d 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp @@ -0,0 +1,382 @@ +#include "IPakWriter.h" + +#include +#include +#include +#include +#include + +#include "Game/T6/CommonT6.h" +#include "Game/T6/GameT6.h" +#include "ObjContainer/IPak/IPakTypes.h" +#include "Utils/Alignment.h" + +class IPakWriterImpl final : public IPakWriter +{ + static constexpr char BRANDING[] = "Created with OAT - OpenAssetTools"; + static constexpr auto SECTION_COUNT = 3; // Index + Data + Branding + + inline static const std::string PAD_DATA = std::string(256, '\xA7'); + +public: + explicit IPakWriterImpl(std::ostream& stream, ISearchPath* assetSearchPath) + : m_stream(stream), + m_asset_search_path(assetSearchPath), + m_current_offset(0), + m_total_size(0), + m_data_section_offset(0), + m_data_section_size(0u), + m_index_section_offset(0), + m_branding_section_offset(0), + m_file_offset(0u), + m_chunk_buffer_window_start(0), + m_current_block{}, + m_current_block_header_offset(0) + { + m_decompressed_buffer = std::make_unique(ipak_consts::IPAK_CHUNK_SIZE); + m_lzo_work_buffer = std::make_unique(LZO1X_1_MEM_COMPRESS); + } + + void AddImage(std::string imageName) override + { + m_images.emplace_back(std::move(imageName)); + } + + void GoTo(const int64_t offset) + { + m_stream.seekp(offset, std::ios::beg); + m_current_offset = offset; + } + + void Write(const void* data, const size_t dataSize) + { + m_stream.write(static_cast(data), dataSize); + m_current_offset += dataSize; + } + + void Pad(const size_t paddingSize) + { + auto paddingSizeLeft = paddingSize; + while (paddingSizeLeft > 0) + { + const auto writeSize = std::min(paddingSizeLeft, PAD_DATA.size()); + Write(PAD_DATA.data(), writeSize); + + paddingSizeLeft -= writeSize; + } + } + + void AlignToChunk() + { + Pad(static_cast(utils::Align(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)) - m_current_offset)); + } + + void AlignToBlockHeader() + { + Pad(static_cast(utils::Align(m_current_offset, static_cast(sizeof(IPakDataBlockHeader))) - m_current_offset)); + } + + void WriteHeaderData() + { + GoTo(0); + + const IPakHeader header{ + ipak_consts::IPAK_MAGIC, + ipak_consts::IPAK_VERSION, + static_cast(m_total_size), + SECTION_COUNT + }; + + const IPakSection dataSection{ + ipak_consts::IPAK_DATA_SECTION, + static_cast(m_data_section_offset), + static_cast(m_data_section_size), + static_cast(m_index_entries.size()) + }; + + const IPakSection indexSection{ + ipak_consts::IPAK_INDEX_SECTION, + static_cast(m_index_section_offset), + static_cast(sizeof(IPakIndexEntry) * m_index_entries.size()), + static_cast(m_index_entries.size()) + }; + + const IPakSection brandingSection{ + ipak_consts::IPAK_BRANDING_SECTION, + static_cast(m_branding_section_offset), + std::extent_v, + 1 + }; + + Write(&header, sizeof(header)); + Write(&dataSection, sizeof(dataSection)); + Write(&indexSection, sizeof(indexSection)); + Write(&brandingSection, sizeof(brandingSection)); + } + + static std::string ImageFileName(const std::string& imageName) + { + std::ostringstream ss; + ss << "images/" << imageName << ".iwi"; + + return ss.str(); + } + + std::unique_ptr ReadImageDataFromSearchPath(const std::string& imageName, size_t& imageSize) const + { + const auto fileName = ImageFileName(imageName); + + const auto openFile = m_asset_search_path->Open(fileName); + if (!openFile.IsOpen()) + { + std::cerr << "Could not open image for writing to IPak \"" << fileName << "\"\n"; + return nullptr; + } + + imageSize = static_cast(openFile.m_length); + auto imageData = std::make_unique(imageSize); + openFile.m_stream->read(imageData.get(), imageSize); + + return imageData; + } + + void FlushBlock() + { + if (m_current_block_header_offset > 0) + { + const auto previousOffset = m_current_offset; + + GoTo(m_current_block_header_offset); + Write(&m_current_block, sizeof(m_current_block)); + GoTo(previousOffset); + } + } + + void FlushChunk() + { + FlushBlock(); + AlignToBlockHeader(); + + const auto nextChunkOffset = utils::Align(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)); + const auto sizeToSkip = static_cast(nextChunkOffset - m_current_offset); + + if (sizeToSkip >= sizeof(IPakDataBlockHeader)) + { + IPakDataBlockHeader skipBlockHeader{}; + skipBlockHeader.countAndOffset.count = 1; + skipBlockHeader.commands[0].compressed = ipak_consts::IPAK_COMMAND_SKIP; + skipBlockHeader.commands[0].size = sizeToSkip - sizeof(IPakDataBlockHeader); + Write(&skipBlockHeader, sizeof(skipBlockHeader)); + } + + AlignToChunk(); + m_chunk_buffer_window_start = m_current_offset; + } + + void StartNewBlock() + { + AlignToBlockHeader(); + + // Skip to the next chunk when only the header could fit into the current chunk anyway + if (static_cast(utils::Align(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)) - m_current_offset) <= sizeof(IPakDataBlockHeader)) + FlushChunk(); + + m_current_block_header_offset = m_current_offset; + m_current_block = {}; + m_current_block.countAndOffset.offset = static_cast(m_file_offset); + + // Reserve space to later write actual block header data + GoTo(m_current_offset + sizeof(IPakDataBlockHeader)); + } + + void WriteChunkData(const void* data, const size_t dataSize) + { + auto dataOffset = 0u; + while (dataOffset < dataSize) + { + if (m_current_block.countAndOffset.count >= std::extent_v) + { + FlushBlock(); + StartNewBlock(); + } + + const auto remainingSize = dataSize - dataOffset; + const auto remainingChunkBufferWindowSize = std::max((ipak_consts::IPAK_CHUNK_COUNT_PER_READ * ipak_consts::IPAK_CHUNK_SIZE) + - static_cast(m_current_offset - m_chunk_buffer_window_start), 0u); + + if (remainingChunkBufferWindowSize == 0) + { + FlushChunk(); + StartNewBlock(); + continue; + } + + const auto commandSize = std::min(std::min(remainingSize, ipak_consts::IPAK_COMMAND_DEFAULT_SIZE), remainingChunkBufferWindowSize); + + auto writeUncompressed = true; + if (USE_COMPRESSION) + { + auto outLen = static_cast(ipak_consts::IPAK_CHUNK_SIZE); + const auto result = lzo1x_1_compress(&static_cast(data)[dataOffset], commandSize, reinterpret_cast(m_decompressed_buffer.get()), &outLen, + m_lzo_work_buffer.get()); + + if (result == LZO_E_OK && outLen < commandSize) + { + writeUncompressed = false; + Write(m_decompressed_buffer.get(), outLen); + + const auto currentCommand = m_current_block.countAndOffset.count; + m_current_block.commands[currentCommand].size = static_cast(outLen); + m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_COMPRESSED; + m_current_block.countAndOffset.count = currentCommand + 1u; + } + } + + if (writeUncompressed) + { + Write(&static_cast(data)[dataOffset], commandSize); + + const auto currentCommand = m_current_block.countAndOffset.count; + m_current_block.commands[currentCommand].size = commandSize; + m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_UNCOMPRESSED; + m_current_block.countAndOffset.count = currentCommand + 1u; + } + + dataOffset += commandSize; + m_file_offset += commandSize; + } + } + + void StartNewFile() + { + FlushBlock(); + + m_file_offset = 0u; + StartNewBlock(); + m_chunk_buffer_window_start = utils::AlignToPrevious(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)); + } + + bool WriteImageData(const std::string& imageName) + { + size_t imageSize; + const auto imageData = ReadImageDataFromSearchPath(imageName, imageSize); + if (!imageData) + return false; + + const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0); + const auto dataHash = static_cast(crc32(0u, reinterpret_cast(imageData.get()), imageSize)); + + StartNewFile(); + const auto startOffset = m_current_block_header_offset; + + IPakIndexEntry indexEntry; + indexEntry.key.nameHash = nameHash; + indexEntry.key.dataHash = dataHash & 0x1FFFFFFF; + indexEntry.offset = static_cast(startOffset - m_data_section_offset); + + WriteChunkData(imageData.get(), imageSize); + const auto writtenImageSize = static_cast(m_current_offset - startOffset); + + indexEntry.size = writtenImageSize; + m_index_entries.emplace_back(indexEntry); + + return true; + } + + bool WriteDataSection() + { + AlignToChunk(); + m_data_section_offset = m_current_offset; + m_data_section_size = 0u; + + m_index_entries.reserve(m_images.size()); + + const auto result = std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName) + { + return WriteImageData(imageName); + }); + + FlushBlock(); + m_data_section_size = static_cast(m_current_offset - m_data_section_offset); + + return result; + } + + static bool CompareIndices(const IPakIndexEntry& entry1, const IPakIndexEntry& entry2) + { + return entry1.key.combinedKey < entry2.key.combinedKey; + } + + void SortIndexSectionEntries() + { + std::sort(m_index_entries.begin(), m_index_entries.end(), CompareIndices); + } + + void WriteIndexSection() + { + AlignToChunk(); + m_index_section_offset = m_current_offset; + + SortIndexSectionEntries(); + + for (const auto& indexEntry : m_index_entries) + Write(&indexEntry, sizeof(indexEntry)); + } + + void WriteBrandingSection() + { + AlignToChunk(); + m_branding_section_offset = m_current_offset; + + Write(BRANDING, std::extent_v); + } + + void WriteFileEnding() + { + AlignToChunk(); + m_total_size = m_current_offset; + } + + bool Write() override + { + // We will write the header and sections later since they need complementary data + GoTo(sizeof(IPakHeader) + sizeof(IPakSection) * SECTION_COUNT); + AlignToChunk(); + + if (!WriteDataSection()) + return false; + + WriteIndexSection(); + WriteBrandingSection(); + WriteFileEnding(); + + WriteHeaderData(); + + return true; + } + +private: + std::ostream& m_stream; + ISearchPath* m_asset_search_path; + std::vector m_images; + + int64_t m_current_offset; + std::vector m_index_entries; + int64_t m_total_size; + int64_t m_data_section_offset; + size_t m_data_section_size; + int64_t m_index_section_offset; + int64_t m_branding_section_offset; + + std::unique_ptr m_decompressed_buffer; + std::unique_ptr m_lzo_work_buffer; + size_t m_file_offset; + int64_t m_chunk_buffer_window_start; + IPakDataBlockHeader m_current_block; + int64_t m_current_block_header_offset; +}; + +std::unique_ptr IPakWriter::Create(std::ostream& stream, ISearchPath* assetSearchPath) +{ + return std::make_unique(stream, assetSearchPath); +} diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h index e69de29b..6b9f0a3a 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h @@ -0,0 +1,24 @@ +#pragma once +#include +#include + +#include "SearchPath/ISearchPath.h" + +class IPakWriter +{ +public: + static constexpr auto USE_COMPRESSION = true; + + IPakWriter() = default; + virtual ~IPakWriter() = default; + + IPakWriter(const IPakWriter& other) = default; + IPakWriter(IPakWriter&& other) noexcept = default; + IPakWriter& operator=(const IPakWriter& other) = default; + IPakWriter& operator=(IPakWriter&& other) noexcept = default; + + virtual void AddImage(std::string imageName) = 0; + virtual bool Write() = 0; + + static std::unique_ptr Create(std::ostream& stream, ISearchPath* assetSearchPath); +}; diff --git a/src/Utils/Utils/Alignment.h b/src/Utils/Utils/Alignment.h index 9b0ba20d..053a4636 100644 --- a/src/Utils/Utils/Alignment.h +++ b/src/Utils/Utils/Alignment.h @@ -3,10 +3,18 @@ namespace utils { template - constexpr T Align(T value, T toNext) + constexpr T Align(const T value, const T toNext) { if (toNext > 0) return (value + toNext - 1) / toNext * toNext; return value; } + + template + constexpr T AlignToPrevious(const T value, const T toPrevious) + { + if (toPrevious > 0) + return value / toPrevious * toPrevious; + return value; + } } diff --git a/src/Utils/Utils/StringUtils.cpp b/src/Utils/Utils/StringUtils.cpp index e7f84d9a..3edf0c59 100644 --- a/src/Utils/Utils/StringUtils.cpp +++ b/src/Utils/Utils/StringUtils.cpp @@ -88,4 +88,16 @@ namespace utils inEscape = true; } } + + void MakeStringLowerCase(std::string& str) + { + for (auto& c : str) + c = static_cast(tolower(c)); + } + + void MakeStringUpperCase(std::string& str) + { + for (auto& c : str) + c = static_cast(toupper(c)); + } } diff --git a/src/Utils/Utils/StringUtils.h b/src/Utils/Utils/StringUtils.h index 860f8dbb..d49de6b9 100644 --- a/src/Utils/Utils/StringUtils.h +++ b/src/Utils/Utils/StringUtils.h @@ -11,4 +11,7 @@ namespace utils void EscapeStringForQuotationMarks(std::ostream& stream, const std::string_view& str); std::string UnescapeStringFromQuotationMarks(const std::string_view& str); void UnescapeStringFromQuotationMarks(std::ostream& stream, const std::string_view& str); + + void MakeStringLowerCase(std::string& str); + void MakeStringUpperCase(std::string& str); } diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.cpp new file mode 100644 index 00000000..39a35625 --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.cpp @@ -0,0 +1,19 @@ +#include "SequenceZoneDefinitionAssetList.h" + +#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h" + +SequenceZoneDefinitionAssetList::SequenceZoneDefinitionAssetList() +{ + const ZoneDefinitionMatcherFactory create(this); + + AddMatchers({ + create.Keyword("assetlist"), + create.Char(','), + create.Field().Capture(CAPTURE_ASSET_LIST_NAME) + }); +} + +void SequenceZoneDefinitionAssetList::ProcessMatch(ZoneDefinition* state, SequenceResult& result) const +{ + state->m_asset_lists.emplace_back(result.NextCapture(CAPTURE_ASSET_LIST_NAME).FieldValue()); +} diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.h b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.h new file mode 100644 index 00000000..f9a336e4 --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.h @@ -0,0 +1,14 @@ +#pragma once + +#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h" + +class SequenceZoneDefinitionAssetList final : public ZoneDefinitionParser::sequence_t +{ + static constexpr auto CAPTURE_ASSET_LIST_NAME = 1; + +protected: + void ProcessMatch(ZoneDefinition* state, SequenceResult& result) const override; + +public: + SequenceZoneDefinitionAssetList(); +}; diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp index ed52f58f..a6ae9552 100644 --- a/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp +++ b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp @@ -1,5 +1,6 @@ #include "ZoneDefinitionParser.h" +#include "Sequence/SequenceZoneDefinitionAssetList.h" #include "Sequence/SequenceZoneDefinitionEntry.h" #include "Sequence/SequenceZoneDefinitionIgnore.h" #include "Sequence/SequenceZoneDefinitionInclude.h" @@ -16,6 +17,7 @@ const std::vector::seq new SequenceZoneDefinitionMetaData(), new SequenceZoneDefinitionInclude(), new SequenceZoneDefinitionIgnore(), + new SequenceZoneDefinitionAssetList(), new SequenceZoneDefinitionEntry() }); diff --git a/src/ZoneCommon/Zone/AssetList/AssetList.cpp b/src/ZoneCommon/Zone/AssetList/AssetList.cpp index 56bb487e..05a13562 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetList.cpp +++ b/src/ZoneCommon/Zone/AssetList/AssetList.cpp @@ -1,10 +1,13 @@ #include "AssetList.h" AssetListEntry::AssetListEntry() -= default; - -AssetListEntry::AssetListEntry(std::string type, std::string name) - : m_type(std::move(type)), - m_name(std::move(name)) + : m_is_reference(false) +{ +} + +AssetListEntry::AssetListEntry(std::string type, std::string name, const bool isReference) + : m_type(std::move(type)), + m_name(std::move(name)), + m_is_reference(isReference) { } diff --git a/src/ZoneCommon/Zone/AssetList/AssetList.h b/src/ZoneCommon/Zone/AssetList/AssetList.h index 0bd7c0f3..186288fd 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetList.h +++ b/src/ZoneCommon/Zone/AssetList/AssetList.h @@ -1,12 +1,20 @@ #pragma once #include +#include class AssetListEntry { public: std::string m_type; std::string m_name; + bool m_is_reference; AssetListEntry(); - AssetListEntry(std::string type, std::string name); + AssetListEntry(std::string type, std::string name, bool isReference); }; + +class AssetList +{ +public: + std::vector m_entries; +}; \ No newline at end of file diff --git a/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp b/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp index e1d84954..546ec169 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp +++ b/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp @@ -9,7 +9,7 @@ bool AssetListInputStream::NextEntry(AssetListEntry& entry) const { std::vector row; - while(true) + while (true) { if (!m_stream.NextRow(row)) return false; @@ -18,8 +18,17 @@ bool AssetListInputStream::NextEntry(AssetListEntry& entry) const continue; entry.m_type = row[0]; - if (row.size() >= 2) + if (row.size() >= 3 && row[1].empty()) + { + entry.m_name = row[2]; + entry.m_is_reference = true; + } + else + { entry.m_name = row[1]; + entry.m_is_reference = false; + } + return true; } } diff --git a/src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp b/src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp index 1c3ed7e2..0c710d47 100644 --- a/src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp +++ b/src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp @@ -29,7 +29,15 @@ void ZoneDefinition::AddMetaData(std::string key, std::string value) m_metadata_lookup.emplace(std::make_pair(metaDataPtr->m_key, metaDataPtr)); } -void ZoneDefinition::Include(ZoneDefinition& definitionToInclude) +void ZoneDefinition::Include(const AssetList& assetListToInclude) +{ + for (const auto& entry : assetListToInclude.m_entries) + { + m_assets.emplace_back(entry.m_type, entry.m_name, false); + } +} + +void ZoneDefinition::Include(const ZoneDefinition& definitionToInclude) { for (const auto& metaData : definitionToInclude.m_metadata) { diff --git a/src/ZoneCommon/Zone/Definition/ZoneDefinition.h b/src/ZoneCommon/Zone/Definition/ZoneDefinition.h index 2a093f8d..4ec8a915 100644 --- a/src/ZoneCommon/Zone/Definition/ZoneDefinition.h +++ b/src/ZoneCommon/Zone/Definition/ZoneDefinition.h @@ -5,6 +5,8 @@ #include #include +#include "Zone/AssetList/AssetList.h" + class ZoneDefinitionEntry { public: @@ -33,9 +35,11 @@ public: std::vector> m_metadata; std::unordered_multimap m_metadata_lookup; std::vector m_includes; + std::vector m_asset_lists; std::vector m_ignores; std::vector m_assets; void AddMetaData(std::string key, std::string value); - void Include(ZoneDefinition& definitionToInclude); + void Include(const AssetList& assetListToInclude); + void Include(const ZoneDefinition& definitionToInclude); }; \ No newline at end of file