From 85143784653503a95be006525955df8772e761cb Mon Sep 17 00:00:00 2001 From: Jan Date: Sat, 7 Oct 2023 19:41:54 +0200 Subject: [PATCH] Write IPak base skeleton without data --- src/Common/Game/T6/CommonT6.cpp | 10 + src/Common/Game/T6/CommonT6.h | 1 + src/Linker/Game/IW3/ZoneCreatorIW3.cpp | 2 +- src/Linker/Game/IW4/ZoneCreatorIW4.cpp | 2 +- src/Linker/Game/IW5/ZoneCreatorIW5.cpp | 2 +- src/Linker/Game/T5/ZoneCreatorT5.cpp | 2 +- src/Linker/Game/T6/ZoneCreatorT6.cpp | 2 +- src/Linker/Linker.cpp | 37 ++- src/ObjCommon/ObjContainer/IPak/IPakTypes.h | 91 ++++--- src/ObjLoading/ObjContainer/IPak/IPak.cpp | 11 +- .../ObjContainer/IPak/IPakWriter.cpp | 241 ++++++++++++++++++ src/ObjWriting/ObjContainer/IPak/IPakWriter.h | 22 ++ src/ZoneCommon/Zone/AssetList/AssetList.cpp | 13 +- src/ZoneCommon/Zone/AssetList/AssetList.h | 3 +- .../Zone/AssetList/AssetListStream.cpp | 13 +- 15 files changed, 390 insertions(+), 62 deletions(-) diff --git a/src/Common/Game/T6/CommonT6.cpp b/src/Common/Game/T6/CommonT6.cpp index 3d181616..d2ea3577 100644 --- a/src/Common/Game/T6/CommonT6.cpp +++ b/src/Common/Game/T6/CommonT6.cpp @@ -58,6 +58,16 @@ int Common::Com_HashString(const char* str, const int len) return result; } +uint32_t Common::R_HashString(const char* str, uint32_t hash) +{ + for (const auto* pos = str; *pos; pos++) + { + hash = 33 * hash ^ (*pos | 0x20); + } + + return hash; +} + PackedTexCoords Common::Vec2PackTexCoords(const vec2_t* in) { return PackedTexCoords{ Pack32::Vec2PackTexCoords(in->v) }; diff --git a/src/Common/Game/T6/CommonT6.h b/src/Common/Game/T6/CommonT6.h index ad8f38c0..2c63c62e 100644 --- a/src/Common/Game/T6/CommonT6.h +++ b/src/Common/Game/T6/CommonT6.h @@ -10,6 +10,7 @@ namespace T6 static int Com_HashKey(const char* str, int maxLen); static int Com_HashString(const char* str); static int Com_HashString(const char* str, int len); + static uint32_t R_HashString(const char* str, uint32_t hash); static PackedTexCoords Vec2PackTexCoords(const vec2_t* in); static PackedUnitVec Vec3PackUnitVec(const vec3_t* in); diff --git a/src/Linker/Game/IW3/ZoneCreatorIW3.cpp b/src/Linker/Game/IW3/ZoneCreatorIW3.cpp index 1bdd19ff..9c6f5012 100644 --- a/src/Linker/Game/IW3/ZoneCreatorIW3.cpp +++ b/src/Linker/Game/IW3/ZoneCreatorIW3.cpp @@ -72,7 +72,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW4/ZoneCreatorIW4.cpp b/src/Linker/Game/IW4/ZoneCreatorIW4.cpp index 5030fbaa..f0946826 100644 --- a/src/Linker/Game/IW4/ZoneCreatorIW4.cpp +++ b/src/Linker/Game/IW4/ZoneCreatorIW4.cpp @@ -71,7 +71,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW5/ZoneCreatorIW5.cpp b/src/Linker/Game/IW5/ZoneCreatorIW5.cpp index 51943da4..27d2dfb1 100644 --- a/src/Linker/Game/IW5/ZoneCreatorIW5.cpp +++ b/src/Linker/Game/IW5/ZoneCreatorIW5.cpp @@ -71,7 +71,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/T5/ZoneCreatorT5.cpp b/src/Linker/Game/T5/ZoneCreatorT5.cpp index 117a64c7..72e682e8 100644 --- a/src/Linker/Game/T5/ZoneCreatorT5.cpp +++ b/src/Linker/Game/T5/ZoneCreatorT5.cpp @@ -72,7 +72,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/T6/ZoneCreatorT6.cpp b/src/Linker/Game/T6/ZoneCreatorT6.cpp index cdd923b6..a2885949 100644 --- a/src/Linker/Game/T6/ZoneCreatorT6.cpp +++ b/src/Linker/Game/T6/ZoneCreatorT6.cpp @@ -126,7 +126,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Linker.cpp b/src/Linker/Linker.cpp index d94d39d8..d626364c 100644 --- a/src/Linker/Linker.cpp +++ b/src/Linker/Linker.cpp @@ -23,6 +23,7 @@ #include "Game/IW5/ZoneCreatorIW5.h" #include "Game/T5/ZoneCreatorT5.h" #include "Game/T6/ZoneCreatorT6.h" +#include "ObjContainer/IPak/IPakWriter.h" #include "Utils/ObjFileStream.h" #include "Utils/StringUtils.h" @@ -141,7 +142,7 @@ class LinkerImpl final : public Linker { for (const auto& entry : zoneDefinition->m_assets) { - assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name); + assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name, entry.m_is_reference); } return true; } @@ -414,9 +415,39 @@ class LinkerImpl final : public Linker return result; } - bool BuildIPak(const std::string& projectName, ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& sourceSearchPaths) + bool BuildIPak(const std::string& projectName, const ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& sourceSearchPaths) const { - return false; + const fs::path ipakFolderPath(m_args.GetOutputFolderPathForProject(projectName)); + auto ipakFilePath(ipakFolderPath); + ipakFilePath.append(zoneDefinition.m_name + ".ipak"); + + fs::create_directories(ipakFolderPath); + + std::ofstream stream(ipakFilePath, std::fstream::out | std::fstream::binary); + if (!stream.is_open()) + return false; + + const auto ipakWriter = IPakWriter::Create(stream, &assetSearchPaths); + for (const auto& assetEntry : zoneDefinition.m_assets) + { + if (assetEntry.m_is_reference) + continue; + + if (assetEntry.m_asset_type == "image") + ipakWriter->AddImage(assetEntry.m_asset_name); + } + + if (!ipakWriter->Write()) + { + std::cout << "Writing ipak failed." << std::endl; + stream.close(); + return false; + } + + std::cout << "Created ipak \"" << ipakFilePath.string() << "\"\n"; + + stream.close(); + return true; } bool BuildProject(const std::string& projectName) diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index ee259f1d..b496d0db 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -2,65 +2,78 @@ #include -typedef uint32_t IPakHash; +#include "Utils/FileUtils.h" namespace ipak_consts { - static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; - static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; + static constexpr uint32_t IPAK_MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I'); + static constexpr uint32_t IPAK_VERSION = 0x50000; + + static constexpr uint32_t IPAK_INDEX_SECTION = 1; + static constexpr uint32_t IPAK_DATA_SECTION = 2; + static constexpr uint32_t IPAK_BRANDING_SECTION = FileUtils::MakeMagic32('M', 'E', 'T', 'A'); + + static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; + static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; } +typedef uint32_t IPakHash; + struct IPakHeader { - uint32_t magic; - uint32_t version; - uint32_t size; - uint32_t sectionCount; + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t sectionCount; }; struct IPakSection { - uint32_t type; - uint32_t offset; - uint32_t size; - uint32_t itemCount; + uint32_t type; + uint32_t offset; + uint32_t size; + uint32_t itemCount; }; union IPakIndexEntryKey { - struct - { - IPakHash dataHash; - IPakHash nameHash; - }; - uint64_t combinedKey; + struct + { + IPakHash dataHash; + IPakHash nameHash; + }; + + uint64_t combinedKey; }; struct IPakIndexEntry { - IPakIndexEntryKey key; - uint32_t offset; - uint32_t size; + IPakIndexEntryKey key; + uint32_t offset; + uint32_t size; }; struct IPakDataBlockHeader { - union - { - uint32_t countAndOffset; - struct - { - uint32_t offset : 24; - uint32_t count : 8; - }; - }; - union - { - uint32_t commands[31]; - struct - { - uint32_t size : 24; - uint32_t compressed : 8; - }_commands[31]; - }; -}; \ No newline at end of file + union + { + uint32_t countAndOffset; + + struct + { + uint32_t offset : 24; + uint32_t count : 8; + }; + }; + + union + { + uint32_t commands[31]; + + struct + { + uint32_t size : 24; + uint32_t compressed : 8; + } _commands[31]; + }; +}; diff --git a/src/ObjLoading/ObjContainer/IPak/IPak.cpp b/src/ObjLoading/ObjContainer/IPak/IPak.cpp index b02f1c4e..953cc81e 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPak.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPak.cpp @@ -18,9 +18,6 @@ ObjContainerRepository IPak::Repository; class IPak::Impl : public ObjContainerReferenceable { - static const uint32_t MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I'); - static const uint32_t VERSION = 0x50000; - std::string m_path; std::unique_ptr m_stream; @@ -82,11 +79,11 @@ class IPak::Impl : public ObjContainerReferenceable switch (section.type) { - case 1: + case ipak_consts::IPAK_INDEX_SECTION: m_index_section = std::make_unique(section); break; - case 2: + case ipak_consts::IPAK_DATA_SECTION: m_data_section = std::make_unique(section); break; @@ -108,13 +105,13 @@ class IPak::Impl : public ObjContainerReferenceable return false; } - if (header.magic != MAGIC) + if (header.magic != ipak_consts::IPAK_MAGIC) { printf("Invalid ipak magic '0x%x'.\n", header.magic); return false; } - if (header.version != VERSION) + if (header.version != ipak_consts::IPAK_VERSION) { printf("Unsupported ipak version '%u'.\n", header.version); return false; diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp index e69de29b..c29387b3 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp @@ -0,0 +1,241 @@ +#include "IPakWriter.h" + +#include +#include +#include +#include + +#include "Game/T6/CommonT6.h" +#include "Game/T6/GameT6.h" +#include "ObjContainer/IPak/IPakTypes.h" +#include "Utils/Alignment.h" + +class IPakWriterImpl final : public IPakWriter +{ + static constexpr char BRANDING[] = "Created with OAT - OpenAssetTools"; + static constexpr auto SECTION_COUNT = 3; // Index + Data + Branding + + inline static const std::string PAD_DATA = std::string(256, '\xA7'); + +public: + explicit IPakWriterImpl(std::ostream& stream, ISearchPath* assetSearchPath) + : m_stream(stream), + m_asset_search_path(assetSearchPath), + m_current_offset(0), + m_total_size(0), + m_data_section_offset(0), + m_data_section_size(0u), + m_index_section_offset(0), + m_branding_section_offset(0) + { + } + + void AddImage(std::string imageName) override + { + m_images.emplace_back(std::move(imageName)); + } + + void GoTo(const int64_t offset) + { + m_stream.seekp(offset, std::ios::beg); + m_current_offset = offset; + } + + void Write(const void* data, const size_t dataSize) + { + m_stream.write(static_cast(data), dataSize); + m_current_offset += dataSize; + } + + void Pad(const size_t paddingSize) + { + auto paddingSizeLeft = paddingSize; + while (paddingSizeLeft > 0) + { + const auto writeSize = std::min(paddingSizeLeft, PAD_DATA.size()); + Write(PAD_DATA.data(), writeSize); + + paddingSizeLeft -= writeSize; + } + } + + void AlignToChunkWrite() + { + Pad(static_cast(utils::Align(m_current_offset, 0x8000i64) - m_current_offset)); + } + + void WriteHeaderData() + { + GoTo(0); + + const IPakHeader header{ + ipak_consts::IPAK_MAGIC, + ipak_consts::IPAK_VERSION, + static_cast(m_total_size), + SECTION_COUNT + }; + + const IPakSection dataSection{ + ipak_consts::IPAK_DATA_SECTION, + static_cast(m_data_section_offset), + static_cast(m_data_section_size), + static_cast(m_index_entries.size()) + }; + + const IPakSection indexSection{ + ipak_consts::IPAK_INDEX_SECTION, + static_cast(m_index_section_offset), + static_cast(sizeof(IPakIndexEntry) * m_index_entries.size()), + static_cast(m_index_entries.size()) + }; + + const IPakSection brandingSection{ + ipak_consts::IPAK_BRANDING_SECTION, + static_cast(m_branding_section_offset), + std::extent_v, + 1 + }; + + Write(&header, sizeof(header)); + Write(&dataSection, sizeof(dataSection)); + Write(&indexSection, sizeof(indexSection)); + Write(&brandingSection, sizeof(brandingSection)); + } + + static std::string ImageFileName(const std::string& imageName) + { + std::ostringstream ss; + ss << "images/" << imageName << ".iwi"; + + return ss.str(); + } + + std::unique_ptr ReadImageDataFromSearchPath(const std::string& imageName, size_t& imageSize) const + { + const auto fileName = ImageFileName(imageName); + + const auto openFile = m_asset_search_path->Open(fileName); + if (!openFile.IsOpen()) + { + std::cerr << "Could not open image for writing to IPak \"" << fileName << "\"\n"; + return nullptr; + } + + imageSize = static_cast(openFile.m_length); + auto imageData = std::make_unique(imageSize); + openFile.m_stream->read(imageData.get(), imageSize); + + return imageData; + } + + bool WriteImageData(const std::string& imageName) + { + size_t imageSize; + const auto imageData = ReadImageDataFromSearchPath(imageName, imageSize); + if (!imageData) + return false; + + const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0); + const auto dataHash = static_cast(crc32(0u, reinterpret_cast(imageData.get()), imageSize)); + + IPakIndexEntry indexEntry{}; + indexEntry.key.nameHash = nameHash; + indexEntry.key.dataHash = dataHash & 0x1FFFFFFF; + indexEntry.offset = static_cast(m_current_offset - m_data_section_offset); + + size_t writtenImageSize = 0u; + + // TODO: Write image data + + indexEntry.size = writtenImageSize; + m_index_entries.emplace_back(indexEntry); + + m_data_section_size += writtenImageSize; + return true; + } + + bool WriteDataSection() + { + AlignToChunkWrite(); + m_data_section_offset = m_current_offset; + m_data_section_size = 0u; + + m_index_entries.reserve(m_images.size()); + + return std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName) + { + return WriteImageData(imageName); + }); + } + + static bool CompareIndices(const IPakIndexEntry& entry1, const IPakIndexEntry& entry2) + { + return entry1.key.combinedKey < entry2.key.combinedKey; + } + + void SortIndexSectionEntries() + { + std::sort(m_index_entries.begin(), m_index_entries.end(), CompareIndices); + } + + void WriteIndexSection() + { + AlignToChunkWrite(); + m_index_section_offset = m_current_offset; + + SortIndexSectionEntries(); + + for (const auto& indexEntry : m_index_entries) + Write(&indexEntry, sizeof(indexEntry)); + } + + void WriteBrandingSection() + { + AlignToChunkWrite(); + m_branding_section_offset = m_current_offset; + + Write(BRANDING, std::extent_v); + } + + void WriteFileEnding() + { + AlignToChunkWrite(); + m_total_size = m_current_offset; + } + + bool Write() override + { + // We will write the header and sections later since they need complementary data + GoTo(sizeof(IPakHeader) + sizeof(IPakSection) * SECTION_COUNT); + AlignToChunkWrite(); + + if (!WriteDataSection()) + return false; + + WriteIndexSection(); + WriteBrandingSection(); + WriteFileEnding(); + + WriteHeaderData(); + + return true; + } + +private: + std::ostream& m_stream; + ISearchPath* m_asset_search_path; + std::vector m_images; + + int64_t m_current_offset; + std::vector m_index_entries; + int64_t m_total_size; + int64_t m_data_section_offset; + size_t m_data_section_size; + int64_t m_index_section_offset; + int64_t m_branding_section_offset; +}; + +std::unique_ptr IPakWriter::Create(std::ostream& stream, ISearchPath* assetSearchPath) +{ + return std::make_unique(stream, assetSearchPath); +} diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h index e69de29b..f7051873 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h @@ -0,0 +1,22 @@ +#pragma once +#include +#include + +#include "SearchPath/ISearchPath.h" + +class IPakWriter +{ +public: + IPakWriter() = default; + virtual ~IPakWriter() = default; + + IPakWriter(const IPakWriter& other) = default; + IPakWriter(IPakWriter&& other) noexcept = default; + IPakWriter& operator=(const IPakWriter& other) = default; + IPakWriter& operator=(IPakWriter&& other) noexcept = default; + + virtual void AddImage(std::string imageName) = 0; + virtual bool Write() = 0; + + static std::unique_ptr Create(std::ostream& stream, ISearchPath* assetSearchPath); +}; diff --git a/src/ZoneCommon/Zone/AssetList/AssetList.cpp b/src/ZoneCommon/Zone/AssetList/AssetList.cpp index 56bb487e..05a13562 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetList.cpp +++ b/src/ZoneCommon/Zone/AssetList/AssetList.cpp @@ -1,10 +1,13 @@ #include "AssetList.h" AssetListEntry::AssetListEntry() -= default; - -AssetListEntry::AssetListEntry(std::string type, std::string name) - : m_type(std::move(type)), - m_name(std::move(name)) + : m_is_reference(false) +{ +} + +AssetListEntry::AssetListEntry(std::string type, std::string name, const bool isReference) + : m_type(std::move(type)), + m_name(std::move(name)), + m_is_reference(isReference) { } diff --git a/src/ZoneCommon/Zone/AssetList/AssetList.h b/src/ZoneCommon/Zone/AssetList/AssetList.h index 1f02b329..186288fd 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetList.h +++ b/src/ZoneCommon/Zone/AssetList/AssetList.h @@ -7,9 +7,10 @@ class AssetListEntry public: std::string m_type; std::string m_name; + bool m_is_reference; AssetListEntry(); - AssetListEntry(std::string type, std::string name); + AssetListEntry(std::string type, std::string name, bool isReference); }; class AssetList diff --git a/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp b/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp index e1d84954..546ec169 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp +++ b/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp @@ -9,7 +9,7 @@ bool AssetListInputStream::NextEntry(AssetListEntry& entry) const { std::vector row; - while(true) + while (true) { if (!m_stream.NextRow(row)) return false; @@ -18,8 +18,17 @@ bool AssetListInputStream::NextEntry(AssetListEntry& entry) const continue; entry.m_type = row[0]; - if (row.size() >= 2) + if (row.size() >= 3 && row[1].empty()) + { + entry.m_name = row[2]; + entry.m_is_reference = true; + } + else + { entry.m_name = row[1]; + entry.m_is_reference = false; + } + return true; } }