Merge pull request #20 from Laupetin/feature/ipak-building

Add IPak projects and building
This commit is contained in:
Jan 2023-10-15 22:53:16 +02:00 committed by GitHub
commit 1384c912e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 1236 additions and 382 deletions

View File

@ -58,6 +58,16 @@ int Common::Com_HashString(const char* str, const int len)
return result; return result;
} }
uint32_t Common::R_HashString(const char* str, uint32_t hash)
{
for (const auto* pos = str; *pos; pos++)
{
hash = 33 * hash ^ (*pos | 0x20);
}
return hash;
}
PackedTexCoords Common::Vec2PackTexCoords(const vec2_t* in) PackedTexCoords Common::Vec2PackTexCoords(const vec2_t* in)
{ {
return PackedTexCoords{ Pack32::Vec2PackTexCoords(in->v) }; return PackedTexCoords{ Pack32::Vec2PackTexCoords(in->v) };

View File

@ -10,6 +10,7 @@ namespace T6
static int Com_HashKey(const char* str, int maxLen); static int Com_HashKey(const char* str, int maxLen);
static int Com_HashString(const char* str); static int Com_HashString(const char* str);
static int Com_HashString(const char* str, int len); static int Com_HashString(const char* str, int len);
static uint32_t R_HashString(const char* str, uint32_t hash);
static PackedTexCoords Vec2PackTexCoords(const vec2_t* in); static PackedTexCoords Vec2PackTexCoords(const vec2_t* in);
static PackedUnitVec Vec3PackUnitVec(const vec3_t* in); static PackedUnitVec Vec3PackUnitVec(const vec3_t* in);

View File

@ -765,10 +765,10 @@ namespace T6
int platform[2]; int platform[2];
}; };
struct GfxStreamedPartInfo struct GfxStreamedPartInfo
{ {
unsigned int levelCountAndSize; uint32_t levelCount : 4;
uint32_t levelSize : 28;
unsigned int hash; unsigned int hash;
uint16_t width; uint16_t width;
uint16_t height; uint16_t height;

View File

@ -22,7 +22,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name)
m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType));
} }
std::vector<Gdt*> ZoneCreator::CreateGdtList(ZoneCreationContext& context) std::vector<Gdt*> ZoneCreator::CreateGdtList(const ZoneCreationContext& context)
{ {
std::vector<Gdt*> gdtList; std::vector<Gdt*> gdtList;
gdtList.reserve(context.m_gdt_files.size()); gdtList.reserve(context.m_gdt_files.size());
@ -32,9 +32,9 @@ std::vector<Gdt*> ZoneCreator::CreateGdtList(ZoneCreationContext& context)
return gdtList; return gdtList;
} }
bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const
{ {
for (const auto& ignoreEntry : context.m_ignored_assets) for (const auto& ignoreEntry : context.m_ignored_assets.m_entries)
{ {
const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type);
if (foundAssetTypeEntry == m_asset_types_by_name.end()) if (foundAssetTypeEntry == m_asset_types_by_name.end())
@ -72,7 +72,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
if (!assetEntry.m_is_reference) if (!assetEntry.m_is_reference)
continue; continue;
context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
} }
const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context)); const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

View File

@ -12,8 +12,8 @@ namespace IW3
std::unordered_map<std::string, asset_type_t> m_asset_types_by_name; std::unordered_map<std::string, asset_type_t> m_asset_types_by_name;
void AddAssetTypeName(asset_type_t assetType, std::string name); void AddAssetTypeName(asset_type_t assetType, std::string name);
static std::vector<Gdt*> CreateGdtList(ZoneCreationContext& context); static std::vector<Gdt*> CreateGdtList(const ZoneCreationContext& context);
bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const; bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const;
void CreateZoneAssetPools(Zone* zone) const; void CreateZoneAssetPools(Zone* zone) const;
public: public:

View File

@ -21,7 +21,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name)
m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType));
} }
std::vector<Gdt*> ZoneCreator::CreateGdtList(ZoneCreationContext& context) std::vector<Gdt*> ZoneCreator::CreateGdtList(const ZoneCreationContext& context)
{ {
std::vector<Gdt*> gdtList; std::vector<Gdt*> gdtList;
gdtList.reserve(context.m_gdt_files.size()); gdtList.reserve(context.m_gdt_files.size());
@ -31,9 +31,9 @@ std::vector<Gdt*> ZoneCreator::CreateGdtList(ZoneCreationContext& context)
return gdtList; return gdtList;
} }
bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const
{ {
for (const auto& ignoreEntry : context.m_ignored_assets) for (const auto& ignoreEntry : context.m_ignored_assets.m_entries)
{ {
const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type);
if (foundAssetTypeEntry == m_asset_types_by_name.end()) if (foundAssetTypeEntry == m_asset_types_by_name.end())
@ -71,7 +71,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
if (!assetEntry.m_is_reference) if (!assetEntry.m_is_reference)
continue; continue;
context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
} }
const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context)); const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

View File

@ -12,8 +12,8 @@ namespace IW4
std::unordered_map<std::string, asset_type_t> m_asset_types_by_name; std::unordered_map<std::string, asset_type_t> m_asset_types_by_name;
void AddAssetTypeName(asset_type_t assetType, std::string name); void AddAssetTypeName(asset_type_t assetType, std::string name);
static std::vector<Gdt*> CreateGdtList(ZoneCreationContext& context); static std::vector<Gdt*> CreateGdtList(const ZoneCreationContext& context);
bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const; bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const;
void CreateZoneAssetPools(Zone* zone) const; void CreateZoneAssetPools(Zone* zone) const;
public: public:

View File

@ -21,7 +21,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name)
m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType));
} }
std::vector<Gdt*> ZoneCreator::CreateGdtList(ZoneCreationContext& context) std::vector<Gdt*> ZoneCreator::CreateGdtList(const ZoneCreationContext& context)
{ {
std::vector<Gdt*> gdtList; std::vector<Gdt*> gdtList;
gdtList.reserve(context.m_gdt_files.size()); gdtList.reserve(context.m_gdt_files.size());
@ -31,9 +31,9 @@ std::vector<Gdt*> ZoneCreator::CreateGdtList(ZoneCreationContext& context)
return gdtList; return gdtList;
} }
bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const
{ {
for (const auto& ignoreEntry : context.m_ignored_assets) for (const auto& ignoreEntry : context.m_ignored_assets.m_entries)
{ {
const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type);
if (foundAssetTypeEntry == m_asset_types_by_name.end()) if (foundAssetTypeEntry == m_asset_types_by_name.end())
@ -71,7 +71,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
if (!assetEntry.m_is_reference) if (!assetEntry.m_is_reference)
continue; continue;
context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
} }
const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context)); const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

View File

@ -12,8 +12,8 @@ namespace IW5
std::unordered_map<std::string, asset_type_t> m_asset_types_by_name; std::unordered_map<std::string, asset_type_t> m_asset_types_by_name;
void AddAssetTypeName(asset_type_t assetType, std::string name); void AddAssetTypeName(asset_type_t assetType, std::string name);
static std::vector<Gdt*> CreateGdtList(ZoneCreationContext& context); static std::vector<Gdt*> CreateGdtList(const ZoneCreationContext& context);
bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const; bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const;
void CreateZoneAssetPools(Zone* zone) const; void CreateZoneAssetPools(Zone* zone) const;
public: public:

View File

@ -32,9 +32,9 @@ std::vector<Gdt*> ZoneCreator::CreateGdtList(ZoneCreationContext& context)
return gdtList; return gdtList;
} }
bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const
{ {
for (const auto& ignoreEntry : context.m_ignored_assets) for (const auto& ignoreEntry : context.m_ignored_assets.m_entries)
{ {
const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type);
if (foundAssetTypeEntry == m_asset_types_by_name.end()) if (foundAssetTypeEntry == m_asset_types_by_name.end())
@ -72,7 +72,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
if (!assetEntry.m_is_reference) if (!assetEntry.m_is_reference)
continue; continue;
context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
} }
const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context)); const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

View File

@ -13,7 +13,7 @@ namespace T5
void AddAssetTypeName(asset_type_t assetType, std::string name); void AddAssetTypeName(asset_type_t assetType, std::string name);
static std::vector<Gdt*> CreateGdtList(ZoneCreationContext& context); static std::vector<Gdt*> CreateGdtList(ZoneCreationContext& context);
bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const; bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const;
void CreateZoneAssetPools(Zone* zone) const; void CreateZoneAssetPools(Zone* zone) const;
public: public:

View File

@ -23,7 +23,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name)
m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType));
} }
std::vector<Gdt*> ZoneCreator::CreateGdtList(ZoneCreationContext& context) std::vector<Gdt*> ZoneCreator::CreateGdtList(const ZoneCreationContext& context)
{ {
std::vector<Gdt*> gdtList; std::vector<Gdt*> gdtList;
gdtList.reserve(context.m_gdt_files.size()); gdtList.reserve(context.m_gdt_files.size());
@ -33,9 +33,9 @@ std::vector<Gdt*> ZoneCreator::CreateGdtList(ZoneCreationContext& context)
return gdtList; return gdtList;
} }
bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const
{ {
for (const auto& ignoreEntry : context.m_ignored_assets) for (const auto& ignoreEntry : context.m_ignored_assets.m_entries)
{ {
const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type);
if (foundAssetTypeEntry == m_asset_types_by_name.end()) if (foundAssetTypeEntry == m_asset_types_by_name.end())
@ -58,7 +58,7 @@ void ZoneCreator::CreateZoneAssetPools(Zone* zone) const
zone->m_pools->InitPoolDynamic(assetType); zone->m_pools->InitPoolDynamic(assetType);
} }
void ZoneCreator::HandleMetadata(Zone* zone, ZoneCreationContext& context) const void ZoneCreator::HandleMetadata(Zone* zone, const ZoneCreationContext& context) const
{ {
std::vector<KeyValuePair> kvpList; std::vector<KeyValuePair> kvpList;
@ -126,7 +126,7 @@ std::unique_ptr<Zone> ZoneCreator::CreateZoneForDefinition(ZoneCreationContext&
if (!assetEntry.m_is_reference) if (!assetEntry.m_is_reference)
continue; continue;
context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference);
} }
const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context)); const auto assetLoadingContext = std::make_unique<AssetLoadingContext>(zone.get(), context.m_asset_search_path, CreateGdtList(context));

View File

@ -12,10 +12,10 @@ namespace T6
std::unordered_map<std::string, asset_type_t> m_asset_types_by_name; std::unordered_map<std::string, asset_type_t> m_asset_types_by_name;
void AddAssetTypeName(asset_type_t assetType, std::string name); void AddAssetTypeName(asset_type_t assetType, std::string name);
static std::vector<Gdt*> CreateGdtList(ZoneCreationContext& context); static std::vector<Gdt*> CreateGdtList(const ZoneCreationContext& context);
bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const; bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map<std::string, asset_type_t>& ignoredAssetMap) const;
void CreateZoneAssetPools(Zone* zone) const; void CreateZoneAssetPools(Zone* zone) const;
void HandleMetadata(Zone* zone, ZoneCreationContext& context) const; void HandleMetadata(Zone* zone, const ZoneCreationContext& context) const;
public: public:
ZoneCreator(); ZoneCreator();

View File

@ -12,9 +12,9 @@
#include "ObjWriting.h" #include "ObjWriting.h"
#include "ObjLoading.h" #include "ObjLoading.h"
#include "SearchPath/SearchPaths.h" #include "SearchPath/SearchPaths.h"
#include "SearchPath/SearchPathFilesystem.h"
#include "ObjContainer/IWD/IWD.h" #include "ObjContainer/IWD/IWD.h"
#include "LinkerArgs.h" #include "LinkerArgs.h"
#include "LinkerSearchPaths.h"
#include "ZoneWriting.h" #include "ZoneWriting.h"
#include "Game/IW3/ZoneCreatorIW3.h" #include "Game/IW3/ZoneCreatorIW3.h"
#include "ZoneCreation/ZoneCreationContext.h" #include "ZoneCreation/ZoneCreationContext.h"
@ -23,8 +23,10 @@
#include "Game/IW5/ZoneCreatorIW5.h" #include "Game/IW5/ZoneCreatorIW5.h"
#include "Game/T5/ZoneCreatorT5.h" #include "Game/T5/ZoneCreatorT5.h"
#include "Game/T6/ZoneCreatorT6.h" #include "Game/T6/ZoneCreatorT6.h"
#include "ObjContainer/IPak/IPakWriter.h"
#include "Utils/ObjFileStream.h" #include "Utils/ObjFileStream.h"
#include "Utils/StringUtils.h"
#include "Zone/AssetList/AssetList.h" #include "Zone/AssetList/AssetList.h"
#include "Zone/AssetList/AssetListStream.h" #include "Zone/AssetList/AssetListStream.h"
#include "Zone/Definition/ZoneDefinitionStream.h" #include "Zone/Definition/ZoneDefinitionStream.h"
@ -40,195 +42,31 @@ const IZoneCreator* const ZONE_CREATORS[]
new T6::ZoneCreator() new T6::ZoneCreator()
}; };
class Linker::Impl enum class ProjectType
{
FASTFILE,
IPAK,
MAX
};
constexpr const char* PROJECT_TYPE_NAMES[static_cast<unsigned>(ProjectType::MAX)]
{
"fastfile",
"ipak"
};
class LinkerImpl final : public Linker
{ {
static constexpr const char* METADATA_NAME = "name";
static constexpr const char* METADATA_GAME = "game"; static constexpr const char* METADATA_GAME = "game";
static constexpr const char* METADATA_GDT = "gdt"; static constexpr const char* METADATA_GDT = "gdt";
static constexpr const char* METADATA_NAME = "name";
static constexpr const char* METADATA_TYPE = "type";
LinkerArgs m_args; LinkerArgs m_args;
std::vector<std::unique_ptr<ISearchPath>> m_loaded_project_search_paths; LinkerSearchPaths m_search_paths;
SearchPaths m_asset_search_paths;
SearchPaths m_gdt_search_paths;
SearchPaths m_source_search_paths;
std::vector<std::unique_ptr<Zone>> m_loaded_zones; std::vector<std::unique_ptr<Zone>> m_loaded_zones;
/**
* \brief Loads a search path.
* \param searchPath The search path to load.
*/
void LoadSearchPath(ISearchPath* searchPath) const
{
if (m_args.m_verbose)
{
printf("Loading search path: \"%s\"\n", searchPath->GetPath().c_str());
}
ObjLoading::LoadIWDsInSearchPath(searchPath);
}
/**
* \brief Unloads a search path.
* \param searchPath The search path to unload.
*/
void UnloadSearchPath(ISearchPath* searchPath) const
{
if (m_args.m_verbose)
{
printf("Unloading search path: \"%s\"\n", searchPath->GetPath().c_str());
}
ObjLoading::UnloadIWDsInSearchPath(searchPath);
}
SearchPaths GetAssetSearchPathsForProject(const std::string& gameName, const std::string& projectName)
{
SearchPaths searchPathsForProject;
for (const auto& searchPathStr : m_args.GetAssetSearchPathsForProject(gameName, projectName))
{
auto absolutePath = fs::absolute(searchPathStr);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding asset search path: " << absolutePath.string() << std::endl;
auto searchPath = std::make_unique<SearchPathFilesystem>(searchPathStr);
LoadSearchPath(searchPath.get());
searchPathsForProject.IncludeSearchPath(searchPath.get());
m_loaded_project_search_paths.emplace_back(std::move(searchPath));
}
searchPathsForProject.IncludeSearchPath(&m_asset_search_paths);
for (auto* iwd : IWD::Repository)
{
searchPathsForProject.IncludeSearchPath(iwd);
}
return searchPathsForProject;
}
SearchPaths GetGdtSearchPathsForProject(const std::string& gameName, const std::string& projectName)
{
SearchPaths searchPathsForProject;
for (const auto& searchPathStr : m_args.GetGdtSearchPathsForProject(gameName, projectName))
{
auto absolutePath = fs::absolute(searchPathStr);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Adding gdt search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl;
searchPathsForProject.CommitSearchPath(std::make_unique<SearchPathFilesystem>(searchPathStr));
}
searchPathsForProject.IncludeSearchPath(&m_gdt_search_paths);
return searchPathsForProject;
}
SearchPaths GetSourceSearchPathsForProject(const std::string& projectName)
{
SearchPaths searchPathsForProject;
for (const auto& searchPathStr : m_args.GetSourceSearchPathsForProject(projectName))
{
auto absolutePath = fs::absolute(searchPathStr);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Adding source search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding source search path: " << absolutePath.string() << std::endl;
searchPathsForProject.CommitSearchPath(std::make_unique<SearchPathFilesystem>(searchPathStr));
}
searchPathsForProject.IncludeSearchPath(&m_source_search_paths);
return searchPathsForProject;
}
/**
* \brief Initializes the Linker object's search paths based on the user's input.
* \return \c true if building the search paths was successful, otherwise \c false.
*/
bool BuildProjectIndependentSearchPaths()
{
for (const auto& path : m_args.GetProjectIndependentAssetSearchPaths())
{
auto absolutePath = fs::absolute(path);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding asset search path: " << absolutePath.string() << std::endl;
auto searchPath = std::make_unique<SearchPathFilesystem>(absolutePath.string());
LoadSearchPath(searchPath.get());
m_asset_search_paths.CommitSearchPath(std::move(searchPath));
}
for (const auto& path : m_args.GetProjectIndependentGdtSearchPaths())
{
auto absolutePath = fs::absolute(path);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Loading gdt search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl;
m_gdt_search_paths.CommitSearchPath(std::make_unique<SearchPathFilesystem>(absolutePath.string()));
}
for (const auto& path : m_args.GetProjectIndependentSourceSearchPaths())
{
auto absolutePath = fs::absolute(path);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Loading source search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding source search path: " << absolutePath.string() << std::endl;
m_source_search_paths.CommitSearchPath(std::make_unique<SearchPathFilesystem>(absolutePath.string()));
}
return true;
}
bool IncludeAdditionalZoneDefinitions(const std::string& initialFileName, ZoneDefinition& zoneDefinition, ISearchPath* sourceSearchPath) const bool IncludeAdditionalZoneDefinitions(const std::string& initialFileName, ZoneDefinition& zoneDefinition, ISearchPath* sourceSearchPath) const
{ {
std::set<std::string> sourceNames; std::set<std::string> sourceNames;
@ -278,6 +116,58 @@ class Linker::Impl
return true; return true;
} }
bool ReadAssetList(const std::string& zoneName, AssetList& assetList, ISearchPath* sourceSearchPath) const
{
{
const auto assetListFileName = "assetlist/" + zoneName + ".csv";
const auto assetListStream = sourceSearchPath->Open(assetListFileName);
if (assetListStream.IsOpen())
{
const AssetListInputStream stream(*assetListStream.m_stream);
AssetListEntry entry;
while (stream.NextEntry(entry))
{
assetList.m_entries.emplace_back(std::move(entry));
}
return true;
}
}
{
const auto zoneDefinition = ReadZoneDefinition(zoneName, sourceSearchPath);
if (zoneDefinition)
{
for (const auto& entry : zoneDefinition->m_assets)
{
assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name, entry.m_is_reference);
}
return true;
}
}
return false;
}
bool IncludeAssetLists(ZoneDefinition& zoneDefinition, ISearchPath* sourceSearchPath) const
{
for (const auto& assetListName : zoneDefinition.m_asset_lists)
{
AssetList assetList;
if (!ReadAssetList(assetListName, assetList, sourceSearchPath))
{
std::cerr << "Failed to read asset list \"" << assetListName << "\"\n";
return false;
}
zoneDefinition.Include(assetList);
}
return true;
}
static bool GetNameFromZoneDefinition(std::string& name, const std::string& projectName, const ZoneDefinition& zoneDefinition) static bool GetNameFromZoneDefinition(std::string& name, const std::string& projectName, const ZoneDefinition& zoneDefinition)
{ {
auto firstNameEntry = true; auto firstNameEntry = true;
@ -333,44 +223,12 @@ class Linker::Impl
if (!IncludeAdditionalZoneDefinitions(projectName, *zoneDefinition, sourceSearchPath)) if (!IncludeAdditionalZoneDefinitions(projectName, *zoneDefinition, sourceSearchPath))
return nullptr; return nullptr;
if (!IncludeAssetLists(*zoneDefinition, sourceSearchPath))
return nullptr;
return zoneDefinition; return zoneDefinition;
} }
bool ReadAssetList(const std::string& zoneName, std::vector<AssetListEntry>& assetList, ISearchPath* sourceSearchPath) const
{
{
const auto assetListFileName = "assetlist/" + zoneName + ".csv";
const auto assetListStream = sourceSearchPath->Open(assetListFileName);
if (assetListStream.IsOpen())
{
const AssetListInputStream stream(*assetListStream.m_stream);
AssetListEntry entry;
while (stream.NextEntry(entry))
{
assetList.emplace_back(std::move(entry));
}
return true;
}
}
{
const auto zoneDefinition = ReadZoneDefinition(zoneName, sourceSearchPath);
if (zoneDefinition)
{
for (const auto& entry : zoneDefinition->m_assets)
{
assetList.emplace_back(entry.m_asset_type, entry.m_asset_name);
}
return true;
}
}
return false;
}
bool ProcessZoneDefinitionIgnores(const std::string& projectName, ZoneCreationContext& context, ISearchPath* sourceSearchPath) const bool ProcessZoneDefinitionIgnores(const std::string& projectName, ZoneCreationContext& context, ISearchPath* sourceSearchPath) const
{ {
if (context.m_definition->m_ignores.empty()) if (context.m_definition->m_ignores.empty())
@ -397,6 +255,55 @@ class Linker::Impl
return true; return true;
} }
static bool ProjectTypeByName(ProjectType& projectType, const std::string& projectTypeName)
{
for (auto i = 0u; i < static_cast<unsigned>(ProjectType::MAX); i++)
{
if (projectTypeName == PROJECT_TYPE_NAMES[i])
{
projectType = static_cast<ProjectType>(i);
return true;
}
}
return false;
}
static bool GetProjectTypeFromZoneDefinition(ProjectType& projectType, const std::string& projectName, const ZoneDefinition& zoneDefinition)
{
auto firstGameEntry = true;
const auto [rangeBegin, rangeEnd] = zoneDefinition.m_metadata_lookup.equal_range(METADATA_TYPE);
for (auto i = rangeBegin; i != rangeEnd; ++i)
{
ProjectType parsedProjectType;
if (!ProjectTypeByName(parsedProjectType, i->second->m_value))
{
std::cerr << "Not a valid project type: \"" << i->second->m_value << "\"\n";
return false;
}
if (firstGameEntry)
{
projectType = parsedProjectType;
firstGameEntry = false;
}
else
{
if (projectType != parsedProjectType)
{
std::cerr << "Conflicting types in project \"" << projectName << "\": " << PROJECT_TYPE_NAMES[static_cast<unsigned>(projectType)]
<< " != " << PROJECT_TYPE_NAMES[static_cast<unsigned>(parsedProjectType)] << std::endl;
return false;
}
}
}
if (firstGameEntry)
projectType = ProjectType::FASTFILE;
return true;
}
static bool GetGameNameFromZoneDefinition(std::string& gameName, const std::string& projectName, const ZoneDefinition& zoneDefinition) static bool GetGameNameFromZoneDefinition(std::string& gameName, const std::string& projectName, const ZoneDefinition& zoneDefinition)
{ {
auto firstGameEntry = true; auto firstGameEntry = true;
@ -498,34 +405,88 @@ class Linker::Impl
return true; return true;
} }
bool BuildFastFile(const std::string& projectName, ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& gdtSearchPaths, SearchPaths& sourceSearchPaths) const
{
const auto zone = CreateZoneForDefinition(projectName, zoneDefinition, &assetSearchPaths, &gdtSearchPaths, &sourceSearchPaths);
auto result = zone != nullptr;
if (zone)
result = WriteZoneToFile(projectName, zone.get());
return result;
}
bool BuildIPak(const std::string& projectName, const ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& sourceSearchPaths) const
{
const fs::path ipakFolderPath(m_args.GetOutputFolderPathForProject(projectName));
auto ipakFilePath(ipakFolderPath);
ipakFilePath.append(zoneDefinition.m_name + ".ipak");
fs::create_directories(ipakFolderPath);
std::ofstream stream(ipakFilePath, std::fstream::out | std::fstream::binary);
if (!stream.is_open())
return false;
const auto ipakWriter = IPakWriter::Create(stream, &assetSearchPaths);
for (const auto& assetEntry : zoneDefinition.m_assets)
{
if (assetEntry.m_is_reference)
continue;
if (assetEntry.m_asset_type == "image")
ipakWriter->AddImage(assetEntry.m_asset_name);
}
if (!ipakWriter->Write())
{
std::cout << "Writing ipak failed." << std::endl;
stream.close();
return false;
}
std::cout << "Created ipak \"" << ipakFilePath.string() << "\"\n";
stream.close();
return true;
}
bool BuildProject(const std::string& projectName) bool BuildProject(const std::string& projectName)
{ {
auto sourceSearchPaths = GetSourceSearchPathsForProject(projectName); auto sourceSearchPaths = m_search_paths.GetSourceSearchPathsForProject(projectName);
const auto zoneDefinition = ReadZoneDefinition(projectName, &sourceSearchPaths); const auto zoneDefinition = ReadZoneDefinition(projectName, &sourceSearchPaths);
if (!zoneDefinition) if (!zoneDefinition)
return false; return false;
ProjectType projectType;
if (!GetProjectTypeFromZoneDefinition(projectType, projectName, *zoneDefinition))
return false;
std::string gameName; std::string gameName;
if (!GetGameNameFromZoneDefinition(gameName, projectName, *zoneDefinition)) if (!GetGameNameFromZoneDefinition(gameName, projectName, *zoneDefinition))
return false; return false;
utils::MakeStringLowerCase(gameName);
for (auto& c : gameName) auto assetSearchPaths = m_search_paths.GetAssetSearchPathsForProject(gameName, projectName);
c = static_cast<char>(std::tolower(c)); auto gdtSearchPaths = m_search_paths.GetGdtSearchPathsForProject(gameName, projectName);
auto assetSearchPaths = GetAssetSearchPathsForProject(gameName, projectName); auto result = false;
auto gdtSearchPaths = GetGdtSearchPathsForProject(gameName, projectName); switch (projectType)
const auto zone = CreateZoneForDefinition(projectName, *zoneDefinition, &assetSearchPaths, &gdtSearchPaths, &sourceSearchPaths);
auto result = zone != nullptr;
if (zone)
result = WriteZoneToFile(projectName, zone.get());
for (const auto& loadedSearchPath : m_loaded_project_search_paths)
{ {
UnloadSearchPath(loadedSearchPath.get()); case ProjectType::FASTFILE:
result = BuildFastFile(projectName, *zoneDefinition, assetSearchPaths, gdtSearchPaths, sourceSearchPaths);
break;
case ProjectType::IPAK:
result = BuildIPak(projectName, *zoneDefinition, assetSearchPaths, sourceSearchPaths);
break;
default:
assert(false);
break;
} }
m_loaded_project_search_paths.clear();
m_search_paths.UnloadProjectSpecificSearchPaths();
return result; return result;
} }
@ -576,18 +537,17 @@ class Linker::Impl
} }
public: public:
Impl() LinkerImpl()
= default; : m_search_paths(m_args)
{
}
/** bool Start(const int argc, const char** argv) override
* \copydoc Linker::Start
*/
bool Start(const int argc, const char** argv)
{ {
if (!m_args.ParseArgs(argc, argv)) if (!m_args.ParseArgs(argc, argv))
return false; return false;
if (!BuildProjectIndependentSearchPaths()) if (!m_search_paths.BuildProjectIndependentSearchPaths())
return false; return false;
if (!LoadZones()) if (!LoadZones())
@ -609,18 +569,7 @@ public:
} }
}; };
Linker::Linker() std::unique_ptr<Linker> Linker::Create()
{ {
m_impl = new Impl(); return std::make_unique<LinkerImpl>();
}
Linker::~Linker()
{
delete m_impl;
m_impl = nullptr;
}
bool Linker::Start(const int argc, const char** argv) const
{
return m_impl->Start(argc, argv);
} }

View File

@ -1,13 +1,11 @@
#pragma once #pragma once
#include <memory>
class Linker class Linker
{ {
class Impl;
Impl* m_impl;
public: public:
Linker(); Linker() = default;
~Linker(); virtual ~Linker() = default;
Linker(const Linker& other) = delete; Linker(const Linker& other) = delete;
Linker(Linker&& other) noexcept = delete; Linker(Linker&& other) noexcept = delete;
@ -20,5 +18,7 @@ public:
* \param argv The command line arguments. * \param argv The command line arguments.
* \return \c true if the application was successful or \c false if an error occurred. * \return \c true if the application was successful or \c false if an error occurred.
*/ */
bool Start(int argc, const char** argv) const; virtual bool Start(int argc, const char** argv) = 0;
static std::unique_ptr<Linker> Create();
}; };

View File

@ -0,0 +1,190 @@
#include "LinkerSearchPaths.h"
#include <filesystem>
#include <iostream>
#include "ObjLoading.h"
#include "ObjContainer/IWD/IWD.h"
#include "SearchPath/SearchPathFilesystem.h"
namespace fs = std::filesystem;
LinkerSearchPaths::LinkerSearchPaths(const LinkerArgs& args)
: m_args(args)
{
}
void LinkerSearchPaths::LoadSearchPath(ISearchPath* searchPath) const
{
if (m_args.m_verbose)
{
printf("Loading search path: \"%s\"\n", searchPath->GetPath().c_str());
}
ObjLoading::LoadIWDsInSearchPath(searchPath);
}
void LinkerSearchPaths::UnloadSearchPath(ISearchPath* searchPath) const
{
if (m_args.m_verbose)
{
printf("Unloading search path: \"%s\"\n", searchPath->GetPath().c_str());
}
ObjLoading::UnloadIWDsInSearchPath(searchPath);
}
SearchPaths LinkerSearchPaths::GetAssetSearchPathsForProject(const std::string& gameName, const std::string& projectName)
{
SearchPaths searchPathsForProject;
for (const auto& searchPathStr : m_args.GetAssetSearchPathsForProject(gameName, projectName))
{
auto absolutePath = fs::absolute(searchPathStr);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding asset search path: " << absolutePath.string() << std::endl;
auto searchPath = std::make_unique<SearchPathFilesystem>(searchPathStr);
LoadSearchPath(searchPath.get());
searchPathsForProject.IncludeSearchPath(searchPath.get());
m_loaded_project_search_paths.emplace_back(std::move(searchPath));
}
searchPathsForProject.IncludeSearchPath(&m_asset_search_paths);
for (auto* iwd : IWD::Repository)
{
searchPathsForProject.IncludeSearchPath(iwd);
}
return searchPathsForProject;
}
SearchPaths LinkerSearchPaths::GetGdtSearchPathsForProject(const std::string& gameName, const std::string& projectName)
{
SearchPaths searchPathsForProject;
for (const auto& searchPathStr : m_args.GetGdtSearchPathsForProject(gameName, projectName))
{
auto absolutePath = fs::absolute(searchPathStr);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Adding gdt search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl;
searchPathsForProject.CommitSearchPath(std::make_unique<SearchPathFilesystem>(searchPathStr));
}
searchPathsForProject.IncludeSearchPath(&m_gdt_search_paths);
return searchPathsForProject;
}
SearchPaths LinkerSearchPaths::GetSourceSearchPathsForProject(const std::string& projectName)
{
SearchPaths searchPathsForProject;
for (const auto& searchPathStr : m_args.GetSourceSearchPathsForProject(projectName))
{
auto absolutePath = fs::absolute(searchPathStr);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Adding source search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding source search path: " << absolutePath.string() << std::endl;
searchPathsForProject.CommitSearchPath(std::make_unique<SearchPathFilesystem>(searchPathStr));
}
searchPathsForProject.IncludeSearchPath(&m_source_search_paths);
return searchPathsForProject;
}
bool LinkerSearchPaths::BuildProjectIndependentSearchPaths()
{
for (const auto& path : m_args.GetProjectIndependentAssetSearchPaths())
{
auto absolutePath = fs::absolute(path);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding asset search path: " << absolutePath.string() << std::endl;
auto searchPath = std::make_unique<SearchPathFilesystem>(absolutePath.string());
LoadSearchPath(searchPath.get());
m_asset_search_paths.CommitSearchPath(std::move(searchPath));
}
for (const auto& path : m_args.GetProjectIndependentGdtSearchPaths())
{
auto absolutePath = fs::absolute(path);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Loading gdt search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl;
m_gdt_search_paths.CommitSearchPath(std::make_unique<SearchPathFilesystem>(absolutePath.string()));
}
for (const auto& path : m_args.GetProjectIndependentSourceSearchPaths())
{
auto absolutePath = fs::absolute(path);
if (!fs::is_directory(absolutePath))
{
if (m_args.m_verbose)
std::cout << "Loading source search path (Not found): " << absolutePath.string() << std::endl;
continue;
}
if (m_args.m_verbose)
std::cout << "Adding source search path: " << absolutePath.string() << std::endl;
m_source_search_paths.CommitSearchPath(std::make_unique<SearchPathFilesystem>(absolutePath.string()));
}
return true;
}
void LinkerSearchPaths::UnloadProjectSpecificSearchPaths()
{
for (const auto& loadedSearchPath : m_loaded_project_search_paths)
{
UnloadSearchPath(loadedSearchPath.get());
}
m_loaded_project_search_paths.clear();
}

View File

@ -0,0 +1,46 @@
#pragma once
#include "SearchPath/SearchPaths.h"
#include <memory>
#include <vector>
#include "LinkerArgs.h"
class LinkerSearchPaths
{
public:
explicit LinkerSearchPaths(const LinkerArgs& args);
/**
* \brief Loads a search path.
* \param searchPath The search path to load.
*/
void LoadSearchPath(ISearchPath* searchPath) const;
/**
* \brief Unloads a search path.
* \param searchPath The search path to unload.
*/
void UnloadSearchPath(ISearchPath* searchPath) const;
SearchPaths GetAssetSearchPathsForProject(const std::string& gameName, const std::string& projectName);
SearchPaths GetGdtSearchPathsForProject(const std::string& gameName, const std::string& projectName);
SearchPaths GetSourceSearchPathsForProject(const std::string& projectName);
/**
* \brief Initializes the Linker object's search paths based on the user's input.
* \return \c true if building the search paths was successful, otherwise \c false.
*/
bool BuildProjectIndependentSearchPaths();
void UnloadProjectSpecificSearchPaths();
private:
const LinkerArgs& m_args;
std::vector<std::unique_ptr<ISearchPath>> m_loaded_project_search_paths;
SearchPaths m_asset_search_paths;
SearchPaths m_gdt_search_paths;
SearchPaths m_source_search_paths;
};

View File

@ -2,7 +2,6 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include <memory> #include <memory>
#include <unordered_map>
#include "SearchPath/ISearchPath.h" #include "SearchPath/ISearchPath.h"
#include "Obj/Gdt/Gdt.h" #include "Obj/Gdt/Gdt.h"
@ -16,7 +15,7 @@ public:
ISearchPath* m_asset_search_path; ISearchPath* m_asset_search_path;
ZoneDefinition* m_definition; ZoneDefinition* m_definition;
std::vector<std::unique_ptr<Gdt>> m_gdt_files; std::vector<std::unique_ptr<Gdt>> m_gdt_files;
std::vector<AssetListEntry> m_ignored_assets; AssetList m_ignored_assets;
ZoneCreationContext(); ZoneCreationContext();
ZoneCreationContext(ISearchPath* assetSearchPath, ZoneDefinition* definition); ZoneCreationContext(ISearchPath* assetSearchPath, ZoneDefinition* definition);

View File

@ -2,7 +2,7 @@
int main(const int argc, const char** argv) int main(const int argc, const char** argv)
{ {
Linker linker; const auto linker = Linker::Create();
return linker.Start(argc, argv) ? 0 : 1; return linker->Start(argc, argv) ? 0 : 1;
} }

View File

@ -2,14 +2,30 @@
#include <cstdint> #include <cstdint>
typedef uint32_t IPakHash; #include "Utils/FileUtils.h"
namespace ipak_consts namespace ipak_consts
{ {
static constexpr uint32_t IPAK_MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I');
static constexpr uint32_t IPAK_VERSION = 0x50000;
static constexpr uint32_t IPAK_INDEX_SECTION = 1;
static constexpr uint32_t IPAK_DATA_SECTION = 2;
static constexpr uint32_t IPAK_BRANDING_SECTION = FileUtils::MakeMagic32('M', 'E', 'T', 'A');
static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; static constexpr size_t IPAK_CHUNK_SIZE = 0x8000;
static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8;
static constexpr uint32_t IPAK_COMMAND_DEFAULT_SIZE = 0x7F00;
static constexpr uint32_t IPAK_COMMAND_UNCOMPRESSED = 0;
static constexpr uint32_t IPAK_COMMAND_COMPRESSED = 1;
static constexpr uint32_t IPAK_COMMAND_SKIP = 0xCF;
static_assert(IPAK_COMMAND_DEFAULT_SIZE <= IPAK_CHUNK_SIZE);
} }
typedef uint32_t IPakHash;
struct IPakHeader struct IPakHeader
{ {
uint32_t magic; uint32_t magic;
@ -33,6 +49,7 @@ union IPakIndexEntryKey
IPakHash dataHash; IPakHash dataHash;
IPakHash nameHash; IPakHash nameHash;
}; };
uint64_t combinedKey; uint64_t combinedKey;
}; };
@ -43,24 +60,26 @@ struct IPakIndexEntry
uint32_t size; uint32_t size;
}; };
struct IPakDataBlockHeader struct IPakDataBlockCountAndOffset
{
union
{
uint32_t countAndOffset;
struct
{ {
uint32_t offset : 24; uint32_t offset : 24;
uint32_t count : 8; uint32_t count : 8;
}; };
};
union static_assert(sizeof(IPakDataBlockCountAndOffset) == 4);
{
uint32_t commands[31]; struct IPakDataBlockCommand
struct
{ {
uint32_t size : 24; uint32_t size : 24;
uint32_t compressed : 8; uint32_t compressed : 8;
}_commands[31];
}; };
static_assert(sizeof(IPakDataBlockCommand) == 4);
struct IPakDataBlockHeader
{
IPakDataBlockCountAndOffset countAndOffset;
IPakDataBlockCommand commands[31];
}; };
static_assert(sizeof(IPakDataBlockHeader) == 128);

View File

@ -0,0 +1,71 @@
#include "AssetLoaderGfxImage.h"
#include <cstring>
#include <iostream>
#include <sstream>
#include <zlib.h>
#include "Game/T6/CommonT6.h"
#include "Game/T6/T6.h"
#include "Image/IwiLoader.h"
#include "Pool/GlobalAssetPool.h"
using namespace T6;
void* AssetLoaderGfxImage::CreateEmptyAsset(const std::string& assetName, MemoryManager* memory)
{
auto* image = memory->Create<GfxImage>();
memset(image, 0, sizeof(GfxImage));
image->name = memory->Dup(assetName.c_str());
return image;
}
bool AssetLoaderGfxImage::CanLoadFromRaw() const
{
return true;
}
bool AssetLoaderGfxImage::LoadFromRaw(const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const
{
const auto fileName = "images/" + assetName + ".iwi";
const auto file = searchPath->Open(fileName);
if (!file.IsOpen())
return false;
const auto fileSize = static_cast<size_t>(file.m_length);
const auto fileData = std::make_unique<char[]>(fileSize);
file.m_stream->read(fileData.get(), fileSize);
const auto dataHash = static_cast<unsigned>(crc32(0u, reinterpret_cast<const Bytef*>(fileData.get()), fileSize));
MemoryManager tempMemory;
IwiLoader iwiLoader(&tempMemory);
std::istringstream ss(std::string(fileData.get(), fileSize));
const auto texture = iwiLoader.LoadIwi(ss);
if (!texture)
{
std::cerr << "Failed to load texture from: " << fileName << "\n";
return false;
}
auto* image = memory->Create<GfxImage>();
memset(image, 0, sizeof(GfxImage));
image->name = memory->Dup(assetName.c_str());
image->hash = Common::R_HashString(image->name, 0);
image->delayLoadPixels = true;
image->noPicmip = !texture->HasMipMaps();
image->width = static_cast<uint16_t>(texture->GetWidth());
image->height = static_cast<uint16_t>(texture->GetHeight());
image->depth = static_cast<uint16_t>(texture->GetDepth());
image->streaming = 1;
image->streamedParts[0].levelCount = 1;
image->streamedParts[0].levelSize = static_cast<uint32_t>(fileSize);
image->streamedParts[0].hash = dataHash & 0x1FFFFFFF;
image->streamedPartCount = 1;
manager->AddAsset(ASSET_TYPE_IMAGE, assetName, image);
return true;
}

View File

@ -0,0 +1,16 @@
#pragma once
#include "Game/T6/T6.h"
#include "AssetLoading/BasicAssetLoader.h"
#include "AssetLoading/IAssetLoadingManager.h"
#include "SearchPath/ISearchPath.h"
namespace T6
{
class AssetLoaderGfxImage final : public BasicAssetLoader<ASSET_TYPE_IMAGE, GfxImage>
{
public:
_NODISCARD void* CreateEmptyAsset(const std::string& assetName, MemoryManager* memory) override;
_NODISCARD bool CanLoadFromRaw() const override;
bool LoadFromRaw(const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const override;
};
}

View File

@ -7,6 +7,7 @@
#include "ObjContainer/IPak/IPak.h" #include "ObjContainer/IPak/IPak.h"
#include "ObjLoading.h" #include "ObjLoading.h"
#include "AssetLoaders/AssetLoaderFontIcon.h" #include "AssetLoaders/AssetLoaderFontIcon.h"
#include "AssetLoaders/AssetLoaderGfxImage.h"
#include "AssetLoaders/AssetLoaderLocalizeEntry.h" #include "AssetLoaders/AssetLoaderLocalizeEntry.h"
#include "AssetLoaders/AssetLoaderPhysConstraints.h" #include "AssetLoaders/AssetLoaderPhysConstraints.h"
#include "AssetLoaders/AssetLoaderPhysPreset.h" #include "AssetLoaders/AssetLoaderPhysPreset.h"
@ -45,7 +46,7 @@ namespace T6
REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_XMODEL, XModel)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_XMODEL, XModel))
REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_MATERIAL, Material)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_MATERIAL, Material))
REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_TECHNIQUE_SET, MaterialTechniqueSet)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_TECHNIQUE_SET, MaterialTechniqueSet))
REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_IMAGE, GfxImage)) REGISTER_ASSET_LOADER(AssetLoaderGfxImage)
REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_SOUND, SndBank)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_SOUND, SndBank))
REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_SOUND_PATCH, SndPatch)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_SOUND_PATCH, SndPatch))
REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_CLIPMAP, clipMap_t)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_CLIPMAP, clipMap_t))

View File

@ -18,9 +18,6 @@ ObjContainerRepository<IPak, Zone> IPak::Repository;
class IPak::Impl : public ObjContainerReferenceable class IPak::Impl : public ObjContainerReferenceable
{ {
static const uint32_t MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I');
static const uint32_t VERSION = 0x50000;
std::string m_path; std::string m_path;
std::unique_ptr<std::istream> m_stream; std::unique_ptr<std::istream> m_stream;
@ -73,8 +70,8 @@ class IPak::Impl : public ObjContainerReferenceable
{ {
IPakSection section{}; IPakSection section{};
m_stream->read(reinterpret_cast<char*>(&section), sizeof section); m_stream->read(reinterpret_cast<char*>(&section), sizeof(section));
if (m_stream->gcount() != sizeof section) if (m_stream->gcount() != sizeof(section))
{ {
printf("Unexpected eof when trying to load section.\n"); printf("Unexpected eof when trying to load section.\n");
return false; return false;
@ -82,11 +79,11 @@ class IPak::Impl : public ObjContainerReferenceable
switch (section.type) switch (section.type)
{ {
case 1: case ipak_consts::IPAK_INDEX_SECTION:
m_index_section = std::make_unique<IPakSection>(section); m_index_section = std::make_unique<IPakSection>(section);
break; break;
case 2: case ipak_consts::IPAK_DATA_SECTION:
m_data_section = std::make_unique<IPakSection>(section); m_data_section = std::make_unique<IPakSection>(section);
break; break;
@ -101,20 +98,20 @@ class IPak::Impl : public ObjContainerReferenceable
{ {
IPakHeader header{}; IPakHeader header{};
m_stream->read(reinterpret_cast<char*>(&header), sizeof header); m_stream->read(reinterpret_cast<char*>(&header), sizeof(header));
if (m_stream->gcount() != sizeof header) if (m_stream->gcount() != sizeof header)
{ {
printf("Unexpected eof when trying to load header.\n"); printf("Unexpected eof when trying to load header.\n");
return false; return false;
} }
if (header.magic != MAGIC) if (header.magic != ipak_consts::IPAK_MAGIC)
{ {
printf("Invalid ipak magic '0x%x'.\n", header.magic); printf("Invalid ipak magic '0x%x'.\n", header.magic);
return false; return false;
} }
if (header.version != VERSION) if (header.version != ipak_consts::IPAK_VERSION)
{ {
printf("Unsupported ipak version '%u'.\n", header.version); printf("Unsupported ipak version '%u'.\n", header.version);
return false; return false;

View File

@ -40,9 +40,11 @@ IPakEntryReadStream::~IPakEntryReadStream()
size_t IPakEntryReadStream::ReadChunks(uint8_t* buffer, const int64_t startPos, const size_t chunkCount) const size_t IPakEntryReadStream::ReadChunks(uint8_t* buffer, const int64_t startPos, const size_t chunkCount) const
{ {
m_stream_manager_actions->StartReading(); m_stream_manager_actions->StartReading();
m_stream.seekg(startPos); m_stream.seekg(startPos);
m_stream.read(reinterpret_cast<char*>(buffer), static_cast<std::streamsize>(chunkCount) * IPAK_CHUNK_SIZE); m_stream.read(reinterpret_cast<char*>(buffer), static_cast<std::streamsize>(chunkCount) * IPAK_CHUNK_SIZE);
const auto readSize = static_cast<size_t>(m_stream.gcount()); const auto readSize = static_cast<size_t>(m_stream.gcount());
m_stream_manager_actions->StopReading(); m_stream_manager_actions->StopReading();
return readSize / IPAK_CHUNK_SIZE; return readSize / IPAK_CHUNK_SIZE;
@ -52,7 +54,6 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch
{ {
// Cannot load more than IPAK_CHUNK_COUNT_PER_READ chunks without overflowing the buffer // Cannot load more than IPAK_CHUNK_COUNT_PER_READ chunks without overflowing the buffer
assert(chunkCount <= IPAK_CHUNK_COUNT_PER_READ); assert(chunkCount <= IPAK_CHUNK_COUNT_PER_READ);
if (chunkCount > IPAK_CHUNK_COUNT_PER_READ) if (chunkCount > IPAK_CHUNK_COUNT_PER_READ)
chunkCount = IPAK_CHUNK_COUNT_PER_READ; chunkCount = IPAK_CHUNK_COUNT_PER_READ;
@ -68,8 +69,11 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch
const auto endPos = startPos + static_cast<int64_t>(chunkCount) * IPAK_CHUNK_SIZE; const auto endPos = startPos + static_cast<int64_t>(chunkCount) * IPAK_CHUNK_SIZE;
// Check whether the start position is already part of the loaded data
// We might be able to reuse previously loaded data
if (startPos >= m_buffer_start_pos && startPos < m_buffer_end_pos) if (startPos >= m_buffer_start_pos && startPos < m_buffer_end_pos)
{ {
// Check whether we need to move data from inside the buffer to the start to account for new start
if (m_buffer_start_pos != startPos) if (m_buffer_start_pos != startPos)
{ {
const auto moveEnd = endPos < m_buffer_end_pos ? endPos : m_buffer_end_pos; const auto moveEnd = endPos < m_buffer_end_pos ? endPos : m_buffer_end_pos;
@ -78,6 +82,7 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch
m_buffer_start_pos = startPos; m_buffer_start_pos = startPos;
} }
// Check whether we need to load additional data that was not previously loaded
if (endPos > m_buffer_end_pos) if (endPos > m_buffer_end_pos)
{ {
const auto readChunkCount = ReadChunks(&m_chunk_buffer[m_buffer_end_pos - startPos], m_buffer_end_pos, const auto readChunkCount = ReadChunks(&m_chunk_buffer[m_buffer_end_pos - startPos], m_buffer_end_pos,
@ -92,11 +97,15 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch
return true; return true;
} }
// Check whether the end position is already part of the loaded data
if (endPos > m_buffer_start_pos && endPos <= m_buffer_end_pos) if (endPos > m_buffer_start_pos && endPos <= m_buffer_end_pos)
{ {
assert(IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ - static_cast<size_t>(m_buffer_start_pos - startPos) >= static_cast<size_t>(endPos - m_buffer_start_pos)); assert(IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ - static_cast<size_t>(m_buffer_start_pos - startPos) >= static_cast<size_t>(endPos - m_buffer_start_pos));
// Move data to make sure the end is at the appropriate position to be able to load the missing data in the front
memmove(&m_chunk_buffer[m_buffer_start_pos - startPos], m_chunk_buffer, static_cast<size_t>(endPos - m_buffer_start_pos)); memmove(&m_chunk_buffer[m_buffer_start_pos - startPos], m_chunk_buffer, static_cast<size_t>(endPos - m_buffer_start_pos));
// We already established that the start of the buffer is not already loaded so we will need to load additional data nonetheless
const auto readChunkCount = ReadChunks(m_chunk_buffer, const auto readChunkCount = ReadChunks(m_chunk_buffer,
startPos, startPos,
static_cast<size_t>(m_buffer_start_pos - startPos) / IPAK_CHUNK_SIZE); static_cast<size_t>(m_buffer_start_pos - startPos) / IPAK_CHUNK_SIZE);
@ -109,6 +118,7 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch
return m_buffer_end_pos == endPos; return m_buffer_end_pos == endPos;
} }
// None of the data needed is already loaded -> Load everything and do not reuse any previously loaded data
const auto readChunkCount = ReadChunks(m_chunk_buffer, startPos, chunkCount); const auto readChunkCount = ReadChunks(m_chunk_buffer, startPos, chunkCount);
m_buffer_start_pos = startPos; m_buffer_start_pos = startPos;
@ -117,22 +127,26 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch
return chunkCount == readChunkCount; return chunkCount == readChunkCount;
} }
bool IPakEntryReadStream::ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const bool IPakEntryReadStream::ValidateBlockHeader(const IPakDataBlockHeader* blockHeader) const
{ {
if (blockHeader->count > 31) if (blockHeader->countAndOffset.count > 31)
{ {
printf("IPak block has more than 31 commands: %u -> Invalid\n", blockHeader->count); std::cerr << "IPak block has more than 31 commands: " << blockHeader->countAndOffset.count << " -> Invalid\n";
return false; return false;
} }
if (blockHeader->offset != m_file_head)
// We expect the current file to be continued where we left off
if (blockHeader->countAndOffset.offset != m_file_head)
{ {
// A matching offset is only relevant if a command contains data. // A matching offset is only relevant if a command contains data
for (unsigned currentCommand = 0; currentCommand < blockHeader->count; currentCommand++) for (unsigned currentCommand = 0; currentCommand < blockHeader->countAndOffset.count; currentCommand++)
{ {
if (blockHeader->_commands[currentCommand].compressed == 0 // If compressed is not 0 or 1 it will not be read and therefore it is okay when the offset does not match
|| blockHeader->_commands[currentCommand].compressed == 1) // The game uses IPAK_COMMAND_SKIP as value for compressed when it intends to skip the specified amount of data
if (blockHeader->commands[currentCommand].compressed == 0
|| blockHeader->commands[currentCommand].compressed == 1)
{ {
printf("IPak block offset is not the file head: %u != %lld -> Invalid\n", blockHeader->offset, m_file_head); std::cerr << "IPak block offset (" << blockHeader->countAndOffset.offset << ") is not the file head (" << m_file_head << ") -> Invalid\n";
return false; return false;
} }
} }
@ -141,13 +155,12 @@ bool IPakEntryReadStream::ValidateBlockHeader(IPakDataBlockHeader* blockHeader)
return true; return true;
} }
bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(const IPakDataBlockHeader* blockHeader, const size_t blockOffsetInChunk)
const size_t blockOffsetInChunk)
{ {
size_t commandsSize = 0; size_t commandsSize = 0;
for (unsigned commandIndex = 0; commandIndex < blockHeader->count; commandIndex++) for (unsigned commandIndex = 0; commandIndex < blockHeader->countAndOffset.count; commandIndex++)
{ {
commandsSize += blockHeader->_commands[commandIndex].size; commandsSize += blockHeader->commands[commandIndex].size;
} }
const size_t requiredChunkCount = AlignForward<size_t>(blockOffsetInChunk + sizeof(IPakDataBlockHeader) + commandsSize, IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE; const size_t requiredChunkCount = AlignForward<size_t>(blockOffsetInChunk + sizeof(IPakDataBlockHeader) + commandsSize, IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE;
@ -158,8 +171,7 @@ bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHea
{ {
if (requiredChunkCount > IPAK_CHUNK_COUNT_PER_READ) if (requiredChunkCount > IPAK_CHUNK_COUNT_PER_READ)
{ {
printf("IPak block spans over more than %u blocks (%u), which is not supported.\n", std::cerr << "IPak block spans over more than " << IPAK_CHUNK_COUNT_PER_READ << " chunks (" << requiredChunkCount << "), which is not supported.\n";
IPAK_CHUNK_COUNT_PER_READ, requiredChunkCount);
return false; return false;
} }
@ -180,7 +192,6 @@ bool IPakEntryReadStream::NextBlock()
const auto chunkStartPos = AlignBackwards<int64_t>(m_pos, IPAK_CHUNK_SIZE); const auto chunkStartPos = AlignBackwards<int64_t>(m_pos, IPAK_CHUNK_SIZE);
const auto blockOffsetInChunk = static_cast<size_t>(m_pos - chunkStartPos); const auto blockOffsetInChunk = static_cast<size_t>(m_pos - chunkStartPos);
const auto sizeLeftToRead = m_entry_size - m_file_head;
auto estimatedChunksToRead = AlignForward(m_entry_size - static_cast<size_t>(m_pos - m_base_pos), IPAK_CHUNK_SIZE) auto estimatedChunksToRead = AlignForward(m_entry_size - static_cast<size_t>(m_pos - m_base_pos), IPAK_CHUNK_SIZE)
/ IPAK_CHUNK_SIZE; / IPAK_CHUNK_SIZE;
@ -216,7 +227,7 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com
if (result != LZO_E_OK) if (result != LZO_E_OK)
{ {
printf("Decompressing block with lzo failed: %i!\n", result); std::cerr << "Decompressing block with lzo failed: " << result << "!\n";
return false; return false;
} }
@ -225,6 +236,10 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com
m_current_command_offset = 0; m_current_command_offset = 0;
m_file_head += outputSize; m_file_head += outputSize;
} }
else
{
// Do not process data but instead skip specified commandSize
}
} }
else else
{ {
@ -240,14 +255,14 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com
bool IPakEntryReadStream::AdvanceStream() bool IPakEntryReadStream::AdvanceStream()
{ {
if (m_current_block == nullptr || m_next_command >= m_current_block->count) if (m_current_block == nullptr || m_next_command >= m_current_block->countAndOffset.count)
{ {
if (!NextBlock()) if (!NextBlock())
return false; return false;
} }
ProcessCommand(m_current_block->_commands[m_next_command].size, ProcessCommand(m_current_block->commands[m_next_command].size,
m_current_block->_commands[m_next_command].compressed); m_current_block->commands[m_next_command].compressed);
m_next_command++; m_next_command++;
return true; return true;

View File

@ -45,12 +45,56 @@ class IPakEntryReadStream final : public objbuf
return num / alignTo * alignTo; return num / alignTo * alignTo;
} }
/**
* \brief Reads the specified chunks from disk.
* \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data.
* \param startPos The file offset at which the data to be loaded starts at.
* \param chunkCount The amount of chunks to be loaded.
* \return The amount of chunks that could be successfully loaded.
*/
size_t ReadChunks(uint8_t* buffer, int64_t startPos, size_t chunkCount) const; size_t ReadChunks(uint8_t* buffer, int64_t startPos, size_t chunkCount) const;
/**
* \brief Make sure the loaded chunk buffer window corresponds to the specified parameters and loads additional data if necessary.
* \param startPos The offset in the file that should be the start of the chunk buffer.
* \param chunkCount The amount of chunks that the buffer should hold. Can not exceed IPAK_CHUNK_COUNT_PER_READ.
* \return \c true if the chunk buffer window could successfully be adjusted, \c false otherwise.
*/
bool SetChunkBufferWindow(int64_t startPos, size_t chunkCount); bool SetChunkBufferWindow(int64_t startPos, size_t chunkCount);
bool ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const;
bool AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, size_t blockOffsetInChunk); /**
* \brief Makes sure the specified block can be safely loaded.
* \param blockHeader The block header to check.
* \return \c true if the block can be safely loaded, \c false otherwise.
*/
bool ValidateBlockHeader(const IPakDataBlockHeader* blockHeader) const;
/**
* \brief Makes sure that the specified block fits inside the loaded chunk buffer window and adjusts the chunk buffer window if necessary.
* \param blockHeader The header of the block that needs to fit inside the loaded chunk buffer window.
* \param blockOffsetInChunk The offset of the block inside the current chunk.
* \return \c true if the chunk buffer window was either already valid or was successfully adjusted to have all block data loaded. \c false otherwise.
*/
bool AdjustChunkBufferWindowForBlockHeader(const IPakDataBlockHeader* blockHeader, size_t blockOffsetInChunk);
/**
* \brief Ensures the next valid block is loaded.
* \return \c true if a new block was loaded or \c false if no further valid block could be loaded.
*/
bool NextBlock(); bool NextBlock();
/**
* \brief Processes a command with the specified parameters at the current position.
* \param commandSize The size of the command data
* \param compressed The compression value of the command. Can be \c 0 for uncompressed or \c 1 for lzo compression. Any other value skips the specified size of data.
* \return \c true if the specified command could be correctly processed or \c otherwise.
*/
bool ProcessCommand(size_t commandSize, int compressed); bool ProcessCommand(size_t commandSize, int compressed);
/**
* \brief Ensures that the next valid command is loaded.
* \return \c true if a new command was loaded or \c false if no further valid command could be loaded.
*/
bool AdvanceStream(); bool AdvanceStream();
public: public:

View File

@ -0,0 +1,382 @@
#include "IPakWriter.h"
#include <algorithm>
#include <iostream>
#include <minilzo.h>
#include <sstream>
#include <zlib.h>
#include "Game/T6/CommonT6.h"
#include "Game/T6/GameT6.h"
#include "ObjContainer/IPak/IPakTypes.h"
#include "Utils/Alignment.h"
class IPakWriterImpl final : public IPakWriter
{
static constexpr char BRANDING[] = "Created with OAT - OpenAssetTools";
static constexpr auto SECTION_COUNT = 3; // Index + Data + Branding
inline static const std::string PAD_DATA = std::string(256, '\xA7');
public:
explicit IPakWriterImpl(std::ostream& stream, ISearchPath* assetSearchPath)
: m_stream(stream),
m_asset_search_path(assetSearchPath),
m_current_offset(0),
m_total_size(0),
m_data_section_offset(0),
m_data_section_size(0u),
m_index_section_offset(0),
m_branding_section_offset(0),
m_file_offset(0u),
m_chunk_buffer_window_start(0),
m_current_block{},
m_current_block_header_offset(0)
{
m_decompressed_buffer = std::make_unique<char[]>(ipak_consts::IPAK_CHUNK_SIZE);
m_lzo_work_buffer = std::make_unique<char[]>(LZO1X_1_MEM_COMPRESS);
}
void AddImage(std::string imageName) override
{
m_images.emplace_back(std::move(imageName));
}
void GoTo(const int64_t offset)
{
m_stream.seekp(offset, std::ios::beg);
m_current_offset = offset;
}
void Write(const void* data, const size_t dataSize)
{
m_stream.write(static_cast<const char*>(data), dataSize);
m_current_offset += dataSize;
}
void Pad(const size_t paddingSize)
{
auto paddingSizeLeft = paddingSize;
while (paddingSizeLeft > 0)
{
const auto writeSize = std::min(paddingSizeLeft, PAD_DATA.size());
Write(PAD_DATA.data(), writeSize);
paddingSizeLeft -= writeSize;
}
}
void AlignToChunk()
{
Pad(static_cast<size_t>(utils::Align(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE)) - m_current_offset));
}
void AlignToBlockHeader()
{
Pad(static_cast<size_t>(utils::Align(m_current_offset, static_cast<int64_t>(sizeof(IPakDataBlockHeader))) - m_current_offset));
}
void WriteHeaderData()
{
GoTo(0);
const IPakHeader header{
ipak_consts::IPAK_MAGIC,
ipak_consts::IPAK_VERSION,
static_cast<uint32_t>(m_total_size),
SECTION_COUNT
};
const IPakSection dataSection{
ipak_consts::IPAK_DATA_SECTION,
static_cast<uint32_t>(m_data_section_offset),
static_cast<uint32_t>(m_data_section_size),
static_cast<uint32_t>(m_index_entries.size())
};
const IPakSection indexSection{
ipak_consts::IPAK_INDEX_SECTION,
static_cast<uint32_t>(m_index_section_offset),
static_cast<uint32_t>(sizeof(IPakIndexEntry) * m_index_entries.size()),
static_cast<uint32_t>(m_index_entries.size())
};
const IPakSection brandingSection{
ipak_consts::IPAK_BRANDING_SECTION,
static_cast<uint32_t>(m_branding_section_offset),
std::extent_v<decltype(BRANDING)>,
1
};
Write(&header, sizeof(header));
Write(&dataSection, sizeof(dataSection));
Write(&indexSection, sizeof(indexSection));
Write(&brandingSection, sizeof(brandingSection));
}
static std::string ImageFileName(const std::string& imageName)
{
std::ostringstream ss;
ss << "images/" << imageName << ".iwi";
return ss.str();
}
std::unique_ptr<char[]> ReadImageDataFromSearchPath(const std::string& imageName, size_t& imageSize) const
{
const auto fileName = ImageFileName(imageName);
const auto openFile = m_asset_search_path->Open(fileName);
if (!openFile.IsOpen())
{
std::cerr << "Could not open image for writing to IPak \"" << fileName << "\"\n";
return nullptr;
}
imageSize = static_cast<size_t>(openFile.m_length);
auto imageData = std::make_unique<char[]>(imageSize);
openFile.m_stream->read(imageData.get(), imageSize);
return imageData;
}
void FlushBlock()
{
if (m_current_block_header_offset > 0)
{
const auto previousOffset = m_current_offset;
GoTo(m_current_block_header_offset);
Write(&m_current_block, sizeof(m_current_block));
GoTo(previousOffset);
}
}
void FlushChunk()
{
FlushBlock();
AlignToBlockHeader();
const auto nextChunkOffset = utils::Align(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE));
const auto sizeToSkip = static_cast<size_t>(nextChunkOffset - m_current_offset);
if (sizeToSkip >= sizeof(IPakDataBlockHeader))
{
IPakDataBlockHeader skipBlockHeader{};
skipBlockHeader.countAndOffset.count = 1;
skipBlockHeader.commands[0].compressed = ipak_consts::IPAK_COMMAND_SKIP;
skipBlockHeader.commands[0].size = sizeToSkip - sizeof(IPakDataBlockHeader);
Write(&skipBlockHeader, sizeof(skipBlockHeader));
}
AlignToChunk();
m_chunk_buffer_window_start = m_current_offset;
}
void StartNewBlock()
{
AlignToBlockHeader();
// Skip to the next chunk when only the header could fit into the current chunk anyway
if (static_cast<size_t>(utils::Align(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE)) - m_current_offset) <= sizeof(IPakDataBlockHeader))
FlushChunk();
m_current_block_header_offset = m_current_offset;
m_current_block = {};
m_current_block.countAndOffset.offset = static_cast<uint32_t>(m_file_offset);
// Reserve space to later write actual block header data
GoTo(m_current_offset + sizeof(IPakDataBlockHeader));
}
void WriteChunkData(const void* data, const size_t dataSize)
{
auto dataOffset = 0u;
while (dataOffset < dataSize)
{
if (m_current_block.countAndOffset.count >= std::extent_v<decltype(IPakDataBlockHeader::commands)>)
{
FlushBlock();
StartNewBlock();
}
const auto remainingSize = dataSize - dataOffset;
const auto remainingChunkBufferWindowSize = std::max((ipak_consts::IPAK_CHUNK_COUNT_PER_READ * ipak_consts::IPAK_CHUNK_SIZE)
- static_cast<size_t>(m_current_offset - m_chunk_buffer_window_start), 0u);
if (remainingChunkBufferWindowSize == 0)
{
FlushChunk();
StartNewBlock();
continue;
}
const auto commandSize = std::min(std::min(remainingSize, ipak_consts::IPAK_COMMAND_DEFAULT_SIZE), remainingChunkBufferWindowSize);
auto writeUncompressed = true;
if (USE_COMPRESSION)
{
auto outLen = static_cast<lzo_uint>(ipak_consts::IPAK_CHUNK_SIZE);
const auto result = lzo1x_1_compress(&static_cast<const unsigned char*>(data)[dataOffset], commandSize, reinterpret_cast<unsigned char*>(m_decompressed_buffer.get()), &outLen,
m_lzo_work_buffer.get());
if (result == LZO_E_OK && outLen < commandSize)
{
writeUncompressed = false;
Write(m_decompressed_buffer.get(), outLen);
const auto currentCommand = m_current_block.countAndOffset.count;
m_current_block.commands[currentCommand].size = static_cast<uint32_t>(outLen);
m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_COMPRESSED;
m_current_block.countAndOffset.count = currentCommand + 1u;
}
}
if (writeUncompressed)
{
Write(&static_cast<const char*>(data)[dataOffset], commandSize);
const auto currentCommand = m_current_block.countAndOffset.count;
m_current_block.commands[currentCommand].size = commandSize;
m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_UNCOMPRESSED;
m_current_block.countAndOffset.count = currentCommand + 1u;
}
dataOffset += commandSize;
m_file_offset += commandSize;
}
}
void StartNewFile()
{
FlushBlock();
m_file_offset = 0u;
StartNewBlock();
m_chunk_buffer_window_start = utils::AlignToPrevious(m_current_offset, static_cast<int64_t>(ipak_consts::IPAK_CHUNK_SIZE));
}
bool WriteImageData(const std::string& imageName)
{
size_t imageSize;
const auto imageData = ReadImageDataFromSearchPath(imageName, imageSize);
if (!imageData)
return false;
const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0);
const auto dataHash = static_cast<unsigned>(crc32(0u, reinterpret_cast<const Bytef*>(imageData.get()), imageSize));
StartNewFile();
const auto startOffset = m_current_block_header_offset;
IPakIndexEntry indexEntry;
indexEntry.key.nameHash = nameHash;
indexEntry.key.dataHash = dataHash & 0x1FFFFFFF;
indexEntry.offset = static_cast<uint32_t>(startOffset - m_data_section_offset);
WriteChunkData(imageData.get(), imageSize);
const auto writtenImageSize = static_cast<size_t>(m_current_offset - startOffset);
indexEntry.size = writtenImageSize;
m_index_entries.emplace_back(indexEntry);
return true;
}
bool WriteDataSection()
{
AlignToChunk();
m_data_section_offset = m_current_offset;
m_data_section_size = 0u;
m_index_entries.reserve(m_images.size());
const auto result = std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName)
{
return WriteImageData(imageName);
});
FlushBlock();
m_data_section_size = static_cast<size_t>(m_current_offset - m_data_section_offset);
return result;
}
static bool CompareIndices(const IPakIndexEntry& entry1, const IPakIndexEntry& entry2)
{
return entry1.key.combinedKey < entry2.key.combinedKey;
}
void SortIndexSectionEntries()
{
std::sort(m_index_entries.begin(), m_index_entries.end(), CompareIndices);
}
void WriteIndexSection()
{
AlignToChunk();
m_index_section_offset = m_current_offset;
SortIndexSectionEntries();
for (const auto& indexEntry : m_index_entries)
Write(&indexEntry, sizeof(indexEntry));
}
void WriteBrandingSection()
{
AlignToChunk();
m_branding_section_offset = m_current_offset;
Write(BRANDING, std::extent_v<decltype(BRANDING)>);
}
void WriteFileEnding()
{
AlignToChunk();
m_total_size = m_current_offset;
}
bool Write() override
{
// We will write the header and sections later since they need complementary data
GoTo(sizeof(IPakHeader) + sizeof(IPakSection) * SECTION_COUNT);
AlignToChunk();
if (!WriteDataSection())
return false;
WriteIndexSection();
WriteBrandingSection();
WriteFileEnding();
WriteHeaderData();
return true;
}
private:
std::ostream& m_stream;
ISearchPath* m_asset_search_path;
std::vector<std::string> m_images;
int64_t m_current_offset;
std::vector<IPakIndexEntry> m_index_entries;
int64_t m_total_size;
int64_t m_data_section_offset;
size_t m_data_section_size;
int64_t m_index_section_offset;
int64_t m_branding_section_offset;
std::unique_ptr<char[]> m_decompressed_buffer;
std::unique_ptr<char[]> m_lzo_work_buffer;
size_t m_file_offset;
int64_t m_chunk_buffer_window_start;
IPakDataBlockHeader m_current_block;
int64_t m_current_block_header_offset;
};
std::unique_ptr<IPakWriter> IPakWriter::Create(std::ostream& stream, ISearchPath* assetSearchPath)
{
return std::make_unique<IPakWriterImpl>(stream, assetSearchPath);
}

View File

@ -0,0 +1,24 @@
#pragma once
#include <memory>
#include <ostream>
#include "SearchPath/ISearchPath.h"
class IPakWriter
{
public:
static constexpr auto USE_COMPRESSION = true;
IPakWriter() = default;
virtual ~IPakWriter() = default;
IPakWriter(const IPakWriter& other) = default;
IPakWriter(IPakWriter&& other) noexcept = default;
IPakWriter& operator=(const IPakWriter& other) = default;
IPakWriter& operator=(IPakWriter&& other) noexcept = default;
virtual void AddImage(std::string imageName) = 0;
virtual bool Write() = 0;
static std::unique_ptr<IPakWriter> Create(std::ostream& stream, ISearchPath* assetSearchPath);
};

View File

@ -3,10 +3,18 @@
namespace utils namespace utils
{ {
template <typename T> template <typename T>
constexpr T Align(T value, T toNext) constexpr T Align(const T value, const T toNext)
{ {
if (toNext > 0) if (toNext > 0)
return (value + toNext - 1) / toNext * toNext; return (value + toNext - 1) / toNext * toNext;
return value; return value;
} }
template <typename T>
constexpr T AlignToPrevious(const T value, const T toPrevious)
{
if (toPrevious > 0)
return value / toPrevious * toPrevious;
return value;
}
} }

View File

@ -88,4 +88,16 @@ namespace utils
inEscape = true; inEscape = true;
} }
} }
void MakeStringLowerCase(std::string& str)
{
for (auto& c : str)
c = static_cast<char>(tolower(c));
}
void MakeStringUpperCase(std::string& str)
{
for (auto& c : str)
c = static_cast<char>(toupper(c));
}
} }

View File

@ -11,4 +11,7 @@ namespace utils
void EscapeStringForQuotationMarks(std::ostream& stream, const std::string_view& str); void EscapeStringForQuotationMarks(std::ostream& stream, const std::string_view& str);
std::string UnescapeStringFromQuotationMarks(const std::string_view& str); std::string UnescapeStringFromQuotationMarks(const std::string_view& str);
void UnescapeStringFromQuotationMarks(std::ostream& stream, const std::string_view& str); void UnescapeStringFromQuotationMarks(std::ostream& stream, const std::string_view& str);
void MakeStringLowerCase(std::string& str);
void MakeStringUpperCase(std::string& str);
} }

View File

@ -0,0 +1,19 @@
#include "SequenceZoneDefinitionAssetList.h"
#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h"
SequenceZoneDefinitionAssetList::SequenceZoneDefinitionAssetList()
{
const ZoneDefinitionMatcherFactory create(this);
AddMatchers({
create.Keyword("assetlist"),
create.Char(','),
create.Field().Capture(CAPTURE_ASSET_LIST_NAME)
});
}
void SequenceZoneDefinitionAssetList::ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const
{
state->m_asset_lists.emplace_back(result.NextCapture(CAPTURE_ASSET_LIST_NAME).FieldValue());
}

View File

@ -0,0 +1,14 @@
#pragma once
#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
class SequenceZoneDefinitionAssetList final : public ZoneDefinitionParser::sequence_t
{
static constexpr auto CAPTURE_ASSET_LIST_NAME = 1;
protected:
void ProcessMatch(ZoneDefinition* state, SequenceResult<ZoneDefinitionParserValue>& result) const override;
public:
SequenceZoneDefinitionAssetList();
};

View File

@ -1,5 +1,6 @@
#include "ZoneDefinitionParser.h" #include "ZoneDefinitionParser.h"
#include "Sequence/SequenceZoneDefinitionAssetList.h"
#include "Sequence/SequenceZoneDefinitionEntry.h" #include "Sequence/SequenceZoneDefinitionEntry.h"
#include "Sequence/SequenceZoneDefinitionIgnore.h" #include "Sequence/SequenceZoneDefinitionIgnore.h"
#include "Sequence/SequenceZoneDefinitionInclude.h" #include "Sequence/SequenceZoneDefinitionInclude.h"
@ -16,6 +17,7 @@ const std::vector<AbstractParser<ZoneDefinitionParserValue, ZoneDefinition>::seq
new SequenceZoneDefinitionMetaData(), new SequenceZoneDefinitionMetaData(),
new SequenceZoneDefinitionInclude(), new SequenceZoneDefinitionInclude(),
new SequenceZoneDefinitionIgnore(), new SequenceZoneDefinitionIgnore(),
new SequenceZoneDefinitionAssetList(),
new SequenceZoneDefinitionEntry() new SequenceZoneDefinitionEntry()
}); });

View File

@ -1,10 +1,13 @@
#include "AssetList.h" #include "AssetList.h"
AssetListEntry::AssetListEntry() AssetListEntry::AssetListEntry()
= default; : m_is_reference(false)
{
AssetListEntry::AssetListEntry(std::string type, std::string name) }
: m_type(std::move(type)),
m_name(std::move(name)) AssetListEntry::AssetListEntry(std::string type, std::string name, const bool isReference)
: m_type(std::move(type)),
m_name(std::move(name)),
m_is_reference(isReference)
{ {
} }

View File

@ -1,12 +1,20 @@
#pragma once #pragma once
#include <string> #include <string>
#include <vector>
class AssetListEntry class AssetListEntry
{ {
public: public:
std::string m_type; std::string m_type;
std::string m_name; std::string m_name;
bool m_is_reference;
AssetListEntry(); AssetListEntry();
AssetListEntry(std::string type, std::string name); AssetListEntry(std::string type, std::string name, bool isReference);
};
class AssetList
{
public:
std::vector<AssetListEntry> m_entries;
}; };

View File

@ -18,8 +18,17 @@ bool AssetListInputStream::NextEntry(AssetListEntry& entry) const
continue; continue;
entry.m_type = row[0]; entry.m_type = row[0];
if (row.size() >= 2) if (row.size() >= 3 && row[1].empty())
{
entry.m_name = row[2];
entry.m_is_reference = true;
}
else
{
entry.m_name = row[1]; entry.m_name = row[1];
entry.m_is_reference = false;
}
return true; return true;
} }
} }

View File

@ -29,7 +29,15 @@ void ZoneDefinition::AddMetaData(std::string key, std::string value)
m_metadata_lookup.emplace(std::make_pair(metaDataPtr->m_key, metaDataPtr)); m_metadata_lookup.emplace(std::make_pair(metaDataPtr->m_key, metaDataPtr));
} }
void ZoneDefinition::Include(ZoneDefinition& definitionToInclude) void ZoneDefinition::Include(const AssetList& assetListToInclude)
{
for (const auto& entry : assetListToInclude.m_entries)
{
m_assets.emplace_back(entry.m_type, entry.m_name, false);
}
}
void ZoneDefinition::Include(const ZoneDefinition& definitionToInclude)
{ {
for (const auto& metaData : definitionToInclude.m_metadata) for (const auto& metaData : definitionToInclude.m_metadata)
{ {

View File

@ -5,6 +5,8 @@
#include <unordered_map> #include <unordered_map>
#include <vector> #include <vector>
#include "Zone/AssetList/AssetList.h"
class ZoneDefinitionEntry class ZoneDefinitionEntry
{ {
public: public:
@ -33,9 +35,11 @@ public:
std::vector<std::unique_ptr<ZoneMetaDataEntry>> m_metadata; std::vector<std::unique_ptr<ZoneMetaDataEntry>> m_metadata;
std::unordered_multimap<std::string, ZoneMetaDataEntry*> m_metadata_lookup; std::unordered_multimap<std::string, ZoneMetaDataEntry*> m_metadata_lookup;
std::vector<std::string> m_includes; std::vector<std::string> m_includes;
std::vector<std::string> m_asset_lists;
std::vector<std::string> m_ignores; std::vector<std::string> m_ignores;
std::vector<ZoneDefinitionEntry> m_assets; std::vector<ZoneDefinitionEntry> m_assets;
void AddMetaData(std::string key, std::string value); void AddMetaData(std::string key, std::string value);
void Include(ZoneDefinition& definitionToInclude); void Include(const AssetList& assetListToInclude);
void Include(const ZoneDefinition& definitionToInclude);
}; };