zone/OpenAssetTools
2
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2026-05-16 23:11:42 +00:00
Code Activity

chore: enable reading of big endian ipaks

Browse Source
This commit is contained in:
Jan Laupetin
2026-05-11 23:54:25 +02:00
parent 7ae7cf85ff
commit e1bb8ae4d2
7 changed files with 183 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ObjCommon/ObjContainer/IPak/IPakTypes.h
+10 -4
View File
@@ -6,7 +6,8 @@
namespace ipak_consts
{
static constexpr uint32_t IPAK_MAGIC = utils::MakeMagic32('K', 'A', 'P', 'I');
static constexpr uint32_t IPAK_MAGIC_LITTLE_ENDIAN = utils::MakeMagic32('K', 'A', 'P', 'I');
static constexpr uint32_t IPAK_MAGIC_BIG_ENDIAN = utils::MakeMagic32('I', 'P', 'A', 'K');
static constexpr uint32_t IPAK_VERSION = 0x50000;
static constexpr uint32_t IPAK_INDEX_SECTION = 1;
@@ -60,10 +61,15 @@ struct IPakIndexEntry
uint32_t size;
};
struct IPakDataBlockCountAndOffset
union IPakDataBlockCountAndOffset
{
uint32_t offset : 24;
uint32_t count : 8;
struct
{
uint32_t offset : 24;
uint32_t count : 8;
};
uint32_t raw;
};
static_assert(sizeof(IPakDataBlockCountAndOffset) == 4);
src/ObjCompiling/Image/IPak/IPakCreator.cpp
+1 -1
View File
@@ -100,7 +100,7 @@ namespace
{
GoTo(0);
const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC,
const IPakHeader header{.magic = ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN,
.version = ipak_consts::IPAK_VERSION,
.size = static_cast<uint32_t>(m_total_size),
.sectionCount = SECTION_COUNT};
src/ObjLoading/ObjContainer/IPak/IPak.cpp
+39 -5
View File
@@ -2,8 +2,10 @@
#include "IPakStreamManager.h"
#include "ObjContainer/IPak/IPakTypes.h"
#include "Utils/Endianness.h"
#include "Utils/Logging/Log.h"
#include <concepts>
#include <filesystem>
#include <format>
#include <iostream>
@@ -37,9 +39,10 @@ namespace
: m_path(std::move(path)),
m_stream(std::move(stream)),
m_initialized(false),
m_little_endian(true),
m_index_section(nullptr),
m_data_section(nullptr),
m_stream_manager(*m_stream)
m_stream_manager(nullptr)
{
}
@@ -65,7 +68,7 @@ namespace
{
if (entry.key.combinedKey == wantedKey.combinedKey)
{
return m_stream_manager.OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
return m_stream_manager->OpenStream(static_cast<int64_t>(m_data_section->offset) + entry.offset, entry.size);
}
else if (entry.key.combinedKey > wantedKey.combinedKey)
{
@@ -102,7 +105,11 @@ namespace
return false;
}
m_index_entries.push_back(indexEntry);
SwapBytesIfNecessary(indexEntry.key.combinedKey);
SwapBytesIfNecessary(indexEntry.offset);
SwapBytesIfNecessary(indexEntry.size);
m_index_entries.emplace_back(indexEntry);
}
std::ranges::sort(m_index_entries,
@@ -125,6 +132,11 @@ namespace
return false;
}
SwapBytesIfNecessary(section.type);
SwapBytesIfNecessary(section.offset);
SwapBytesIfNecessary(section.size);
SwapBytesIfNecessary(section.itemCount);
switch (section.type)
{
case ipak_consts::IPAK_INDEX_SECTION:
@@ -153,18 +165,31 @@ namespace
return false;
}
if (header.magic != ipak_consts::IPAK_MAGIC)
if (header.magic == ipak_consts::IPAK_MAGIC_LITTLE_ENDIAN)
{
m_little_endian = true;
m_stream_manager = IPakStreamManager::Create(*m_stream, true);
}
else if (header.magic == ipak_consts::IPAK_MAGIC_BIG_ENDIAN)
{
m_little_endian = false;
m_stream_manager = IPakStreamManager::Create(*m_stream, false);
}
else
{
con::error("Invalid ipak magic '{:#x}'.", header.magic);
return false;
}
SwapBytesIfNecessary(header.version);
if (header.version != ipak_consts::IPAK_VERSION)
{
con::error("Unsupported ipak version '{}'.", header.version);
return false;
}
SwapBytesIfNecessary(header.size);
SwapBytesIfNecessary(header.sectionCount);
for (unsigned section = 0; section < header.sectionCount; section++)
{
if (!ReadSection())
@@ -189,17 +214,26 @@ namespace
return true;
}
template<std::integral T> void SwapBytesIfNecessary(T& value)
{
if (m_little_endian)
value = endianness::FromLittleEndian(value);
else
value = endianness::FromBigEndian(value);
}
std::string m_path;
std::unique_ptr<std::istream> m_stream;
bool m_initialized;
bool m_little_endian;
std::unique_ptr<IPakSection> m_index_section;
std::unique_ptr<IPakSection> m_data_section;
std::vector<IPakIndexEntry> m_index_entries;
IPakStreamManager m_stream_manager;
std::unique_ptr<IPakStreamManager> m_stream_manager;
};
} // namespace
src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp
+19 -2
View File
@@ -11,9 +11,14 @@
using namespace ipak_consts;
IPakEntryReadStream::IPakEntryReadStream(
std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, const int64_t startOffset, const size_t entrySize)
IPakEntryReadStream::IPakEntryReadStream(std::istream& stream,
const bool isLittleEndian,
IPakStreamManagerActions* streamManagerActions,
uint8_t* chunkBuffer,
const int64_t startOffset,
const size_t entrySize)
: m_chunk_buffer(chunkBuffer),
m_little_endian(isLittleEndian),
m_stream(stream),
m_stream_manager_actions(streamManagerActions),
m_file_offset(0),
@@ -198,6 +203,13 @@ bool IPakEntryReadStream::NextBlock()
return false;
m_current_block = reinterpret_cast<IPakDataBlockHeader*>(&m_chunk_buffer[blockOffsetInChunk]);
SwapBytesIfNecessary(m_current_block->countAndOffset.raw);
for (auto& command : m_current_block->commands)
{
auto size = command.size;
SwapBytesIfNecessary(size);
command.size = size;
}
if (!ValidateBlockHeader(m_current_block))
return false;
@@ -231,6 +243,11 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com
m_current_command_offset = 0;
m_file_head += static_cast<int64_t>(outputSize);
}
else if (compressed == 2)
{
// This seems to use XMemDecompress
assert(false);
}
else
{
// Do not process data but instead skip specified commandSize
src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h
+14 -1
View File
@@ -2,14 +2,17 @@
#include "IPakStreamManager.h"
#include "ObjContainer/IPak/IPakTypes.h"
#include "Utils/Endianness.h"
#include "Utils/ObjStream.h"
#include <concepts>
#include <istream>
class IPakEntryReadStream final : public objbuf
{
public:
IPakEntryReadStream(std::istream& stream, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
IPakEntryReadStream(
std::istream& stream, bool isLittleEndian, IPakStreamManagerActions* streamManagerActions, uint8_t* chunkBuffer, int64_t startOffset, size_t entrySize);
~IPakEntryReadStream() override;
[[nodiscard]] bool is_open() const override;
@@ -34,6 +37,14 @@ private:
return num / alignTo * alignTo;
}
template<std::integral T> void SwapBytesIfNecessary(T& value)
{
if (m_little_endian)
value = endianness::FromLittleEndian(value);
else
value = endianness::FromBigEndian(value);
}
/**
* \brief Reads the specified chunks from disk.
* \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data.
@@ -90,6 +101,8 @@ private:
uint8_t* m_chunk_buffer;
bool m_little_endian;
std::istream& m_stream;
IPakStreamManagerActions* m_stream_manager_actions;
src/ObjLoading/ObjContainer/IPak/IPakStreamManager.cpp
+95 -106
View File
@@ -8,10 +8,8 @@
using namespace ipak_consts;
class IPakStreamManager::Impl final : public IPakStreamManagerActions
namespace
{
static constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
class ChunkBuffer
{
public:
@@ -32,125 +30,116 @@ class IPakStreamManager::Impl final : public IPakStreamManagerActions
}
};
std::istream& m_stream;
constexpr int CHUNK_BUFFER_COUNT_IDLE_LIMIT = 3;
std::mutex m_read_mutex;
std::mutex m_stream_mutex;
std::vector<ManagedStream> m_open_streams;
std::vector<ChunkBuffer*> m_chunk_buffers;
public:
explicit Impl(std::istream& stream)
: m_stream(stream)
class IPakStreamManagerImpl final : public IPakStreamManager, public IPakStreamManagerActions
{
m_chunk_buffers.push_back(new ChunkBuffer());
}
Impl(const Impl& other) = delete;
Impl(Impl&& other) noexcept = delete;
virtual ~Impl()
{
m_stream_mutex.lock();
for (const auto& openStream : m_open_streams)
public:
IPakStreamManagerImpl(std::istream& stream, const bool isLittleEndian)
: m_stream(stream),
m_little_endian(isLittleEndian)
{
openStream.m_stream->close();
m_chunk_buffers.push_back(new ChunkBuffer());
}
m_open_streams.clear();
m_stream_mutex.unlock();
}
Impl& operator=(const Impl& other) = delete;
Impl& operator=(Impl&& other) noexcept = delete;
std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length)
{
m_stream_mutex.lock();
ChunkBuffer* reservedChunkBuffer;
const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers,
[](ChunkBuffer* chunkBuffer)
{
return chunkBuffer->m_using_stream == nullptr;
});
if (freeChunkBuffer == m_chunk_buffers.end())
~IPakStreamManagerImpl() override
{
reservedChunkBuffer = new ChunkBuffer();
m_chunk_buffers.push_back(reservedChunkBuffer);
}
else
reservedChunkBuffer = *freeChunkBuffer;
m_stream_mutex.lock();
auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, this, reservedChunkBuffer->m_buffer, startPosition, length);
reservedChunkBuffer->m_using_stream = ipakEntryStream.get();
m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer);
m_stream_mutex.unlock();
return std::make_unique<iobjstream>(std::move(ipakEntryStream));
}
void StartReading() override
{
m_read_mutex.lock();
}
void StopReading() override
{
m_read_mutex.unlock();
}
void CloseStream(objbuf* stream) override
{
m_stream_mutex.lock();
const auto openStreamEntry = std::ranges::find_if(m_open_streams,
[stream](const ManagedStream& managedStream)
{
return managedStream.m_stream == stream;
});
if (openStreamEntry != m_open_streams.end())
{
auto* chunkBuffer = openStreamEntry->m_chunk_buffer;
m_open_streams.erase(openStreamEntry);
chunkBuffer->m_using_stream = nullptr;
// Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers
if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT)
for (const auto& openStream : m_open_streams)
{
const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer);
openStream.m_stream->close();
}
m_open_streams.clear();
if (chunkBufferEntry != m_chunk_buffers.end())
m_stream_mutex.unlock();
}
std::unique_ptr<iobjstream> OpenStream(const int64_t startPosition, const size_t length) override
{
m_stream_mutex.lock();
ChunkBuffer* reservedChunkBuffer;
const auto freeChunkBuffer = std::ranges::find_if(m_chunk_buffers,
[](ChunkBuffer* chunkBuffer)
{
return chunkBuffer->m_using_stream == nullptr;
});
if (freeChunkBuffer == m_chunk_buffers.end())
{
reservedChunkBuffer = new ChunkBuffer();
m_chunk_buffers.push_back(reservedChunkBuffer);
}
else
reservedChunkBuffer = *freeChunkBuffer;
auto ipakEntryStream = std::make_unique<IPakEntryReadStream>(m_stream, m_little_endian, this, reservedChunkBuffer->m_buffer, startPosition, length);
reservedChunkBuffer->m_using_stream = ipakEntryStream.get();
m_open_streams.emplace_back(ipakEntryStream.get(), reservedChunkBuffer);
m_stream_mutex.unlock();
return std::make_unique<iobjstream>(std::move(ipakEntryStream));
}
void StartReading() override
{
m_read_mutex.lock();
}
void StopReading() override
{
m_read_mutex.unlock();
}
void CloseStream(objbuf* stream) override
{
m_stream_mutex.lock();
const auto openStreamEntry = std::ranges::find_if(m_open_streams,
[stream](const ManagedStream& managedStream)
{
return managedStream.m_stream == stream;
});
if (openStreamEntry != m_open_streams.end())
{
auto* chunkBuffer = openStreamEntry->m_chunk_buffer;
m_open_streams.erase(openStreamEntry);
chunkBuffer->m_using_stream = nullptr;
// Only keep previously allocated chunk buffer if we did not get over the limit of idle chunk buffers
if (m_chunk_buffers.size() > CHUNK_BUFFER_COUNT_IDLE_LIMIT)
{
m_chunk_buffers.erase(chunkBufferEntry);
delete chunkBuffer;
const auto chunkBufferEntry = std::ranges::find(m_chunk_buffers, chunkBuffer);
if (chunkBufferEntry != m_chunk_buffers.end())
{
m_chunk_buffers.erase(chunkBufferEntry);
delete chunkBuffer;
}
}
}
m_stream_mutex.unlock();
}
m_stream_mutex.unlock();
}
};
private:
std::istream& m_stream;
bool m_little_endian;
IPakStreamManager::IPakStreamManager(std::istream& stream)
: m_impl(new Impl(stream))
{
}
std::mutex m_read_mutex;
std::mutex m_stream_mutex;
IPakStreamManager::~IPakStreamManager()
{
delete m_impl;
m_impl = nullptr;
}
std::vector<ManagedStream> m_open_streams;
std::vector<ChunkBuffer*> m_chunk_buffers;
};
} // namespace
std::unique_ptr<iobjstream> IPakStreamManager::OpenStream(const int64_t startPosition, const size_t length) const
std::unique_ptr<IPakStreamManager> IPakStreamManager::Create(std::istream& stream, const bool isLittleEndian)
{
return m_impl->OpenStream(startPosition, length);
return std::make_unique<IPakStreamManagerImpl>(stream, isLittleEndian);
}
src/ObjLoading/ObjContainer/IPak/IPakStreamManager.h
+5 -10
View File
@@ -4,6 +4,7 @@
#include <cstdint>
#include <istream>
#include <memory>
#include <mutex>
class IPakStreamManagerActions
@@ -17,17 +18,11 @@ public:
class IPakStreamManager
{
class Impl;
Impl* m_impl;
public:
explicit IPakStreamManager(std::istream& stream);
IPakStreamManager(const IPakStreamManager& other) = delete;
IPakStreamManager(IPakStreamManager&& other) noexcept = delete;
~IPakStreamManager();
IPakStreamManager() = default;
virtual ~IPakStreamManager() = default;
IPakStreamManager& operator=(const IPakStreamManager& other) = delete;
IPakStreamManager& operator=(IPakStreamManager&& other) noexcept = delete;
static std::unique_ptr<IPakStreamManager> Create(std::istream& stream, bool isLittleEndian);
[[nodiscard]] std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) const;
[[nodiscard]] virtual std::unique_ptr<iobjstream> OpenStream(int64_t startPosition, size_t length) = 0;
};