GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix packaging issues to properly read asset data after it's serialized with a new format

Browse Source
This commit is contained in:
Wojtek Figat
2025-01-22 21:24:49 +01:00
parent 326bc498b8
commit af416fe0c8
7 changed files with 181 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
Source/Engine/Content/Storage/FlaxStorage.cpp
View File

@@ -49,6 +49,15 @@ String AssetHeader::ToString() const
return String::Format(TEXT("ID: {0}, TypeName: {1}, Chunks Count: {2}"), ID, TypeName, GetChunksCount());
}
uint32 AssetInitData::GetHashCode() const
{
// Do not use Metadata/Dependencies because it may not be loaded (it's optional)
uint32 hashCode = GetHash(Header.ID);
hashCode = (hashCode * 397) ^ SerializedVersion;
hashCode = (hashCode * 397) ^ CustomData.Length();
return hashCode;
}
void FlaxChunk::RegisterUsage()
{
LastAccessTime = Platform::GetTimeSeconds();
@@ -605,6 +614,65 @@ bool FlaxStorage::Reload()
return failed;
}
bool FlaxStorage::ReloadSilent()
{
ScopeLock lock(_loadLocker);
if (!IsLoaded())
return false;
// Open file
auto stream = OpenFile();
if (stream == nullptr)
return true;
// Magic Code
uint32 magicCode;
stream->ReadUint32(&magicCode);
if (magicCode != MagicCode)
{
LOG(Warning, "Invalid asset magic code in {0}", ToString());
return true;
}
// Version
uint32 version;
stream->ReadUint32(&version);
if (version == 9)
{
_version = version;
// Custom storage data
CustomData customData;
stream->Read(customData);
// Entries
int32 assetsCount;
stream->ReadInt32(&assetsCount);
if (assetsCount != GetEntriesCount())
{
LOG(Warning, "Incorrect amount of assets ({} instead of {}) saved in {}", assetsCount, GetEntriesCount(), ToString());
return true;
}
for (int32 i = 0; i < assetsCount; i++)
{
SerializedEntryV9 se;
stream->ReadBytes(&se, sizeof(se));
Entry e(se.ID, se.TypeName.Data, se.Address);
SetEntry(i, e);
}
// TODO: load chunks metadata? compare against loaded chunks?
}
else
{
// Fallback to full reload
LOG(Warning, "Fallback to full reload of {0}", ToString());
Reload();
}
return false;
}
#endif
bool FlaxStorage::LoadAssetHeader(const Guid& id, AssetInitData& data)
@@ -809,7 +877,7 @@ FlaxChunk* FlaxStorage::AllocateChunk()
#if USE_EDITOR
bool FlaxStorage::Create(const StringView& path, const AssetInitData* data, int32 dataCount, bool silentMode, const CustomData* customData)
bool FlaxStorage::Create(const StringView& path, Span<AssetInitData> assets, bool silentMode, const CustomData* customData)
{
PROFILE_CPU();
ZoneText(*path, path.Length());
@@ -824,24 +892,32 @@ bool FlaxStorage::Create(const StringView& path, const AssetInitData* data, int3
return true;
// Create package
bool result = Create(stream, data, dataCount, customData);
bool result = Create(stream, assets, customData);
// Close file
Delete(stream);
// Reload storage container (only if not in silent mode)
if (storage && !silentMode)
// Reload storage container
if (storage && storage->IsLoaded())
{
storage->Reload();
if (silentMode)
{
// Silent data-only reload (loaded chunks location in file has been updated)
storage->ReloadSilent();
}
else
{
// Full reload
storage->Reload();
}
}
return result;
}
bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 dataCount, const CustomData* customData)
bool FlaxStorage::Create(WriteStream* stream, Span<AssetInitData> assets, const CustomData* customData)
{
// Validate inputs
if (data == nullptr || dataCount <= 0)
if (assets.Get() == nullptr || assets.Length() <= 0)
{
LOG(Warning, "Cannot create new package. No assets to write.");
return true;
@@ -849,27 +925,31 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
// Prepare data
Array<SerializedEntryV9, InlinedAllocation<1>> entries;
entries.Resize(dataCount);
entries.Resize(assets.Length());
Array<FlaxChunk*> chunks;
// Get all chunks
for (int32 i = 0; i < dataCount; i++)
data[i].Header.GetLoadedChunks(chunks);
int32 chunksCount = chunks.Count();
for (const AssetInitData& asset : assets)
{
for (FlaxChunk* chunk : asset.Header.Chunks)
{
if (chunk && chunk->IsLoaded())
chunks.Add(chunk);
}
}
const int32 chunksCount = chunks.Count();
// TODO: sort chunks by size? smaller ones first?
// Calculate start address of the first asset header location
// 0 -> Header -> Entries Count -> Entries -> Chunks Count -> Chunk Locations
int32 currentAddress = sizeof(Header)
+ sizeof(int32)
+ sizeof(SerializedEntryV9) * dataCount
+ sizeof(SerializedEntryV9) * assets.Length()
+ sizeof(int32)
+ (sizeof(FlaxChunk::Location) + sizeof(int32)) * chunksCount;
// Initialize entries offsets in the file
for (int32 i = 0; i < dataCount; i++)
for (int32 i = 0; i < assets.Length(); i++)
{
auto& asset = data[i];
const AssetInitData& asset = assets[i];
entries[i] = SerializedEntryV9(asset.Header.ID, asset.Header.TypeName, currentAddress);
// Move forward by asset header data size
@@ -934,8 +1014,8 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
stream->Write(mainHeader);
// Write asset entries
stream->WriteInt32(dataCount);
stream->WriteBytes(entries.Get(), sizeof(SerializedEntryV9) * dataCount);
stream->WriteInt32(assets.Length());
stream->WriteBytes(entries.Get(), sizeof(SerializedEntryV9) * assets.Length());
// Write chunk locations and meta
stream->WriteInt32(chunksCount);
@@ -948,20 +1028,18 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
}
#if ASSETS_LOADING_EXTRA_VERIFICATION
// Check calculated position of first asset header
if (dataCount > 0 && stream->GetPosition() != entries[0].Address)
if (assets.Length() > 0 && stream->GetPosition() != entries[0].Address)
{
LOG(Warning, "Error while asset header location computation.");
return true;
}
#endif
// Write asset headers
for (int32 i = 0; i < dataCount; i++)
for (int32 i = 0; i < assets.Length(); i++)
{
auto& header = data[i];
const AssetInitData& header = assets[i];
// ID
stream->Write(header.Header.ID);
@@ -974,9 +1052,10 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
stream->WriteUint32(header.SerializedVersion);
// Chunks mapping
for (FlaxChunk* chunk : header.Header.Chunks)
for (const FlaxChunk* chunk : header.Header.Chunks)
{
const int32 index = chunks.Find(chunk);
ASSERT_LOW_LAYER(index >= -1 && index <= ALL_ASSET_CHUNKS);
stream->WriteInt32(index);
}
@@ -998,14 +1077,12 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
}
#if ASSETS_LOADING_EXTRA_VERIFICATION
// Check calculated position of first asset chunk
if (chunksCount > 0 && stream->GetPosition() != chunks[0]->LocationInFile.Address)
{
LOG(Warning, "Error while asset data chunk location computation.");
return true;
}
#endif
// Write chunks data
@@ -1033,7 +1110,7 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
return false;
}
bool FlaxStorage::Save(AssetInitData& data, bool silentMode)
bool FlaxStorage::Save(const AssetInitData& data, bool silentMode)
{
// Check if can modify the storage
if (!AllowDataModifications())
@@ -1475,12 +1552,21 @@ int32 FlaxFile::GetEntriesCount() const
return _asset.ID.IsValid() ? 1 : 0;
}
void FlaxFile::GetEntry(int32 index, Entry& output) const
void FlaxFile::GetEntry(int32 index, Entry& value) const
{
ASSERT(index == 0);
output = _asset;
value = _asset;
}
#if USE_EDITOR
void FlaxFile::SetEntry(int32 index, const Entry& value)
{
ASSERT(index == 0);
_asset = value;
}
#endif
void FlaxFile::GetEntries(Array<Entry>& output) const
{
if (_asset.ID.IsValid())
@@ -1544,19 +1630,36 @@ int32 FlaxPackage::GetEntriesCount() const
return _entries.Count();
}
void FlaxPackage::GetEntry(int32 index, Entry& output) const
void FlaxPackage::GetEntry(int32 index, Entry& value) const
{
ASSERT(index >= 0 && index < _entries.Count());
for (auto i = _entries.Begin(); i.IsNotEnd(); ++i)
{
if (index-- <= 0)
{
output = i->Value;
value = i->Value;
return;
}
}
}
#if USE_EDITOR
void FlaxPackage::SetEntry(int32 index, const Entry& value)
{
ASSERT(index >= 0 && index < _entries.Count());
for (auto i = _entries.Begin(); i.IsNotEnd(); ++i)
{
if (index-- <= 0)
{
i->Value = value;
return;
}
}
}
#endif
void FlaxPackage::GetEntries(Array<Entry>& output) const
{
_entries.GetValues(output);