GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix packaging issues to properly read asset data after it's serialized with a new format

Browse Source
This commit is contained in:
Wojtek Figat
2025-01-22 21:24:49 +01:00
parent 326bc498b8
commit af416fe0c8
7 changed files with 181 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Source/Editor/Cooker/Steps/CookAssetsStep.cpp
View File

@@ -368,11 +368,21 @@ bool CookAssetsStep::Process(CookingData& data, CacheData& cache, Asset* asset)
// Virtual assets are not included into the build
return false;
}
const bool wasLoaded = asset->IsLoaded();
if (asset->WaitForLoaded())
{
LOG(Error, "Failed to load asset \'{0}\'", asset->ToString());
return true;
}
if (!wasLoaded)
{
// HACK: give some time to resave any old assets in Asset::onLoad after it's loaded
// This assumes that if Load Thread enters Asset::Save then it will get asset lock and hold it until asset is saved
// So we can take the same lock to wait for save end but first we need to wait for it to get that lock
// (in future try to handle it in a better way)
Platform::Sleep(5);
}
ScopeLock lock(asset->Locker);
// Switch based on an asset type
const auto asBinaryAsset = dynamic_cast<BinaryAsset*>(asset);

1
Source/Engine/Content/BinaryAsset.cpp
View File

@@ -372,6 +372,7 @@ bool BinaryAsset::SaveToAsset(const StringView& path, AssetInitData& data, bool
const auto locks = storage->_chunksLock;
storage->_chunksLock = 0;
result = storage->Save(data, silentMode);
ASSERT(storage->_chunksLock == 0);
storage->_chunksLock = locks;
}
else

44
Source/Engine/Content/Storage/AssetHeader.h
View File

@@ -20,7 +20,7 @@ typedef uint16 AssetChunksFlag;
#define ALL_ASSET_CHUNKS MAX_uint16
/// <summary>
/// Flax Asset header
/// Flax asset file header.
/// </summary>
struct FLAXENGINE_API AssetHeader
{
@@ -51,34 +51,6 @@ public:
}
public:
/// <summary>
/// Gets the chunks.
/// </summary>
/// <param name="output">The output data.</param>
template<typename AllocationType = HeapAllocation>
void GetChunks(Array<FlaxChunk*, AllocationType>& output) const
{
for (int32 i = 0; i < ASSET_FILE_DATA_CHUNKS; i++)
{
if (Chunks[i] != nullptr)
output.Add(Chunks[i]);
}
}
/// <summary>
/// Gets the chunks that are loaded.
/// </summary>
/// <param name="output">The output data.</param>
template<typename AllocationType = HeapAllocation>
void GetLoadedChunks(Array<FlaxChunk*, AllocationType>& output) const
{
for (int32 i = 0; i < ASSET_FILE_DATA_CHUNKS; i++)
{
if (Chunks[i] != nullptr && Chunks[i]->IsLoaded())
output.Add(Chunks[i]);
}
}
/// <summary>
/// Gets the amount of created asset chunks.
/// </summary>
@@ -95,14 +67,13 @@ public:
void UnlinkChunks();
/// <summary>
/// Gets string with a human-readable info about that header
/// Gets string with a human-readable info about that header.
/// </summary>
/// <returns>Header info string</returns>
String ToString() const;
};
/// <summary>
/// Flax Asset header data
/// Flax asset header with data.
/// </summary>
struct FLAXENGINE_API AssetInitData
{
@@ -137,12 +108,5 @@ public:
/// <summary>
/// Gets the hash code.
/// </summary>
uint32 GetHashCode() const
{
// Note: do not use Metadata/Dependencies because it may not be loaded (it's optional)
uint32 hashCode = GetHash(Header.ID);
hashCode = (hashCode * 397) ^ SerializedVersion;
hashCode = (hashCode * 397) ^ CustomData.Length();
return hashCode;
}
uint32 GetHashCode() const;
};

5
Source/Engine/Content/Storage/FlaxFile.h
View File

@@ -27,7 +27,10 @@ public:
bool HasAsset(const Guid& id) const override;
bool HasAsset(const AssetInfo& info) const override;
int32 GetEntriesCount() const override;
void GetEntry(int32 index, Entry& output) const override;
void GetEntry(int32 index, Entry& value) const override;
#if USE_EDITOR
void SetEntry(int32 index, const Entry& value) override;
#endif
void GetEntries(Array<Entry>& output) const override;
void Dispose() override;

5
Source/Engine/Content/Storage/FlaxPackage.h
View File

@@ -28,7 +28,10 @@ public:
bool HasAsset(const Guid& id) const override;
bool HasAsset(const AssetInfo& info) const override;
int32 GetEntriesCount() const override;
void GetEntry(int32 index, Entry& output) const override;
void GetEntry(int32 index, Entry& value) const override;
#if USE_EDITOR
void SetEntry(int32 index, const Entry& value) override;
#endif
void GetEntries(Array<Entry>& output) const override;
void Dispose() override;

167
Source/Engine/Content/Storage/FlaxStorage.cpp
View File

@@ -49,6 +49,15 @@ String AssetHeader::ToString() const
return String::Format(TEXT("ID: {0}, TypeName: {1}, Chunks Count: {2}"), ID, TypeName, GetChunksCount());
}
uint32 AssetInitData::GetHashCode() const
{
// Do not use Metadata/Dependencies because it may not be loaded (it's optional)
uint32 hashCode = GetHash(Header.ID);
hashCode = (hashCode * 397) ^ SerializedVersion;
hashCode = (hashCode * 397) ^ CustomData.Length();
return hashCode;
}
void FlaxChunk::RegisterUsage()
{
LastAccessTime = Platform::GetTimeSeconds();
@@ -605,6 +614,65 @@ bool FlaxStorage::Reload()
return failed;
}
bool FlaxStorage::ReloadSilent()
{
ScopeLock lock(_loadLocker);
if (!IsLoaded())
return false;
// Open file
auto stream = OpenFile();
if (stream == nullptr)
return true;
// Magic Code
uint32 magicCode;
stream->ReadUint32(&magicCode);
if (magicCode != MagicCode)
{
LOG(Warning, "Invalid asset magic code in {0}", ToString());
return true;
}
// Version
uint32 version;
stream->ReadUint32(&version);
if (version == 9)
{
_version = version;
// Custom storage data
CustomData customData;
stream->Read(customData);
// Entries
int32 assetsCount;
stream->ReadInt32(&assetsCount);
if (assetsCount != GetEntriesCount())
{
LOG(Warning, "Incorrect amount of assets ({} instead of {}) saved in {}", assetsCount, GetEntriesCount(), ToString());
return true;
}
for (int32 i = 0; i < assetsCount; i++)
{
SerializedEntryV9 se;
stream->ReadBytes(&se, sizeof(se));
Entry e(se.ID, se.TypeName.Data, se.Address);
SetEntry(i, e);
}
// TODO: load chunks metadata? compare against loaded chunks?
}
else
{
// Fallback to full reload
LOG(Warning, "Fallback to full reload of {0}", ToString());
Reload();
}
return false;
}
#endif
bool FlaxStorage::LoadAssetHeader(const Guid& id, AssetInitData& data)
@@ -809,7 +877,7 @@ FlaxChunk* FlaxStorage::AllocateChunk()
#if USE_EDITOR
bool FlaxStorage::Create(const StringView& path, const AssetInitData* data, int32 dataCount, bool silentMode, const CustomData* customData)
bool FlaxStorage::Create(const StringView& path, Span<AssetInitData> assets, bool silentMode, const CustomData* customData)
{
PROFILE_CPU();
ZoneText(*path, path.Length());
@@ -824,24 +892,32 @@ bool FlaxStorage::Create(const StringView& path, const AssetInitData* data, int3
return true;
// Create package
bool result = Create(stream, data, dataCount, customData);
bool result = Create(stream, assets, customData);
// Close file
Delete(stream);
// Reload storage container (only if not in silent mode)
if (storage && !silentMode)
// Reload storage container
if (storage && storage->IsLoaded())
{
storage->Reload();
if (silentMode)
{
// Silent data-only reload (loaded chunks location in file has been updated)
storage->ReloadSilent();
}
else
{
// Full reload
storage->Reload();
}
}
return result;
}
bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 dataCount, const CustomData* customData)
bool FlaxStorage::Create(WriteStream* stream, Span<AssetInitData> assets, const CustomData* customData)
{
// Validate inputs
if (data == nullptr || dataCount <= 0)
if (assets.Get() == nullptr || assets.Length() <= 0)
{
LOG(Warning, "Cannot create new package. No assets to write.");
return true;
@@ -849,27 +925,31 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
// Prepare data
Array<SerializedEntryV9, InlinedAllocation<1>> entries;
entries.Resize(dataCount);
entries.Resize(assets.Length());
Array<FlaxChunk*> chunks;
// Get all chunks
for (int32 i = 0; i < dataCount; i++)
data[i].Header.GetLoadedChunks(chunks);
int32 chunksCount = chunks.Count();
for (const AssetInitData& asset : assets)
{
for (FlaxChunk* chunk : asset.Header.Chunks)
{
if (chunk && chunk->IsLoaded())
chunks.Add(chunk);
}
}
const int32 chunksCount = chunks.Count();
// TODO: sort chunks by size? smaller ones first?
// Calculate start address of the first asset header location
// 0 -> Header -> Entries Count -> Entries -> Chunks Count -> Chunk Locations
int32 currentAddress = sizeof(Header)
+ sizeof(int32)
+ sizeof(SerializedEntryV9) * dataCount
+ sizeof(SerializedEntryV9) * assets.Length()
+ sizeof(int32)
+ (sizeof(FlaxChunk::Location) + sizeof(int32)) * chunksCount;
// Initialize entries offsets in the file
for (int32 i = 0; i < dataCount; i++)
for (int32 i = 0; i < assets.Length(); i++)
{
auto& asset = data[i];
const AssetInitData& asset = assets[i];
entries[i] = SerializedEntryV9(asset.Header.ID, asset.Header.TypeName, currentAddress);
// Move forward by asset header data size
@@ -934,8 +1014,8 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
stream->Write(mainHeader);
// Write asset entries
stream->WriteInt32(dataCount);
stream->WriteBytes(entries.Get(), sizeof(SerializedEntryV9) * dataCount);
stream->WriteInt32(assets.Length());
stream->WriteBytes(entries.Get(), sizeof(SerializedEntryV9) * assets.Length());
// Write chunk locations and meta
stream->WriteInt32(chunksCount);
@@ -948,20 +1028,18 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
}
#if ASSETS_LOADING_EXTRA_VERIFICATION
// Check calculated position of first asset header
if (dataCount > 0 && stream->GetPosition() != entries[0].Address)
if (assets.Length() > 0 && stream->GetPosition() != entries[0].Address)
{
LOG(Warning, "Error while asset header location computation.");
return true;
}
#endif
// Write asset headers
for (int32 i = 0; i < dataCount; i++)
for (int32 i = 0; i < assets.Length(); i++)
{
auto& header = data[i];
const AssetInitData& header = assets[i];
// ID
stream->Write(header.Header.ID);
@@ -974,9 +1052,10 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
stream->WriteUint32(header.SerializedVersion);
// Chunks mapping
for (FlaxChunk* chunk : header.Header.Chunks)
for (const FlaxChunk* chunk : header.Header.Chunks)
{
const int32 index = chunks.Find(chunk);
ASSERT_LOW_LAYER(index >= -1 && index <= ALL_ASSET_CHUNKS);
stream->WriteInt32(index);
}
@@ -998,14 +1077,12 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
}
#if ASSETS_LOADING_EXTRA_VERIFICATION
// Check calculated position of first asset chunk
if (chunksCount > 0 && stream->GetPosition() != chunks[0]->LocationInFile.Address)
{
LOG(Warning, "Error while asset data chunk location computation.");
return true;
}
#endif
// Write chunks data
@@ -1033,7 +1110,7 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d
return false;
}
bool FlaxStorage::Save(AssetInitData& data, bool silentMode)
bool FlaxStorage::Save(const AssetInitData& data, bool silentMode)
{
// Check if can modify the storage
if (!AllowDataModifications())
@@ -1475,12 +1552,21 @@ int32 FlaxFile::GetEntriesCount() const
return _asset.ID.IsValid() ? 1 : 0;
}
void FlaxFile::GetEntry(int32 index, Entry& output) const
void FlaxFile::GetEntry(int32 index, Entry& value) const
{
ASSERT(index == 0);
output = _asset;
value = _asset;
}
#if USE_EDITOR
void FlaxFile::SetEntry(int32 index, const Entry& value)
{
ASSERT(index == 0);
_asset = value;
}
#endif
void FlaxFile::GetEntries(Array<Entry>& output) const
{
if (_asset.ID.IsValid())
@@ -1544,19 +1630,36 @@ int32 FlaxPackage::GetEntriesCount() const
return _entries.Count();
}
void FlaxPackage::GetEntry(int32 index, Entry& output) const
void FlaxPackage::GetEntry(int32 index, Entry& value) const
{
ASSERT(index >= 0 && index < _entries.Count());
for (auto i = _entries.Begin(); i.IsNotEnd(); ++i)
{
if (index-- <= 0)
{
output = i->Value;
value = i->Value;
return;
}
}
}
#if USE_EDITOR
void FlaxPackage::SetEntry(int32 index, const Entry& value)
{
ASSERT(index >= 0 && index < _entries.Count());
for (auto i = _entries.Begin(); i.IsNotEnd(); ++i)
{
if (index-- <= 0)
{
i->Value = value;
return;
}
}
}
#endif
void FlaxPackage::GetEntries(Array<Entry>& output) const
{
_entries.GetValues(output);

40
Source/Engine/Content/Storage/FlaxStorage.h
View File

@@ -313,8 +313,17 @@ public:
/// Gets the asset entry at given index.
/// </summary>
/// <param name="index">The asset index.</param>
/// <param name="output">The output.</param>
virtual void GetEntry(int32 index, Entry& output) const = 0;
/// <param name="value">The result.</param>
virtual void GetEntry(int32 index, Entry& value) const = 0;
#if USE_EDITOR
/// <summary>
/// Sets the asset entry at given index.
/// </summary>
/// <param name="index">The asset index.</param>
/// <param name="value">The input value.</param>
virtual void SetEntry(int32 index, const Entry& value) = 0;
#endif
/// <summary>
/// Gets all the entries in the storage.
@@ -424,62 +433,58 @@ public:
public:
#if USE_EDITOR
/// <summary>
/// Saves the specified asset data to the storage container.
/// </summary>
/// <param name="data">The data to save.</param>
/// <param name="silentMode">In silent mode don't reload opened storage container that is using target file.</param>
/// <returns>True if cannot save, otherwise false</returns>
bool Save(AssetInitData& data, bool silentMode = false);
bool Save(const AssetInitData& data, bool silentMode = false);
/// <summary>
/// Creates new FlaxFile using specified asset data.
/// </summary>
/// <param name="path">The file path.</param>
/// <param name="data">The data to write.</param>
/// <param name="asset">The asset data to write.</param>
/// <param name="silentMode">In silent mode don't reload opened storage container that is using target file.</param>
/// <param name="customData">Custom options.</param>
/// <returns>True if cannot create package, otherwise false</returns>
FORCE_INLINE static bool Create(const StringView& path, const AssetInitData& data, bool silentMode = false, const CustomData* customData = nullptr)
FORCE_INLINE static bool Create(const StringView& path, const AssetInitData& asset, bool silentMode = false, const CustomData* customData = nullptr)
{
return Create(path, &data, 1, silentMode, customData);
return Create(path, ToSpan(&asset, 1), silentMode, customData);
}
/// <summary>
/// Creates new FlaxFile using specified assets data.
/// </summary>
/// <param name="path">The file path.</param>
/// <param name="data">The data to write.</param>
/// <param name="assets">The assets data to write.</param>
/// <param name="silentMode">In silent mode don't reload opened storage container that is using target file.</param>
/// <param name="customData">Custom options.</param>
/// <returns>True if cannot create package, otherwise false</returns>
FORCE_INLINE static bool Create(const StringView& path, const Array<AssetInitData>& data, bool silentMode = false, const CustomData* customData = nullptr)
FORCE_INLINE static bool Create(const StringView& path, const Array<AssetInitData>& assets, bool silentMode = false, const CustomData* customData = nullptr)
{
return Create(path, data.Get(), data.Count(), silentMode, customData);
return Create(path, ToSpan(assets), silentMode, customData);
}
/// <summary>
/// Creates new FlaxFile using specified assets data.
/// </summary>
/// <param name="path">The file path.</param>
/// <param name="data">The data to write.</param>
/// <param name="dataCount">The data size.</param>
/// <param name="assets">The assets data to write.</param>
/// <param name="silentMode">In silent mode don't reload opened storage container that is using target file.</param>
/// <param name="customData">Custom options.</param>
/// <returns>True if cannot create package, otherwise false</returns>
static bool Create(const StringView& path, const AssetInitData* data, int32 dataCount, bool silentMode = false, const CustomData* customData = nullptr);
static bool Create(const StringView& path, Span<AssetInitData> assets, bool silentMode = false, const CustomData* customData = nullptr);
/// <summary>
/// Creates new FlaxFile using specified assets data.
/// </summary>
/// <param name="stream">The output stream.</param>
/// <param name="data">The data to write.</param>
/// <param name="dataCount">The data size.</param>
/// <param name="assets">The assets data to write.</param>
/// <param name="customData">Custom options.</param>
/// <returns>True if cannot create package, otherwise false</returns>
static bool Create(WriteStream* stream, const AssetInitData* data, int32 dataCount, const CustomData* customData = nullptr);
static bool Create(WriteStream* stream, Span<AssetInitData> assets, const CustomData* customData = nullptr);
#endif
protected:
@@ -488,4 +493,5 @@ protected:
virtual void AddEntry(Entry& e) = 0;
FileReadStream* OpenFile();
virtual bool GetEntry(const Guid& id, Entry& e) = 0;
bool ReloadSilent();
};