diff --git a/Source/Editor/Cooker/Steps/CookAssetsStep.cpp b/Source/Editor/Cooker/Steps/CookAssetsStep.cpp index b9085da0d..0ba6ce861 100644 --- a/Source/Editor/Cooker/Steps/CookAssetsStep.cpp +++ b/Source/Editor/Cooker/Steps/CookAssetsStep.cpp @@ -368,11 +368,21 @@ bool CookAssetsStep::Process(CookingData& data, CacheData& cache, Asset* asset) // Virtual assets are not included into the build return false; } + const bool wasLoaded = asset->IsLoaded(); if (asset->WaitForLoaded()) { LOG(Error, "Failed to load asset \'{0}\'", asset->ToString()); return true; } + if (!wasLoaded) + { + // HACK: give some time to resave any old assets in Asset::onLoad after it's loaded + // This assumes that if Load Thread enters Asset::Save then it will get asset lock and hold it until asset is saved + // So we can take the same lock to wait for save end but first we need to wait for it to get that lock + // (in future try to handle it in a better way) + Platform::Sleep(5); + } + ScopeLock lock(asset->Locker); // Switch based on an asset type const auto asBinaryAsset = dynamic_cast(asset); diff --git a/Source/Engine/Content/BinaryAsset.cpp b/Source/Engine/Content/BinaryAsset.cpp index deb39856e..a78618678 100644 --- a/Source/Engine/Content/BinaryAsset.cpp +++ b/Source/Engine/Content/BinaryAsset.cpp @@ -372,6 +372,7 @@ bool BinaryAsset::SaveToAsset(const StringView& path, AssetInitData& data, bool const auto locks = storage->_chunksLock; storage->_chunksLock = 0; result = storage->Save(data, silentMode); + ASSERT(storage->_chunksLock == 0); storage->_chunksLock = locks; } else diff --git a/Source/Engine/Content/Storage/AssetHeader.h b/Source/Engine/Content/Storage/AssetHeader.h index ad5c8efb9..f8cfeac75 100644 --- a/Source/Engine/Content/Storage/AssetHeader.h +++ b/Source/Engine/Content/Storage/AssetHeader.h @@ -20,7 +20,7 @@ typedef uint16 AssetChunksFlag; #define ALL_ASSET_CHUNKS MAX_uint16 /// -/// Flax Asset header +/// Flax asset file header. /// struct FLAXENGINE_API AssetHeader { @@ -51,34 +51,6 @@ public: } public: - /// - /// Gets the chunks. - /// - /// The output data. - template - void GetChunks(Array& output) const - { - for (int32 i = 0; i < ASSET_FILE_DATA_CHUNKS; i++) - { - if (Chunks[i] != nullptr) - output.Add(Chunks[i]); - } - } - - /// - /// Gets the chunks that are loaded. - /// - /// The output data. - template - void GetLoadedChunks(Array& output) const - { - for (int32 i = 0; i < ASSET_FILE_DATA_CHUNKS; i++) - { - if (Chunks[i] != nullptr && Chunks[i]->IsLoaded()) - output.Add(Chunks[i]); - } - } - /// /// Gets the amount of created asset chunks. /// @@ -95,14 +67,13 @@ public: void UnlinkChunks(); /// - /// Gets string with a human-readable info about that header + /// Gets string with a human-readable info about that header. /// - /// Header info string String ToString() const; }; /// -/// Flax Asset header data +/// Flax asset header with data. /// struct FLAXENGINE_API AssetInitData { @@ -137,12 +108,5 @@ public: /// /// Gets the hash code. /// - uint32 GetHashCode() const - { - // Note: do not use Metadata/Dependencies because it may not be loaded (it's optional) - uint32 hashCode = GetHash(Header.ID); - hashCode = (hashCode * 397) ^ SerializedVersion; - hashCode = (hashCode * 397) ^ CustomData.Length(); - return hashCode; - } + uint32 GetHashCode() const; }; diff --git a/Source/Engine/Content/Storage/FlaxFile.h b/Source/Engine/Content/Storage/FlaxFile.h index 9a1da5244..b2fbac034 100644 --- a/Source/Engine/Content/Storage/FlaxFile.h +++ b/Source/Engine/Content/Storage/FlaxFile.h @@ -27,7 +27,10 @@ public: bool HasAsset(const Guid& id) const override; bool HasAsset(const AssetInfo& info) const override; int32 GetEntriesCount() const override; - void GetEntry(int32 index, Entry& output) const override; + void GetEntry(int32 index, Entry& value) const override; +#if USE_EDITOR + void SetEntry(int32 index, const Entry& value) override; +#endif void GetEntries(Array& output) const override; void Dispose() override; diff --git a/Source/Engine/Content/Storage/FlaxPackage.h b/Source/Engine/Content/Storage/FlaxPackage.h index 11637c1b1..3afc0cdb8 100644 --- a/Source/Engine/Content/Storage/FlaxPackage.h +++ b/Source/Engine/Content/Storage/FlaxPackage.h @@ -28,7 +28,10 @@ public: bool HasAsset(const Guid& id) const override; bool HasAsset(const AssetInfo& info) const override; int32 GetEntriesCount() const override; - void GetEntry(int32 index, Entry& output) const override; + void GetEntry(int32 index, Entry& value) const override; +#if USE_EDITOR + void SetEntry(int32 index, const Entry& value) override; +#endif void GetEntries(Array& output) const override; void Dispose() override; diff --git a/Source/Engine/Content/Storage/FlaxStorage.cpp b/Source/Engine/Content/Storage/FlaxStorage.cpp index 007238889..88c9a0d34 100644 --- a/Source/Engine/Content/Storage/FlaxStorage.cpp +++ b/Source/Engine/Content/Storage/FlaxStorage.cpp @@ -49,6 +49,15 @@ String AssetHeader::ToString() const return String::Format(TEXT("ID: {0}, TypeName: {1}, Chunks Count: {2}"), ID, TypeName, GetChunksCount()); } +uint32 AssetInitData::GetHashCode() const +{ + // Do not use Metadata/Dependencies because it may not be loaded (it's optional) + uint32 hashCode = GetHash(Header.ID); + hashCode = (hashCode * 397) ^ SerializedVersion; + hashCode = (hashCode * 397) ^ CustomData.Length(); + return hashCode; +} + void FlaxChunk::RegisterUsage() { LastAccessTime = Platform::GetTimeSeconds(); @@ -605,6 +614,65 @@ bool FlaxStorage::Reload() return failed; } +bool FlaxStorage::ReloadSilent() +{ + ScopeLock lock(_loadLocker); + if (!IsLoaded()) + return false; + + // Open file + auto stream = OpenFile(); + if (stream == nullptr) + return true; + + // Magic Code + uint32 magicCode; + stream->ReadUint32(&magicCode); + if (magicCode != MagicCode) + { + LOG(Warning, "Invalid asset magic code in {0}", ToString()); + return true; + } + + // Version + uint32 version; + stream->ReadUint32(&version); + if (version == 9) + { + _version = version; + + // Custom storage data + CustomData customData; + stream->Read(customData); + + // Entries + int32 assetsCount; + stream->ReadInt32(&assetsCount); + if (assetsCount != GetEntriesCount()) + { + LOG(Warning, "Incorrect amount of assets ({} instead of {}) saved in {}", assetsCount, GetEntriesCount(), ToString()); + return true; + } + for (int32 i = 0; i < assetsCount; i++) + { + SerializedEntryV9 se; + stream->ReadBytes(&se, sizeof(se)); + Entry e(se.ID, se.TypeName.Data, se.Address); + SetEntry(i, e); + } + + // TODO: load chunks metadata? compare against loaded chunks? + } + else + { + // Fallback to full reload + LOG(Warning, "Fallback to full reload of {0}", ToString()); + Reload(); + } + + return false; +} + #endif bool FlaxStorage::LoadAssetHeader(const Guid& id, AssetInitData& data) @@ -809,7 +877,7 @@ FlaxChunk* FlaxStorage::AllocateChunk() #if USE_EDITOR -bool FlaxStorage::Create(const StringView& path, const AssetInitData* data, int32 dataCount, bool silentMode, const CustomData* customData) +bool FlaxStorage::Create(const StringView& path, Span assets, bool silentMode, const CustomData* customData) { PROFILE_CPU(); ZoneText(*path, path.Length()); @@ -824,24 +892,32 @@ bool FlaxStorage::Create(const StringView& path, const AssetInitData* data, int3 return true; // Create package - bool result = Create(stream, data, dataCount, customData); + bool result = Create(stream, assets, customData); // Close file Delete(stream); - // Reload storage container (only if not in silent mode) - if (storage && !silentMode) + // Reload storage container + if (storage && storage->IsLoaded()) { - storage->Reload(); + if (silentMode) + { + // Silent data-only reload (loaded chunks location in file has been updated) + storage->ReloadSilent(); + } + else + { + // Full reload + storage->Reload(); + } } return result; } -bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 dataCount, const CustomData* customData) +bool FlaxStorage::Create(WriteStream* stream, Span assets, const CustomData* customData) { - // Validate inputs - if (data == nullptr || dataCount <= 0) + if (assets.Get() == nullptr || assets.Length() <= 0) { LOG(Warning, "Cannot create new package. No assets to write."); return true; @@ -849,27 +925,31 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d // Prepare data Array> entries; - entries.Resize(dataCount); + entries.Resize(assets.Length()); Array chunks; - - // Get all chunks - for (int32 i = 0; i < dataCount; i++) - data[i].Header.GetLoadedChunks(chunks); - int32 chunksCount = chunks.Count(); + for (const AssetInitData& asset : assets) + { + for (FlaxChunk* chunk : asset.Header.Chunks) + { + if (chunk && chunk->IsLoaded()) + chunks.Add(chunk); + } + } + const int32 chunksCount = chunks.Count(); // TODO: sort chunks by size? smaller ones first? // Calculate start address of the first asset header location // 0 -> Header -> Entries Count -> Entries -> Chunks Count -> Chunk Locations int32 currentAddress = sizeof(Header) + sizeof(int32) - + sizeof(SerializedEntryV9) * dataCount + + sizeof(SerializedEntryV9) * assets.Length() + sizeof(int32) + (sizeof(FlaxChunk::Location) + sizeof(int32)) * chunksCount; // Initialize entries offsets in the file - for (int32 i = 0; i < dataCount; i++) + for (int32 i = 0; i < assets.Length(); i++) { - auto& asset = data[i]; + const AssetInitData& asset = assets[i]; entries[i] = SerializedEntryV9(asset.Header.ID, asset.Header.TypeName, currentAddress); // Move forward by asset header data size @@ -934,8 +1014,8 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d stream->Write(mainHeader); // Write asset entries - stream->WriteInt32(dataCount); - stream->WriteBytes(entries.Get(), sizeof(SerializedEntryV9) * dataCount); + stream->WriteInt32(assets.Length()); + stream->WriteBytes(entries.Get(), sizeof(SerializedEntryV9) * assets.Length()); // Write chunk locations and meta stream->WriteInt32(chunksCount); @@ -948,20 +1028,18 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d } #if ASSETS_LOADING_EXTRA_VERIFICATION - // Check calculated position of first asset header - if (dataCount > 0 && stream->GetPosition() != entries[0].Address) + if (assets.Length() > 0 && stream->GetPosition() != entries[0].Address) { LOG(Warning, "Error while asset header location computation."); return true; } - #endif // Write asset headers - for (int32 i = 0; i < dataCount; i++) + for (int32 i = 0; i < assets.Length(); i++) { - auto& header = data[i]; + const AssetInitData& header = assets[i]; // ID stream->Write(header.Header.ID); @@ -974,9 +1052,10 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d stream->WriteUint32(header.SerializedVersion); // Chunks mapping - for (FlaxChunk* chunk : header.Header.Chunks) + for (const FlaxChunk* chunk : header.Header.Chunks) { const int32 index = chunks.Find(chunk); + ASSERT_LOW_LAYER(index >= -1 && index <= ALL_ASSET_CHUNKS); stream->WriteInt32(index); } @@ -998,14 +1077,12 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d } #if ASSETS_LOADING_EXTRA_VERIFICATION - // Check calculated position of first asset chunk if (chunksCount > 0 && stream->GetPosition() != chunks[0]->LocationInFile.Address) { LOG(Warning, "Error while asset data chunk location computation."); return true; } - #endif // Write chunks data @@ -1033,7 +1110,7 @@ bool FlaxStorage::Create(WriteStream* stream, const AssetInitData* data, int32 d return false; } -bool FlaxStorage::Save(AssetInitData& data, bool silentMode) +bool FlaxStorage::Save(const AssetInitData& data, bool silentMode) { // Check if can modify the storage if (!AllowDataModifications()) @@ -1475,12 +1552,21 @@ int32 FlaxFile::GetEntriesCount() const return _asset.ID.IsValid() ? 1 : 0; } -void FlaxFile::GetEntry(int32 index, Entry& output) const +void FlaxFile::GetEntry(int32 index, Entry& value) const { ASSERT(index == 0); - output = _asset; + value = _asset; } +#if USE_EDITOR + +void FlaxFile::SetEntry(int32 index, const Entry& value) +{ + ASSERT(index == 0); + _asset = value; +} +#endif + void FlaxFile::GetEntries(Array& output) const { if (_asset.ID.IsValid()) @@ -1544,19 +1630,36 @@ int32 FlaxPackage::GetEntriesCount() const return _entries.Count(); } -void FlaxPackage::GetEntry(int32 index, Entry& output) const +void FlaxPackage::GetEntry(int32 index, Entry& value) const { ASSERT(index >= 0 && index < _entries.Count()); for (auto i = _entries.Begin(); i.IsNotEnd(); ++i) { if (index-- <= 0) { - output = i->Value; + value = i->Value; return; } } } +#if USE_EDITOR + +void FlaxPackage::SetEntry(int32 index, const Entry& value) +{ + ASSERT(index >= 0 && index < _entries.Count()); + for (auto i = _entries.Begin(); i.IsNotEnd(); ++i) + { + if (index-- <= 0) + { + i->Value = value; + return; + } + } +} + +#endif + void FlaxPackage::GetEntries(Array& output) const { _entries.GetValues(output); diff --git a/Source/Engine/Content/Storage/FlaxStorage.h b/Source/Engine/Content/Storage/FlaxStorage.h index 09e8c0bdd..0d5965311 100644 --- a/Source/Engine/Content/Storage/FlaxStorage.h +++ b/Source/Engine/Content/Storage/FlaxStorage.h @@ -313,8 +313,17 @@ public: /// Gets the asset entry at given index. /// /// The asset index. - /// The output. - virtual void GetEntry(int32 index, Entry& output) const = 0; + /// The result. + virtual void GetEntry(int32 index, Entry& value) const = 0; + +#if USE_EDITOR + /// + /// Sets the asset entry at given index. + /// + /// The asset index. + /// The input value. + virtual void SetEntry(int32 index, const Entry& value) = 0; +#endif /// /// Gets all the entries in the storage. @@ -424,62 +433,58 @@ public: public: #if USE_EDITOR - /// /// Saves the specified asset data to the storage container. /// /// The data to save. /// In silent mode don't reload opened storage container that is using target file. /// True if cannot save, otherwise false - bool Save(AssetInitData& data, bool silentMode = false); + bool Save(const AssetInitData& data, bool silentMode = false); /// /// Creates new FlaxFile using specified asset data. /// /// The file path. - /// The data to write. + /// The asset data to write. /// In silent mode don't reload opened storage container that is using target file. /// Custom options. /// True if cannot create package, otherwise false - FORCE_INLINE static bool Create(const StringView& path, const AssetInitData& data, bool silentMode = false, const CustomData* customData = nullptr) + FORCE_INLINE static bool Create(const StringView& path, const AssetInitData& asset, bool silentMode = false, const CustomData* customData = nullptr) { - return Create(path, &data, 1, silentMode, customData); + return Create(path, ToSpan(&asset, 1), silentMode, customData); } /// /// Creates new FlaxFile using specified assets data. /// /// The file path. - /// The data to write. + /// The assets data to write. /// In silent mode don't reload opened storage container that is using target file. /// Custom options. /// True if cannot create package, otherwise false - FORCE_INLINE static bool Create(const StringView& path, const Array& data, bool silentMode = false, const CustomData* customData = nullptr) + FORCE_INLINE static bool Create(const StringView& path, const Array& assets, bool silentMode = false, const CustomData* customData = nullptr) { - return Create(path, data.Get(), data.Count(), silentMode, customData); + return Create(path, ToSpan(assets), silentMode, customData); } /// /// Creates new FlaxFile using specified assets data. /// /// The file path. - /// The data to write. - /// The data size. + /// The assets data to write. /// In silent mode don't reload opened storage container that is using target file. /// Custom options. /// True if cannot create package, otherwise false - static bool Create(const StringView& path, const AssetInitData* data, int32 dataCount, bool silentMode = false, const CustomData* customData = nullptr); + static bool Create(const StringView& path, Span assets, bool silentMode = false, const CustomData* customData = nullptr); /// /// Creates new FlaxFile using specified assets data. /// /// The output stream. - /// The data to write. - /// The data size. + /// The assets data to write. /// Custom options. /// True if cannot create package, otherwise false - static bool Create(WriteStream* stream, const AssetInitData* data, int32 dataCount, const CustomData* customData = nullptr); - + static bool Create(WriteStream* stream, Span assets, const CustomData* customData = nullptr); #endif protected: @@ -488,4 +493,5 @@ protected: virtual void AddEntry(Entry& e) = 0; FileReadStream* OpenFile(); virtual bool GetEntry(const Guid& id, Entry& e) = 0; + bool ReloadSilent(); };