// Copyright (c) Wojciech Figat. All rights reserved. #include "Content.h" #include "JsonAsset.h" #include "SceneReference.h" #include "Cache/AssetsCache.h" #include "Storage/ContentStorageManager.h" #include "Storage/JsonStorageProxy.h" #include "Factories/IAssetFactory.h" #include "Loading/LoadingThread.h" #include "Loading/ContentLoadTask.h" #include "Engine/Core/Log.h" #include "Engine/Core/LogContext.h" #include "Engine/Core/Types/String.h" #include "Engine/Core/ObjectsRemovalService.h" #include "Engine/Serialization/Serialization.h" #include "Engine/Platform/FileSystem.h" #include "Engine/Platform/ConditionVariable.h" #include "Engine/Platform/Thread.h" #include "Engine/Platform/CPUInfo.h" #include "Engine/Threading/Threading.h" #include "Engine/Threading/MainThreadTask.h" #include "Engine/Threading/ConcurrentTaskQueue.h" #include "Engine/Graphics/Graphics.h" #include "Engine/Engine/Engine.h" #include "Engine/Engine/EngineService.h" #include "Engine/Engine/Time.h" #include "Engine/Engine/Globals.h" #include "Engine/Level/Types.h" #include "Engine/Profiler/ProfilerCPU.h" #include "Engine/Profiler/ProfilerMemory.h" #include "Engine/Scripting/ManagedCLR/MClass.h" #include "Engine/Scripting/Internal/InternalCalls.h" #include "Engine/Scripting/Scripting.h" #if USE_EDITOR #include "Editor/Editor.h" #include "Editor/ProjectInfo.h" #endif #if ENABLE_ASSETS_DISCOVERY #include "Engine/Core/Collections/HashSet.h" #endif #if USE_EDITOR && PLATFORM_WINDOWS #include "Engine/Platform/Win32/IncludeWindowsHeaders.h" #include #endif TimeSpan Content::AssetsUpdateInterval = TimeSpan::FromMilliseconds(500); TimeSpan Content::AssetsUnloadInterval = TimeSpan::FromSeconds(10); Delegate Content::AssetDisposing; Delegate Content::AssetReloading; String AssetInfo::ToString() const { return String::Format(TEXT("ID: {0}, TypeName: {1}, Path: \'{2}\'"), ID, TypeName, Path); } void FLAXENGINE_API Serialization::Serialize(ISerializable::SerializeStream& stream, const SceneReference& v, const void* otherObj) { Serialize(stream, v.ID, otherObj); } void FLAXENGINE_API Serialization::Deserialize(ISerializable::DeserializeStream& stream, SceneReference& v, ISerializeModifier* modifier) { Deserialize(stream, v.ID, modifier); } namespace { // Assets CriticalSection AssetsLocker; Dictionary Assets(2048); Array LoadCallAssets(PLATFORM_THREADS_LIMIT); CriticalSection LoadedAssetsToInvokeLocker; Array LoadedAssetsToInvoke(64); Array ToUnload; // Assets Registry Stuff AssetsCache Cache; // Loading assets THREADLOCAL LoadingThread* ThisLoadThread = nullptr; LoadingThread* MainLoadThread = nullptr; Array LoadThreads; ConcurrentTaskQueue LoadTasks; ConditionVariable LoadTasksSignal; CriticalSection LoadTasksMutex; // Unloading assets Dictionary UnloadQueue; TimeSpan LastUnloadCheckTime(0); bool IsExiting = false; #if ENABLE_ASSETS_DISCOVERY DateTime LastWorkspaceDiscovery; CriticalSection WorkspaceDiscoveryLocker; #endif } #if ENABLE_ASSETS_DISCOVERY bool findAsset(const Guid& id, const String& directory, Array& tmpCache, AssetInfo& info); #endif class ContentService : public EngineService { public: ContentService() : EngineService(TEXT("Content"), -600) { } bool Init() override; void Update() override; void LateUpdate() override; void BeforeExit() override; void Dispose() override; }; ContentService ContentServiceInstance; bool ContentService::Init() { PROFILE_MEM(Content); // Load assets registry Cache.Init(); // Create loading threads const CPUInfo cpuInfo = Platform::GetCPUInfo(); const int32 count = Math::Clamp(Math::CeilToInt(LOADING_THREAD_PER_LOGICAL_CORE * (float)cpuInfo.LogicalProcessorCount), 1, 12); LOG(Info, "Creating {0} content loading threads...", count); MainLoadThread = New(); ThisLoadThread = MainLoadThread; LoadThreads.Resize(count); for (int32 i = 0; i < count; i++) { auto thread = New(); LoadThreads[i] = thread; if (thread->Start(String::Format(TEXT("Load Thread {0}"), i))) { LOG(Fatal, "Cannot spawn content thread {0}/{1}", i, count); Delete(thread); return true; } } return false; } void ContentService::Update() { PROFILE_CPU(); ScopeLock lock(LoadedAssetsToInvokeLocker); // Broadcast `OnLoaded` events while (LoadedAssetsToInvoke.HasItems()) { auto asset = LoadedAssetsToInvoke.Dequeue(); asset->onLoaded_MainThread(); } } void ContentService::LateUpdate() { PROFILE_CPU(); PROFILE_MEM(Content); // Check if need to perform an update of unloading assets const TimeSpan timeNow = Time::Update.UnscaledTime; if (timeNow - LastUnloadCheckTime < Content::AssetsUpdateInterval) return; LastUnloadCheckTime = timeNow; AssetsLocker.Lock(); // Verify all assets for (auto i = Assets.Begin(); i.IsNotEnd(); ++i) { Asset* asset = i->Value; // Check if has no references and is not during unloading if (asset->GetReferencesCount() <= 0 && !UnloadQueue.ContainsKey(asset)) { // Add to removes UnloadQueue.Add(asset, timeNow); } } // Find assets to unload in unload queue ToUnload.Clear(); for (auto i = UnloadQueue.Begin(); i != UnloadQueue.End(); ++i) { // Check if asset gain any new reference or if need to unload it if (i->Key->GetReferencesCount() > 0 || timeNow - i->Value >= Content::AssetsUnloadInterval) { ToUnload.Add(i->Key); } } // Unload marked assets for (int32 i = 0; i < ToUnload.Count(); i++) { Asset* asset = ToUnload[i]; // Check if has no references if (asset->GetReferencesCount() <= 0) { Content::UnloadAsset(asset); } // Remove from unload queue UnloadQueue.Remove(asset); } AssetsLocker.Unlock(); // Update cache (for longer sessions it will help to reduce cache misses) Cache.Save(); } void ContentService::BeforeExit() { // Signal threads to end work soon for (auto thread : LoadThreads) thread->NotifyExit(); LoadTasksSignal.NotifyAll(); } void ContentService::Dispose() { IsExiting = true; // Save assets registry before engine closing Cache.Save(); // Flush objects (some asset-related objects/references may be pending to delete) ObjectsRemovalService::Flush(); // Unload all remaining assets { ScopeLock lock(AssetsLocker); for (auto i = Assets.Begin(); i.IsNotEnd(); ++i) { i->Value->DeleteObject(); } } // Flush objects (some assets may be pending to delete) ObjectsRemovalService::Flush(); // NOW dispose graphics device - where there is no loaded assets at all Graphics::DisposeDevice(); // Exit all load threads for (auto thread : LoadThreads) thread->NotifyExit(); LoadTasksSignal.NotifyAll(); for (auto thread : LoadThreads) thread->Join(); LoadThreads.ClearDelete(); Delete(MainLoadThread); MainLoadThread = nullptr; ThisLoadThread = nullptr; // Cancel all remaining tasks (no chance to execute them) LoadTasks.CancelAll(); } IAssetFactory::Collection& IAssetFactory::Get() { static Collection Factories(1024); return Factories; } LoadingThread::LoadingThread() : _exitFlag(false) , _thread(nullptr) , _totalTasksDoneCount(0) { } LoadingThread::~LoadingThread() { if (_thread != nullptr) { _thread->Kill(true); Delete(_thread); } } void LoadingThread::NotifyExit() { Platform::InterlockedIncrement(&_exitFlag); } void LoadingThread::Join() { auto thread = _thread; if (thread) thread->Join(); } bool LoadingThread::Start(const String& name) { ASSERT(_thread == nullptr && name.HasChars()); // Create new thread auto thread = Thread::Create(this, name, ThreadPriority::Normal); if (thread == nullptr) return true; _thread = thread; return false; } void LoadingThread::Run(ContentLoadTask* job) { ASSERT(job); job->Execute(); _totalTasksDoneCount++; } String LoadingThread::ToString() const { return String::Format(TEXT("Loading Thread {0}"), _thread ? _thread->GetID() : 0); } int32 LoadingThread::Run() { PROFILE_MEM(Content); #if USE_EDITOR && PLATFORM_WINDOWS // Initialize COM // TODO: maybe add sth to Thread::Create to indicate that thread will use COM stuff const auto result = CoInitializeEx(nullptr, COINIT_MULTITHREADED); if (FAILED(result)) { LOG(Error, "Failed to init COM for WIC texture importing! Result: {0:x}", static_cast(result)); return -1; } #endif #ifdef LOADING_THREAD_AFFINITY_MASK Platform::SetThreadAffinityMask(LOADING_THREAD_AFFINITY_MASK(LoadThreads.Find(this))); #endif ContentLoadTask* task; ThisLoadThread = this; MONO_THREAD_INFO_TYPE* monoThreadInfo = nullptr; while (Platform::AtomicRead(&_exitFlag) == 0) { if (LoadTasks.try_dequeue(task)) { Run(task); MONO_THREAD_INFO_GET(monoThreadInfo); } else { MONO_ENTER_GC_SAFE_WITH_INFO(monoThreadInfo); LoadTasksMutex.Lock(); LoadTasksSignal.Wait(LoadTasksMutex); LoadTasksMutex.Unlock(); MONO_EXIT_GC_SAFE_WITH_INFO; } } ThisLoadThread = nullptr; return 0; } void LoadingThread::Exit() { LOG(Info, "Content thread '{0}' exited. Load calls: {1}", _thread->GetName(), _totalTasksDoneCount); } String ContentLoadTask::ToString() const { return String::Format(TEXT("Content Load Task ({})"), (int32)GetState()); } void ContentLoadTask::Enqueue() { LoadTasks.Add(this); LoadTasksSignal.NotifyOne(); } bool ContentLoadTask::Run() { const auto result = run(); const bool failed = result != Result::Ok; if (failed) { LOG(Warning, "\'{0}\' failed with result: {1}", ToString(), ToString(result)); } return failed; } AssetsCache* Content::GetRegistry() { return &Cache; } #if USE_EDITOR bool FindAssets(const ProjectInfo* project, HashSet& projects, const Guid& id, Array& tmpCache, AssetInfo& info) { if (projects.Contains(project)) return false; projects.Add(project); bool found = findAsset(id, project->ProjectFolderPath / TEXT("Content"), tmpCache, info); for (const auto& reference : project->References) { if (reference.Project) found |= FindAssets(reference.Project, projects, id, tmpCache, info); } return found; } #endif bool Content::GetAssetInfo(const Guid& id, AssetInfo& info) { if (!id.IsValid()) return false; #if ENABLE_ASSETS_DISCOVERY // Find asset in registry if (Cache.FindAsset(id, info)) return true; PROFILE_CPU(); PROFILE_MEM(Content); // Locking injects some stalls but we need to make it safe (only one thread can pass though it at once) ScopeLock lock(WorkspaceDiscoveryLocker); // Check if we can search workspace // Note: we want to limit searching frequency due to high I/O usage and thread stall // We also perform full workspace discovery so all new assets will be found auto now = DateTime::NowUTC(); auto diff = now - LastWorkspaceDiscovery; if (diff <= TimeSpan::FromSeconds(5)) { //LOG(Warning, "Cannot perform workspace scan for '{1}'. Too often call. Time diff: {0} ms", static_cast(diff.GetTotalMilliseconds()), id); return false; } LastWorkspaceDiscovery = now; // Try to find an asset within the project, engine, plugins workspace folders DateTime startTime = now; int32 startCount = Cache.Size(); Array tmpCache(1024); #if USE_EDITOR HashSet projects; bool found = FindAssets(Editor::Project, projects, id, tmpCache, info); #else bool found = findAsset(id, Globals::ProjectContentFolder, tmpCache, info); #endif if (found) { LOG(Info, "Workspace searching time: {0} ms, new assets found: {1}", static_cast((DateTime::NowUTC() - startTime).GetTotalMilliseconds()), Cache.Size() - startCount); return true; } //LOG(Warning, "Cannot find {0}.", id); return false; #else // Find asset in registry return Cache.FindAsset(id, info); #endif } bool Content::GetAssetInfo(const StringView& path, AssetInfo& info) { #if ENABLE_ASSETS_DISCOVERY // Find asset in registry if (Cache.FindAsset(path, info)) return true; if (!FileSystem::FileExists(path)) return false; PROFILE_CPU(); PROFILE_MEM(Content); const auto extension = FileSystem::GetExtension(path).ToLower(); // Check if it's a binary asset if (ContentStorageManager::IsFlaxStorageExtension(extension)) { // Skip packages in editor (results in conflicts with build game packages if deployed inside project folder) #if USE_EDITOR if (extension == PACKAGE_FILES_EXTENSION) return false; #endif // Open storage auto storage = ContentStorageManager::GetStorage(path); if (storage) { #if BUILD_DEBUG || FLAX_TESTS ASSERT(storage->GetPath() == path); #endif // Register assets from the storage container (will handle duplicated IDs) Cache.RegisterAssets(storage); return Cache.FindAsset(path, info); } } // Check for json resource else if (JsonStorageProxy::IsValidExtension(extension)) { // Check Json storage layer Guid jsonId; String jsonTypeName; if (JsonStorageProxy::GetAssetInfo(path, jsonId, jsonTypeName)) { // Register asset Cache.RegisterAsset(jsonId, jsonTypeName, path); return Cache.FindAsset(path, info); } } return false; #else // Find asset in registry return Cache.FindAsset(path, info); #endif } StringView Content::GetEditorAssetPath(const Guid& id) { return Cache.GetEditorAssetPath(id); } Array Content::GetAllAssets() { Array result; Cache.GetAll(result); return result; } Array Content::GetAllAssetsByType(const MClass* type) { Array result; CHECK_RETURN(type, result); Cache.GetAllByTypeName(String(type->GetFullName()), result); return result; } IAssetFactory* Content::GetAssetFactory(const StringView& typeName) { IAssetFactory* result = nullptr; IAssetFactory::Get().TryGet(typeName, result); return result; } IAssetFactory* Content::GetAssetFactory(const AssetInfo& assetInfo) { IAssetFactory* result = nullptr; if (!IAssetFactory::Get().TryGet(assetInfo.TypeName, result)) { // Check if it's a json asset (in editor only) // In build game all asset factories are valid and json assets are cooked into binary packages #if USE_EDITOR if (assetInfo.Path.EndsWith(DEFAULT_JSON_EXTENSION_DOT)) #endif { IAssetFactory::Get().TryGet(JsonAsset::TypeName, result); } } return result; } String Content::CreateTemporaryAssetPath() { return Globals::TemporaryFolder / (Guid::New().ToString(Guid::FormatType::N) + ASSET_FILES_EXTENSION_WITH_DOT); } ContentStats Content::GetStats() { ContentStats stats; AssetsLocker.Lock(); stats.AssetsCount = Assets.Count(); int32 loadFailedCount = 0; for (const auto& e : Assets) { if (e.Value->IsLoaded()) stats.LoadedAssetsCount++; else if (e.Value->LastLoadFailed()) loadFailedCount++; if (e.Value->IsVirtual()) stats.VirtualAssetsCount++; } stats.LoadingAssetsCount = stats.AssetsCount - loadFailedCount - stats.LoadedAssetsCount; AssetsLocker.Unlock(); return stats; } Asset* Content::LoadAsyncInternal(const StringView& internalPath, const MClass* type) { CHECK_RETURN(type, nullptr); const auto scriptingType = Scripting::FindScriptingType(type->GetFullName()); if (scriptingType) return LoadAsyncInternal(internalPath, scriptingType); LOG(Error, "Failed to find asset type '{0}'.", String(type->GetFullName())); return nullptr; } Asset* Content::LoadAsyncInternal(const StringView& internalPath, const ScriptingTypeHandle& type) { PROFILE_MEM(Content); #if USE_EDITOR const String path = Globals::EngineContentFolder / internalPath + ASSET_FILES_EXTENSION_WITH_DOT; if (!FileSystem::FileExists(path)) { LOG(Error, "Missing file \'{0}\'", path); return nullptr; } #else const String path = Globals::ProjectContentFolder / internalPath + ASSET_FILES_EXTENSION_WITH_DOT; #endif const auto asset = LoadAsync(path, type); if (asset == nullptr) { LOG(Error, "Failed to load \'{0}\' (type: {1})", internalPath, type.ToString()); } return asset; } Asset* Content::LoadAsyncInternal(const Char* internalPath, const ScriptingTypeHandle& type) { return LoadAsyncInternal(StringView(internalPath), type); } FLAXENGINE_API Asset* LoadAsset(const Guid& id, const ScriptingTypeHandle& type) { return Content::LoadAsync(id, type); } Asset* Content::LoadAsync(const StringView& path, const MClass* type) { CHECK_RETURN(type, nullptr); const auto scriptingType = Scripting::FindScriptingType(type->GetFullName()); if (scriptingType) return LoadAsync(path, scriptingType); LOG(Error, "Failed to find asset type '{0}'.", String(type->GetFullName())); return nullptr; } Asset* Content::LoadAsync(const StringView& path, const ScriptingTypeHandle& type) { PROFILE_MEM(Content); // Ensure path is in a valid format String pathNorm(path); ContentStorageManager::FormatPath(pathNorm); const StringView filePath = pathNorm; #if USE_EDITOR if (!FileSystem::FileExists(filePath)) { LOG(Error, "Missing file \'{0}\'", filePath); return nullptr; } #endif AssetInfo assetInfo; if (GetAssetInfo(filePath, assetInfo)) { return LoadAsync(assetInfo.ID, type); } return nullptr; } Array Content::GetAssets() { Array assets; AssetsLocker.Lock(); Assets.GetValues(assets); AssetsLocker.Unlock(); return assets; } const Dictionary& Content::GetAssetsRaw() { AssetsLocker.Lock(); AssetsLocker.Unlock(); return Assets; } Asset* Content::LoadAsync(const Guid& id, const MClass* type) { CHECK_RETURN(type, nullptr); const auto scriptingType = Scripting::FindScriptingType(type->GetFullName()); if (scriptingType) return LoadAsync(id, scriptingType); LOG(Error, "Failed to find asset type '{0}'.", String(type->GetFullName())); return nullptr; } Asset* Content::GetAsset(const StringView& outputPath) { if (outputPath.IsEmpty()) return nullptr; PROFILE_CPU(); ScopeLock lock(AssetsLocker); for (auto i = Assets.Begin(); i.IsNotEnd(); ++i) { if (i->Value->GetPath() == outputPath) { return i->Value; } } return nullptr; } Asset* Content::GetAsset(const Guid& id) { Asset* result = nullptr; AssetsLocker.Lock(); Assets.TryGet(id, result); AssetsLocker.Unlock(); return result; } void Content::DeleteAsset(Asset* asset) { if (asset == nullptr || asset->_deleteFileOnUnload) return; LOG(Info, "Deleting asset {0}...", asset->ToString()); // Ensure that asset is loaded (easier than cancel in-flight loading) asset->WaitForLoaded(); // Mark asset for delete queue (delete it after auto unload) asset->_deleteFileOnUnload = true; // Unload asset->DeleteObject(); } void Content::DeleteAsset(const StringView& path) { PROFILE_CPU(); // Try to delete already loaded asset Asset* asset = GetAsset(path); if (asset != nullptr) { DeleteAsset(asset); return; } #if USE_EDITOR ScopeLock locker(AssetsLocker); // Remove from registry AssetInfo info; if (Cache.DeleteAsset(path, &info)) { LOG(Info, "Deleting asset '{0}':{1}({2})", path, info.ID, info.TypeName); } else { LOG(Info, "Deleting asset '{0}':{1}({2})", path, TEXT("?"), TEXT("?")); info.ID = Guid::Empty; } // Delete file deleteFileSafety(path, info.ID); #endif } void Content::deleteFileSafety(const StringView& path, const Guid& id) { if (!id.IsValid()) { LOG(Warning, "Cannot remove file \'{0}\'. Given ID is invalid.", path); return; } PROFILE_CPU(); // Ensure that file has the same ID (prevent from deleting different assets) auto storage = ContentStorageManager::TryGetStorage(path); if (storage) { storage->CloseFileHandles(); // Close file handle to allow removing it if (!storage->HasAsset(id)) { LOG(Warning, "Cannot remove file \'{0}\'. It doesn\'t contain asset {1}.", path, id); return; } } #if PLATFORM_WINDOWS || PLATFORM_LINUX // Safety way - move file to the recycle bin if (FileSystem::MoveFileToRecycleBin(path)) { LOG(Warning, "Failed to move file to Recycle Bin. Path: \'{0}\'", path); } #else // Remove file if (FileSystem::DeleteFile(path)) { LOG(Warning, "Failed to delete file Path: \'{0}\'", path); } #endif } #if !COMPILE_WITHOUT_CSHARP #include "Engine/Scripting/ManagedCLR/MUtils.h" void* Content::GetAssetsInternal() { AssetsLocker.Lock(); MArray* result = MCore::Array::New(Asset::TypeInitializer.GetClass(), Assets.Count()); int32 i = 0; for (const auto& e : Assets) MCore::GC::WriteArrayRef(result, e.Value->GetOrCreateManagedInstance(), i++); AssetsLocker.Unlock(); return result; } #endif #if USE_EDITOR bool Content::RenameAsset(const StringView& oldPath, const StringView& newPath) { ASSERT(IsInMainThread()); // Cache data Asset* oldAsset = GetAsset(oldPath); Asset* newAsset = GetAsset(newPath); // Validate name if (newAsset != nullptr && newAsset != oldAsset) { LOG(Error, "Invalid name '{0}' when trying to rename '{1}'.", newPath, oldPath); return true; } // Ensure asset is ready for renaming if (oldAsset) { // Wait for data loaded if (oldAsset->WaitForLoaded()) { LOG(Error, "Failed to load asset '{0}'.", oldAsset->ToString()); return true; } // Unload // Don't unload asset fully, only release ref to file, don't call OnUnload so managed asset and all refs will remain alive oldAsset->releaseStorage(); //oldAsset->onUnload_MainThread(); //ScopeLock lock(oldAsset->Locker); //oldAsset->unload(true); } // Ensure to unlock file ContentStorageManager::EnsureAccess(oldPath); // Move file if (FileSystem::MoveFile(newPath, oldPath)) { LOG(Error, "Cannot move file '{0}' to '{1}'", oldPath, newPath); return true; } // Update cache Cache.RenameAsset(oldPath, newPath); ContentStorageManager::OnRenamed(oldPath, newPath); // Check if is loaded if (oldAsset) { // Rename internal call oldAsset->onRename(newPath); // Load //ScopeLock lock(oldAsset->Locker); //oldAsset->startLoading(); } return false; } bool Content::FastTmpAssetClone(const StringView& path, String& resultPath) { ASSERT(path.HasChars()); const String dstPath = Globals::TemporaryFolder / Guid::New().ToString(Guid::FormatType::D) + ASSET_FILES_EXTENSION_WITH_DOT; if (CloneAssetFile(dstPath, path, Guid::New())) return true; resultPath = dstPath; return false; } class CloneAssetFileTask : public MainThreadTask { public: StringView dstPath; StringView srcPath; Guid dstId; bool* output; protected: bool Run() override { *output = Content::CloneAssetFile(dstPath, srcPath, dstId); return false; } }; bool Content::CloneAssetFile(const StringView& dstPath, const StringView& srcPath, const Guid& dstId) { // Best to run this on the main thread to avoid clone conflicts. if (IsInMainThread()) { PROFILE_CPU(); ASSERT(FileSystem::AreFilePathsEqual(srcPath, dstPath) == false && dstId.IsValid()); LOG(Info, "Cloning asset \'{0}\' to \'{1}\'({2}).", srcPath, dstPath, dstId); // Check source file if (!FileSystem::FileExists(srcPath)) { LOG(Warning, "Missing source file."); return true; } // Special case for json resources if (JsonStorageProxy::IsValidExtension(FileSystem::GetExtension(srcPath).ToLower())) { if (FileSystem::CopyFile(dstPath, srcPath)) { LOG(Warning, "Cannot copy file to destination."); return true; } if (JsonStorageProxy::ChangeId(dstPath, dstId)) { LOG(Warning, "Cannot change asset ID."); return true; } return false; } // Check if destination file is missing if (!FileSystem::FileExists(dstPath)) { // Use quick file copy if (FileSystem::CopyFile(dstPath, srcPath)) { LOG(Warning, "Cannot copy file to destination."); return true; } // Change ID auto storage = ContentStorageManager::GetStorage(dstPath); FlaxStorage::Entry e; storage->GetEntry(0, e); if (storage == nullptr || storage->ChangeAssetID(e, dstId)) { LOG(Warning, "Cannot change asset ID."); return true; } } else { // Use temporary file String tmpPath = Globals::TemporaryFolder / Guid::New().ToString(Guid::FormatType::D); if (FileSystem::CopyFile(tmpPath, srcPath)) { LOG(Warning, "Cannot copy file."); return true; } // Change asset ID { auto storage = ContentStorageManager::GetStorage(tmpPath); if (!storage) { LOG(Warning, "Cannot change asset ID."); return true; } FlaxStorage::Entry e; storage->GetEntry(0, e); if (storage->ChangeAssetID(e, dstId)) { LOG(Warning, "Cannot change asset ID."); return true; } } // Unlock destination file ContentStorageManager::EnsureAccess(dstPath); // Copy temp file to the destination if (FileSystem::CopyFile(dstPath, tmpPath)) { LOG(Warning, "Cannot copy file to destination."); return true; } // Cleanup FileSystem::DeleteFile(tmpPath); // Reload storage if (auto storage = ContentStorageManager::GetStorage(dstPath, false)) { storage->Reload(); } } } else { CloneAssetFileTask* task = New(); task->dstId = dstId; task->dstPath = dstPath; task->srcPath = srcPath; bool result = false; task->output = &result; task->Start(); task->Wait(); return result; } return false; } #endif void Content::UnloadAsset(Asset* asset) { if (asset == nullptr) return; asset->DeleteObject(); } Asset* Content::CreateVirtualAsset(const MClass* type) { CHECK_RETURN(type, nullptr); const auto scriptingType = Scripting::FindScriptingType(type->GetFullName()); if (scriptingType) return CreateVirtualAsset(scriptingType); LOG(Error, "Failed to find asset type '{0}'.", String(type->GetFullName())); return nullptr; } Asset* Content::CreateVirtualAsset(const ScriptingTypeHandle& type) { PROFILE_CPU(); PROFILE_MEM(Content); auto& assetType = type.GetType(); // Init mock asset info AssetInfo info; info.ID = Guid::New(); info.TypeName = String(assetType.Fullname.Get(), assetType.Fullname.Length()); info.Path = CreateTemporaryAssetPath(); // Find asset factory based in its type auto factory = GetAssetFactory(info); if (factory == nullptr) { LOG(Error, "Cannot find virtual asset factory."); return nullptr; } if (!factory->SupportsVirtualAssets()) { LOG(Error, "Cannot create virtual asset of type '{0}'.", info.TypeName); return nullptr; } // Create asset object PROFILE_MEM_BEGIN(ContentAssets); auto asset = factory->NewVirtual(info); PROFILE_MEM_END(); if (asset == nullptr) { LOG(Error, "Cannot create virtual asset object."); return nullptr; } asset->RegisterObject(); // Call initializer function PROFILE_MEM_BEGIN(ContentAssets); asset->InitAsVirtual(); PROFILE_MEM_END(); // Register asset AssetsLocker.Lock(); ASSERT(!Assets.ContainsKey(asset->GetID())); Assets.Add(asset->GetID(), asset); AssetsLocker.Unlock(); return asset; } void Content::WaitForTask(ContentLoadTask* loadingTask, double timeoutInMilliseconds) { // Check if call is made from the Loading Thread and task has not been taken yet auto thread = ThisLoadThread; if (thread != nullptr) { // Note: to reproduce this case just include material into material (use layering). // So during loading first material it will wait for child materials loaded calling this function const double timeoutInSeconds = timeoutInMilliseconds * 0.001; const double startTime = Platform::GetTimeSeconds(); int32 loopCounter = 0; Task* task = loadingTask; Array> localQueue; #define CHECK_CONDITIONS() (!Engine::ShouldExit() && (timeoutInSeconds <= 0.0 || Platform::GetTimeSeconds() - startTime < timeoutInSeconds)) do { // Give opportunity for other threads to use the current core if (loopCounter == 0) ; // First run is fast else if (loopCounter < 10) Platform::Yield(); else Platform::Sleep(1); loopCounter++; // Try to execute content tasks while (task->IsQueued() && CHECK_CONDITIONS()) { // Dequeue task from the loading queue ContentLoadTask* tmp; if (LoadTasks.try_dequeue(tmp)) { if (tmp == task) { if (localQueue.Count() != 0) { // Put back queued tasks LoadTasks.enqueue_bulk(localQueue.Get(), localQueue.Count()); localQueue.Clear(); } PROFILE_CPU_NAMED("Inline"); ZoneColor(0xffaaaaaa); thread->Run(tmp); } else { localQueue.Add(tmp); } } else { // No task in queue but it's queued so other thread could have stolen it into own local queue break; } } if (localQueue.Count() != 0) { // Put back queued tasks LoadTasks.enqueue_bulk(localQueue.Get(), localQueue.Count()); localQueue.Clear(); } // Check if task is done if (task->IsEnded()) { // If was fine then wait for the next task if (task->IsFinished()) { task = task->GetContinueWithTask(); if (!task) break; } else { // Failed or cancelled so this wait also fails break; } } } while (CHECK_CONDITIONS()); #undef CHECK_CONDITIONS } else { // Wait for task end loadingTask->Wait(timeoutInMilliseconds); } } void Content::tryCallOnLoaded(Asset* asset) { ScopeLock lock(LoadedAssetsToInvokeLocker); const int32 index = LoadedAssetsToInvoke.Find(asset); if (index != -1) { LoadedAssetsToInvoke.RemoveAtKeepOrder(index); asset->onLoaded_MainThread(); } } void Content::onAssetLoaded(Asset* asset) { // This is called by the asset on loading end ScopeLock locker(LoadedAssetsToInvokeLocker); LoadedAssetsToInvoke.Add(asset); } void Content::onAssetUnload(Asset* asset) { // This is called by the asset on unloading ScopeLock locker(AssetsLocker); Assets.Remove(asset->GetID()); UnloadQueue.Remove(asset); LoadedAssetsToInvoke.Remove(asset); } void Content::onAssetChangeId(Asset* asset, const Guid& oldId, const Guid& newId) { ScopeLock locker(AssetsLocker); Assets.Remove(oldId); Assets.Add(newId, asset); } bool Content::IsAssetTypeIdInvalid(const ScriptingTypeHandle& type, const ScriptingTypeHandle& assetType) { // Skip if no restrictions for the type if (!type || !assetType) return false; #if BUILD_DEBUG || FLAX_TESTS // Peek types for debugging const auto& typeObj = type.GetType(); const auto& assetTypeObj = assetType.GetType(); #endif // Early out if type matches if (type == assetType) return false; // Check if the asset type inherited from the requested type ScriptingTypeHandle it = assetType.GetType().GetBaseType(); while (it) { if (type == it) return false; it = it.GetType().GetBaseType(); } return true; } Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) { if (!id.IsValid()) return nullptr; PROFILE_MEM(Content); // Check if asset has been already loaded Asset* result = nullptr; AssetsLocker.Lock(); Assets.TryGet(id, result); if (result) { AssetsLocker.Unlock(); // Validate type if (IsAssetTypeIdInvalid(type, result->GetTypeHandle()) && !result->Is(type)) { LOG(Warning, "Different loaded asset type! Asset: \'{0}\'. Expected type: {1}", result->ToString(), type.ToString()); LogContext::Print(LogType::Warning); return nullptr; } return result; } // Check if that asset is during loading if (LoadCallAssets.Contains(id)) { AssetsLocker.Unlock(); // Wait for loading end by other thread bool contains = true; while (contains) { Platform::Sleep(1); AssetsLocker.Lock(); contains = LoadCallAssets.Contains(id); AssetsLocker.Unlock(); } Assets.TryGet(id, result); return result; } // Mark asset as loading and release lock so other threads can load other assets LoadCallAssets.Add(id); AssetsLocker.Unlock(); #define LOAD_FAILED() AssetsLocker.Lock(); LoadCallAssets.Remove(id); AssetsLocker.Unlock(); return nullptr // Get cached asset info (from registry) AssetInfo assetInfo; if (!GetAssetInfo(id, assetInfo)) { LOG(Warning, "Invalid or missing asset ({0}, {1}).", id, type.ToString()); LogContext::Print(LogType::Warning); LOAD_FAILED(); } #if ASSETS_LOADING_EXTRA_VERIFICATION if (!FileSystem::FileExists(assetInfo.Path)) { LOG(Error, "Cannot find file '{0}'", assetInfo.Path); LOAD_FAILED(); } #endif // Find asset factory based in its type auto factory = GetAssetFactory(assetInfo); if (factory == nullptr) { LOG(Error, "Cannot find asset factory. Info: {0}", assetInfo.ToString()); LOAD_FAILED(); } // Create asset object PROFILE_MEM_BEGIN(ContentAssets); result = factory->New(assetInfo); PROFILE_MEM_END(); if (result == nullptr) { LOG(Error, "Cannot create asset object. Info: {0}", assetInfo.ToString()); LOAD_FAILED(); } ASSERT(result->GetID() == id); #if ASSETS_LOADING_EXTRA_VERIFICATION if (IsAssetTypeIdInvalid(type, result->GetTypeHandle()) && !result->Is(type)) { LOG(Warning, "Different loaded asset type! Asset: '{0}'. Expected type: {1}", assetInfo.ToString(), type.ToString()); result->DeleteObject(); LOAD_FAILED(); } #endif if (!result->IsInternalType()) result->RegisterObject(); // Register asset AssetsLocker.Lock(); #if ASSETS_LOADING_EXTRA_VERIFICATION ASSERT(!Assets.ContainsKey(id)); #endif Assets.Add(id, result); // Start asset loading result->startLoading(); // Remove from the loading queue and release lock LoadCallAssets.Remove(id); AssetsLocker.Unlock(); #undef LOAD_FAILED return result; } #if ENABLE_ASSETS_DISCOVERY bool findAsset(const Guid& id, const String& directory, Array& tmpCache, AssetInfo& info) { // Get all asset files tmpCache.Clear(); if (FileSystem::DirectoryGetFiles(tmpCache, directory)) { if (FileSystem::DirectoryExists(directory)) LOG(Error, "Cannot query files in folder '{0}'.", directory); return false; } // Start searching for asset with given ID bool result = false; LOG(Info, "Start searching asset with ID: {0} in '{1}'. {2} potential files to check...", id, directory, tmpCache.Count()); for (int32 i = 0; i < tmpCache.Count(); i++) { String& path = tmpCache[i]; // Check if not already in registry // Note: maybe we could disable this check? it would slow down searching but we will find more workspace problems if (!Cache.HasAsset(path)) { auto extension = FileSystem::GetExtension(path).ToLower(); // Check if it's a binary asset if (ContentStorageManager::IsFlaxStorageExtension(extension)) { // Skip packages in editor (results in conflicts with build game packages if deployed inside project folder) #if USE_EDITOR if (extension == PACKAGE_FILES_EXTENSION) continue; #endif // Open storage auto storage = ContentStorageManager::GetStorage(path); if (storage) { // Register assets Cache.RegisterAssets(storage); // Check if that is a missing asset if (storage->HasAsset(id)) { // Found result = Cache.FindAsset(id, info); LOG(Info, "Found {1} at '{0}'!", id, path); } } else { LOG(Error, "Cannot open file '{0}' error code: {1}", path, 0); } } // Check for json resource else if (JsonStorageProxy::IsValidExtension(extension)) { // Check Json storage layer Guid jsonId; String jsonTypeName; if (JsonStorageProxy::GetAssetInfo(path, jsonId, jsonTypeName)) { // Register asset Cache.RegisterAsset(jsonId, jsonTypeName, path); // Check if that is a missing asset if (id == jsonId) { // Found result = Cache.FindAsset(id, info); LOG(Info, "Found {1} at '{0}'!", id, path); } } } } } return result; } #endif