Refactor old ContentLoadingManager into Content for simplicity
This commit is contained in:
@@ -4,16 +4,12 @@
|
|||||||
#include "Content.h"
|
#include "Content.h"
|
||||||
#include "SoftAssetReference.h"
|
#include "SoftAssetReference.h"
|
||||||
#include "Cache/AssetsCache.h"
|
#include "Cache/AssetsCache.h"
|
||||||
#include "Loading/ContentLoadingManager.h"
|
|
||||||
#include "Loading/Tasks/LoadAssetTask.h"
|
#include "Loading/Tasks/LoadAssetTask.h"
|
||||||
#include "Engine/Core/Log.h"
|
#include "Engine/Core/Log.h"
|
||||||
#include "Engine/Core/LogContext.h"
|
#include "Engine/Core/LogContext.h"
|
||||||
#include "Engine/Engine/Engine.h"
|
|
||||||
#include "Engine/Threading/Threading.h"
|
|
||||||
#include "Engine/Profiler/ProfilerCPU.h"
|
#include "Engine/Profiler/ProfilerCPU.h"
|
||||||
#include "Engine/Threading/MainThreadTask.h"
|
|
||||||
#include "Engine/Threading/ConcurrentTaskQueue.h"
|
|
||||||
#include "Engine/Scripting/ManagedCLR/MCore.h"
|
#include "Engine/Scripting/ManagedCLR/MCore.h"
|
||||||
|
#include "Engine/Threading/MainThreadTask.h"
|
||||||
|
|
||||||
AssetReferenceBase::~AssetReferenceBase()
|
AssetReferenceBase::~AssetReferenceBase()
|
||||||
{
|
{
|
||||||
@@ -377,11 +373,6 @@ void Asset::Reload()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace ContentLoadingManagerImpl
|
|
||||||
{
|
|
||||||
extern ConcurrentTaskQueue<ContentLoadTask> Tasks;
|
|
||||||
};
|
|
||||||
|
|
||||||
bool Asset::WaitForLoaded(double timeoutInMilliseconds) const
|
bool Asset::WaitForLoaded(double timeoutInMilliseconds) const
|
||||||
{
|
{
|
||||||
// This function is used many time when some parts of the engine need to wait for asset loading end (it may fail but has to end).
|
// This function is used many time when some parts of the engine need to wait for asset loading end (it may fail but has to end).
|
||||||
@@ -430,80 +421,7 @@ bool Asset::WaitForLoaded(double timeoutInMilliseconds) const
|
|||||||
|
|
||||||
PROFILE_CPU();
|
PROFILE_CPU();
|
||||||
|
|
||||||
// Check if call is made from the Loading Thread and task has not been taken yet
|
Content::WaitForTask(loadingTask, timeoutInMilliseconds);
|
||||||
auto thread = ContentLoadingManager::GetCurrentLoadThread();
|
|
||||||
if (thread != nullptr)
|
|
||||||
{
|
|
||||||
// Note: to reproduce this case just include material into material (use layering).
|
|
||||||
// So during loading first material it will wait for child materials loaded calling this function
|
|
||||||
|
|
||||||
const double timeoutInSeconds = timeoutInMilliseconds * 0.001;
|
|
||||||
const double startTime = Platform::GetTimeSeconds();
|
|
||||||
Task* task = loadingTask;
|
|
||||||
Array<ContentLoadTask*, InlinedAllocation<64>> localQueue;
|
|
||||||
#define CHECK_CONDITIONS() (!Engine::ShouldExit() && (timeoutInSeconds <= 0.0 || Platform::GetTimeSeconds() - startTime < timeoutInSeconds))
|
|
||||||
do
|
|
||||||
{
|
|
||||||
// Try to execute content tasks
|
|
||||||
while (task->IsQueued() && CHECK_CONDITIONS())
|
|
||||||
{
|
|
||||||
// Dequeue task from the loading queue
|
|
||||||
ContentLoadTask* tmp;
|
|
||||||
if (ContentLoadingManagerImpl::Tasks.try_dequeue(tmp))
|
|
||||||
{
|
|
||||||
if (tmp == task)
|
|
||||||
{
|
|
||||||
if (localQueue.Count() != 0)
|
|
||||||
{
|
|
||||||
// Put back queued tasks
|
|
||||||
ContentLoadingManagerImpl::Tasks.enqueue_bulk(localQueue.Get(), localQueue.Count());
|
|
||||||
localQueue.Clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
thread->Run(tmp);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
localQueue.Add(tmp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// No task in queue but it's queued so other thread could have stolen it into own local queue
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (localQueue.Count() != 0)
|
|
||||||
{
|
|
||||||
// Put back queued tasks
|
|
||||||
ContentLoadingManagerImpl::Tasks.enqueue_bulk(localQueue.Get(), localQueue.Count());
|
|
||||||
localQueue.Clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if task is done
|
|
||||||
if (task->IsEnded())
|
|
||||||
{
|
|
||||||
// If was fine then wait for the next task
|
|
||||||
if (task->IsFinished())
|
|
||||||
{
|
|
||||||
task = task->GetContinueWithTask();
|
|
||||||
if (!task)
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Failed or cancelled so this wait also fails
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} while (CHECK_CONDITIONS());
|
|
||||||
#undef CHECK_CONDITIONS
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Wait for task end
|
|
||||||
loadingTask->Wait(timeoutInMilliseconds);
|
|
||||||
}
|
|
||||||
|
|
||||||
// If running on a main thread we can flush asset `Loaded` event
|
// If running on a main thread we can flush asset `Loaded` event
|
||||||
if (IsInMainThread() && IsLoaded())
|
if (IsInMainThread() && IsLoaded())
|
||||||
|
|||||||
@@ -974,6 +974,7 @@ Asset::LoadResult Model::load()
|
|||||||
auto chunk15 = GetChunk(15);
|
auto chunk15 = GetChunk(15);
|
||||||
if (chunk15 && chunk15->IsLoaded() && EnableModelSDF == 1)
|
if (chunk15 && chunk15->IsLoaded() && EnableModelSDF == 1)
|
||||||
{
|
{
|
||||||
|
PROFILE_CPU_NAMED("SDF");
|
||||||
MemoryReadStream sdfStream(chunk15->Get(), chunk15->Size());
|
MemoryReadStream sdfStream(chunk15->Get(), chunk15->Size());
|
||||||
int32 version;
|
int32 version;
|
||||||
sdfStream.ReadInt32(&version);
|
sdfStream.ReadInt32(&version);
|
||||||
|
|||||||
@@ -7,13 +7,13 @@
|
|||||||
// Amount of content loading threads per single physical CPU core
|
// Amount of content loading threads per single physical CPU core
|
||||||
#define LOADING_THREAD_PER_LOGICAL_CORE 0.5f
|
#define LOADING_THREAD_PER_LOGICAL_CORE 0.5f
|
||||||
|
|
||||||
// Enable/disable additional assets metadata verification, note: we should disable it for release builds
|
// Enables additional assets metadata verification
|
||||||
#define ASSETS_LOADING_EXTRA_VERIFICATION (BUILD_DEBUG || USE_EDITOR)
|
#define ASSETS_LOADING_EXTRA_VERIFICATION (BUILD_DEBUG || USE_EDITOR)
|
||||||
|
|
||||||
// Maximum amount of data chunks used by the single asset
|
// Maximum amount of data chunks used by the single asset
|
||||||
#define ASSET_FILE_DATA_CHUNKS 16
|
#define ASSET_FILE_DATA_CHUNKS 16
|
||||||
|
|
||||||
// Enables searching workspace for missing assets (should be disabled in the final builds where assets registry is solid)
|
// Enables searching workspace for missing assets
|
||||||
#define ENABLE_ASSETS_DISCOVERY (USE_EDITOR)
|
#define ENABLE_ASSETS_DISCOVERY (USE_EDITOR)
|
||||||
|
|
||||||
// Default extension for all asset files
|
// Default extension for all asset files
|
||||||
|
|||||||
@@ -3,20 +3,27 @@
|
|||||||
#include "Content.h"
|
#include "Content.h"
|
||||||
#include "JsonAsset.h"
|
#include "JsonAsset.h"
|
||||||
#include "SceneReference.h"
|
#include "SceneReference.h"
|
||||||
#include "Engine/Serialization/Serialization.h"
|
|
||||||
#include "Cache/AssetsCache.h"
|
#include "Cache/AssetsCache.h"
|
||||||
#include "Storage/ContentStorageManager.h"
|
#include "Storage/ContentStorageManager.h"
|
||||||
#include "Storage/JsonStorageProxy.h"
|
#include "Storage/JsonStorageProxy.h"
|
||||||
#include "Factories/IAssetFactory.h"
|
#include "Factories/IAssetFactory.h"
|
||||||
|
#include "Loading/LoadingThread.h"
|
||||||
|
#include "Loading/ContentLoadTask.h"
|
||||||
#include "Engine/Core/Log.h"
|
#include "Engine/Core/Log.h"
|
||||||
#include "Engine/Core/LogContext.h"
|
#include "Engine/Core/LogContext.h"
|
||||||
#include "Engine/Core/Types/String.h"
|
#include "Engine/Core/Types/String.h"
|
||||||
#include "Engine/Core/ObjectsRemovalService.h"
|
#include "Engine/Core/ObjectsRemovalService.h"
|
||||||
#include "Engine/Engine/EngineService.h"
|
#include "Engine/Serialization/Serialization.h"
|
||||||
#include "Engine/Platform/FileSystem.h"
|
#include "Engine/Platform/FileSystem.h"
|
||||||
|
#include "Engine/Platform/ConditionVariable.h"
|
||||||
|
#include "Engine/Platform/Thread.h"
|
||||||
|
#include "Engine/Platform/CPUInfo.h"
|
||||||
#include "Engine/Threading/Threading.h"
|
#include "Engine/Threading/Threading.h"
|
||||||
#include "Engine/Threading/MainThreadTask.h"
|
#include "Engine/Threading/MainThreadTask.h"
|
||||||
|
#include "Engine/Threading/ConcurrentTaskQueue.h"
|
||||||
#include "Engine/Graphics/Graphics.h"
|
#include "Engine/Graphics/Graphics.h"
|
||||||
|
#include "Engine/Engine/Engine.h"
|
||||||
|
#include "Engine/Engine/EngineService.h"
|
||||||
#include "Engine/Engine/Time.h"
|
#include "Engine/Engine/Time.h"
|
||||||
#include "Engine/Engine/Globals.h"
|
#include "Engine/Engine/Globals.h"
|
||||||
#include "Engine/Level/Types.h"
|
#include "Engine/Level/Types.h"
|
||||||
@@ -30,6 +37,10 @@
|
|||||||
#if ENABLE_ASSETS_DISCOVERY
|
#if ENABLE_ASSETS_DISCOVERY
|
||||||
#include "Engine/Core/Collections/HashSet.h"
|
#include "Engine/Core/Collections/HashSet.h"
|
||||||
#endif
|
#endif
|
||||||
|
#if USE_EDITOR && PLATFORM_WINDOWS
|
||||||
|
#include "Engine/Platform/Win32/IncludeWindowsHeaders.h"
|
||||||
|
#include <propidlbase.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
TimeSpan Content::AssetsUpdateInterval = TimeSpan::FromMilliseconds(500);
|
TimeSpan Content::AssetsUpdateInterval = TimeSpan::FromMilliseconds(500);
|
||||||
TimeSpan Content::AssetsUnloadInterval = TimeSpan::FromSeconds(10);
|
TimeSpan Content::AssetsUnloadInterval = TimeSpan::FromSeconds(10);
|
||||||
@@ -64,6 +75,14 @@ namespace
|
|||||||
// Assets Registry Stuff
|
// Assets Registry Stuff
|
||||||
AssetsCache Cache;
|
AssetsCache Cache;
|
||||||
|
|
||||||
|
// Loading assets
|
||||||
|
THREADLOCAL LoadingThread* ThisLoadThread = nullptr;
|
||||||
|
LoadingThread* MainLoadThread = nullptr;
|
||||||
|
Array<LoadingThread*> LoadThreads;
|
||||||
|
ConcurrentTaskQueue<ContentLoadTask> LoadTasks;
|
||||||
|
ConditionVariable LoadTasksSignal;
|
||||||
|
CriticalSection LoadTasksMutex;
|
||||||
|
|
||||||
// Unloading assets
|
// Unloading assets
|
||||||
Dictionary<Asset*, TimeSpan> UnloadQueue;
|
Dictionary<Asset*, TimeSpan> UnloadQueue;
|
||||||
TimeSpan LastUnloadCheckTime(0);
|
TimeSpan LastUnloadCheckTime(0);
|
||||||
@@ -90,6 +109,7 @@ public:
|
|||||||
bool Init() override;
|
bool Init() override;
|
||||||
void Update() override;
|
void Update() override;
|
||||||
void LateUpdate() override;
|
void LateUpdate() override;
|
||||||
|
void BeforeExit() override;
|
||||||
void Dispose() override;
|
void Dispose() override;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -100,6 +120,25 @@ bool ContentService::Init()
|
|||||||
// Load assets registry
|
// Load assets registry
|
||||||
Cache.Init();
|
Cache.Init();
|
||||||
|
|
||||||
|
// Create loading threads
|
||||||
|
const CPUInfo cpuInfo = Platform::GetCPUInfo();
|
||||||
|
const int32 count = Math::Clamp(Math::CeilToInt(LOADING_THREAD_PER_LOGICAL_CORE * (float)cpuInfo.LogicalProcessorCount), 1, 12);
|
||||||
|
LOG(Info, "Creating {0} content loading threads...", count);
|
||||||
|
MainLoadThread = New<LoadingThread>();
|
||||||
|
ThisLoadThread = MainLoadThread;
|
||||||
|
LoadThreads.EnsureCapacity(count);
|
||||||
|
for (int32 i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
auto thread = New<LoadingThread>();
|
||||||
|
if (thread->Start(String::Format(TEXT("Load Thread {0}"), i)))
|
||||||
|
{
|
||||||
|
LOG(Fatal, "Cannot spawn content thread {0}/{1}", i, count);
|
||||||
|
Delete(thread);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
LoadThreads.Add(thread);
|
||||||
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -173,6 +212,14 @@ void ContentService::LateUpdate()
|
|||||||
Cache.Save();
|
Cache.Save();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ContentService::BeforeExit()
|
||||||
|
{
|
||||||
|
// Signal threads to end work soon
|
||||||
|
for (auto thread : LoadThreads)
|
||||||
|
thread->NotifyExit();
|
||||||
|
LoadTasksSignal.NotifyAll();
|
||||||
|
}
|
||||||
|
|
||||||
void ContentService::Dispose()
|
void ContentService::Dispose()
|
||||||
{
|
{
|
||||||
IsExiting = true;
|
IsExiting = true;
|
||||||
@@ -198,6 +245,20 @@ void ContentService::Dispose()
|
|||||||
|
|
||||||
// NOW dispose graphics device - where there is no loaded assets at all
|
// NOW dispose graphics device - where there is no loaded assets at all
|
||||||
Graphics::DisposeDevice();
|
Graphics::DisposeDevice();
|
||||||
|
|
||||||
|
// Exit all load threads
|
||||||
|
for (auto thread : LoadThreads)
|
||||||
|
thread->NotifyExit();
|
||||||
|
LoadTasksSignal.NotifyAll();
|
||||||
|
for (auto thread : LoadThreads)
|
||||||
|
thread->Join();
|
||||||
|
LoadThreads.ClearDelete();
|
||||||
|
Delete(MainLoadThread);
|
||||||
|
MainLoadThread = nullptr;
|
||||||
|
ThisLoadThread = nullptr;
|
||||||
|
|
||||||
|
// Cancel all remaining tasks (no chance to execute them)
|
||||||
|
LoadTasks.CancelAll();
|
||||||
}
|
}
|
||||||
|
|
||||||
IAssetFactory::Collection& IAssetFactory::Get()
|
IAssetFactory::Collection& IAssetFactory::Get()
|
||||||
@@ -206,6 +267,122 @@ IAssetFactory::Collection& IAssetFactory::Get()
|
|||||||
return Factories;
|
return Factories;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
LoadingThread::LoadingThread()
|
||||||
|
: _exitFlag(false)
|
||||||
|
, _thread(nullptr)
|
||||||
|
, _totalTasksDoneCount(0)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
LoadingThread::~LoadingThread()
|
||||||
|
{
|
||||||
|
if (_thread != nullptr)
|
||||||
|
{
|
||||||
|
_thread->Kill(true);
|
||||||
|
Delete(_thread);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void LoadingThread::NotifyExit()
|
||||||
|
{
|
||||||
|
Platform::InterlockedIncrement(&_exitFlag);
|
||||||
|
}
|
||||||
|
|
||||||
|
void LoadingThread::Join()
|
||||||
|
{
|
||||||
|
auto thread = _thread;
|
||||||
|
if (thread)
|
||||||
|
thread->Join();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool LoadingThread::Start(const String& name)
|
||||||
|
{
|
||||||
|
ASSERT(_thread == nullptr && name.HasChars());
|
||||||
|
|
||||||
|
// Create new thread
|
||||||
|
auto thread = Thread::Create(this, name, ThreadPriority::Normal);
|
||||||
|
if (thread == nullptr)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
_thread = thread;
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void LoadingThread::Run(ContentLoadTask* job)
|
||||||
|
{
|
||||||
|
ASSERT(job);
|
||||||
|
|
||||||
|
job->Execute();
|
||||||
|
_totalTasksDoneCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
String LoadingThread::ToString() const
|
||||||
|
{
|
||||||
|
return String::Format(TEXT("Loading Thread {0}"), _thread ? _thread->GetID() : 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
int32 LoadingThread::Run()
|
||||||
|
{
|
||||||
|
#if USE_EDITOR && PLATFORM_WINDOWS
|
||||||
|
// Initialize COM
|
||||||
|
// TODO: maybe add sth to Thread::Create to indicate that thread will use COM stuff
|
||||||
|
const auto result = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
|
||||||
|
if (FAILED(result))
|
||||||
|
{
|
||||||
|
LOG(Error, "Failed to init COM for WIC texture importing! Result: {0:x}", static_cast<uint32>(result));
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
ContentLoadTask* task;
|
||||||
|
ThisLoadThread = this;
|
||||||
|
|
||||||
|
while (Platform::AtomicRead(&_exitFlag) == 0)
|
||||||
|
{
|
||||||
|
if (LoadTasks.try_dequeue(task))
|
||||||
|
{
|
||||||
|
Run(task);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
LoadTasksMutex.Lock();
|
||||||
|
LoadTasksSignal.Wait(LoadTasksMutex);
|
||||||
|
LoadTasksMutex.Unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ThisLoadThread = nullptr;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void LoadingThread::Exit()
|
||||||
|
{
|
||||||
|
LOG(Info, "Content thread '{0}' exited. Load calls: {1}", _thread->GetName(), _totalTasksDoneCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
String ContentLoadTask::ToString() const
|
||||||
|
{
|
||||||
|
return String::Format(TEXT("Content Load Task ({})"), (int32)GetState());
|
||||||
|
}
|
||||||
|
|
||||||
|
void ContentLoadTask::Enqueue()
|
||||||
|
{
|
||||||
|
LoadTasks.Add(this);
|
||||||
|
LoadTasksSignal.NotifyOne();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ContentLoadTask::Run()
|
||||||
|
{
|
||||||
|
const auto result = run();
|
||||||
|
const bool failed = result != Result::Ok;
|
||||||
|
if (failed)
|
||||||
|
{
|
||||||
|
LOG(Warning, "\'{0}\' failed with result: {1}", ToString(), ToString(result));
|
||||||
|
}
|
||||||
|
return failed;
|
||||||
|
}
|
||||||
|
|
||||||
AssetsCache* Content::GetRegistry()
|
AssetsCache* Content::GetRegistry()
|
||||||
{
|
{
|
||||||
return &Cache;
|
return &Cache;
|
||||||
@@ -887,6 +1064,84 @@ Asset* Content::CreateVirtualAsset(const ScriptingTypeHandle& type)
|
|||||||
return asset;
|
return asset;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Content::WaitForTask(ContentLoadTask* loadingTask, double timeoutInMilliseconds)
|
||||||
|
{
|
||||||
|
// Check if call is made from the Loading Thread and task has not been taken yet
|
||||||
|
auto thread = ThisLoadThread;
|
||||||
|
if (thread != nullptr)
|
||||||
|
{
|
||||||
|
// Note: to reproduce this case just include material into material (use layering).
|
||||||
|
// So during loading first material it will wait for child materials loaded calling this function
|
||||||
|
|
||||||
|
const double timeoutInSeconds = timeoutInMilliseconds * 0.001;
|
||||||
|
const double startTime = Platform::GetTimeSeconds();
|
||||||
|
Task* task = loadingTask;
|
||||||
|
Array<ContentLoadTask*, InlinedAllocation<64>> localQueue;
|
||||||
|
#define CHECK_CONDITIONS() (!Engine::ShouldExit() && (timeoutInSeconds <= 0.0 || Platform::GetTimeSeconds() - startTime < timeoutInSeconds))
|
||||||
|
do
|
||||||
|
{
|
||||||
|
// Try to execute content tasks
|
||||||
|
while (task->IsQueued() && CHECK_CONDITIONS())
|
||||||
|
{
|
||||||
|
// Dequeue task from the loading queue
|
||||||
|
ContentLoadTask* tmp;
|
||||||
|
if (LoadTasks.try_dequeue(tmp))
|
||||||
|
{
|
||||||
|
if (tmp == task)
|
||||||
|
{
|
||||||
|
if (localQueue.Count() != 0)
|
||||||
|
{
|
||||||
|
// Put back queued tasks
|
||||||
|
LoadTasks.enqueue_bulk(localQueue.Get(), localQueue.Count());
|
||||||
|
localQueue.Clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
thread->Run(tmp);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
localQueue.Add(tmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// No task in queue but it's queued so other thread could have stolen it into own local queue
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (localQueue.Count() != 0)
|
||||||
|
{
|
||||||
|
// Put back queued tasks
|
||||||
|
LoadTasks.enqueue_bulk(localQueue.Get(), localQueue.Count());
|
||||||
|
localQueue.Clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if task is done
|
||||||
|
if (task->IsEnded())
|
||||||
|
{
|
||||||
|
// If was fine then wait for the next task
|
||||||
|
if (task->IsFinished())
|
||||||
|
{
|
||||||
|
task = task->GetContinueWithTask();
|
||||||
|
if (!task)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Failed or cancelled so this wait also fails
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} while (CHECK_CONDITIONS());
|
||||||
|
#undef CHECK_CONDITIONS
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Wait for task end
|
||||||
|
loadingTask->Wait(timeoutInMilliseconds);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void Content::tryCallOnLoaded(Asset* asset)
|
void Content::tryCallOnLoaded(Asset* asset)
|
||||||
{
|
{
|
||||||
ScopeLock lock(LoadedAssetsToInvokeLocker);
|
ScopeLock lock(LoadedAssetsToInvokeLocker);
|
||||||
|
|||||||
@@ -362,11 +362,10 @@ public:
|
|||||||
API_EVENT() static Delegate<Asset*> AssetReloading;
|
API_EVENT() static Delegate<Asset*> AssetReloading;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
static void WaitForTask(ContentLoadTask* loadingTask, double timeoutInMilliseconds);
|
||||||
static void tryCallOnLoaded(Asset* asset);
|
static void tryCallOnLoaded(Asset* asset);
|
||||||
static void onAssetLoaded(Asset* asset);
|
static void onAssetLoaded(Asset* asset);
|
||||||
static void onAssetUnload(Asset* asset);
|
static void onAssetUnload(Asset* asset);
|
||||||
static void onAssetChangeId(Asset* asset, const Guid& oldId, const Guid& newId);
|
static void onAssetChangeId(Asset* asset, const Guid& oldId, const Guid& newId);
|
||||||
|
|
||||||
private:
|
|
||||||
static void deleteFileSafety(const StringView& path, const Guid& id);
|
static void deleteFileSafety(const StringView& path, const Guid& id);
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,237 +0,0 @@
|
|||||||
// Copyright (c) 2012-2024 Wojciech Figat. All rights reserved.
|
|
||||||
|
|
||||||
#include "ContentLoadingManager.h"
|
|
||||||
#include "ContentLoadTask.h"
|
|
||||||
#include "Engine/Core/Log.h"
|
|
||||||
#include "Engine/Core/Math/Math.h"
|
|
||||||
#include "Engine/Core/Collections/Array.h"
|
|
||||||
#include "Engine/Platform/CPUInfo.h"
|
|
||||||
#include "Engine/Platform/Thread.h"
|
|
||||||
#include "Engine/Platform/ConditionVariable.h"
|
|
||||||
#include "Engine/Content/Config.h"
|
|
||||||
#include "Engine/Engine/EngineService.h"
|
|
||||||
#include "Engine/Threading/Threading.h"
|
|
||||||
#include "Engine/Threading/ConcurrentTaskQueue.h"
|
|
||||||
#if USE_EDITOR && PLATFORM_WINDOWS
|
|
||||||
#include "Engine/Platform/Win32/IncludeWindowsHeaders.h"
|
|
||||||
#include <propidlbase.h>
|
|
||||||
#endif
|
|
||||||
|
|
||||||
namespace ContentLoadingManagerImpl
|
|
||||||
{
|
|
||||||
THREADLOCAL LoadingThread* ThisThread = nullptr;
|
|
||||||
LoadingThread* MainThread = nullptr;
|
|
||||||
Array<LoadingThread*> Threads;
|
|
||||||
ConcurrentTaskQueue<ContentLoadTask> Tasks;
|
|
||||||
ConditionVariable TasksSignal;
|
|
||||||
CriticalSection TasksMutex;
|
|
||||||
};
|
|
||||||
|
|
||||||
using namespace ContentLoadingManagerImpl;
|
|
||||||
|
|
||||||
class ContentLoadingManagerService : public EngineService
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
ContentLoadingManagerService()
|
|
||||||
: EngineService(TEXT("Content Loading Manager"), -500)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
bool Init() override;
|
|
||||||
void BeforeExit() override;
|
|
||||||
void Dispose() override;
|
|
||||||
};
|
|
||||||
|
|
||||||
ContentLoadingManagerService ContentLoadingManagerServiceInstance;
|
|
||||||
|
|
||||||
LoadingThread::LoadingThread()
|
|
||||||
: _exitFlag(false)
|
|
||||||
, _thread(nullptr)
|
|
||||||
, _totalTasksDoneCount(0)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
LoadingThread::~LoadingThread()
|
|
||||||
{
|
|
||||||
// Check if has thread attached
|
|
||||||
if (_thread != nullptr)
|
|
||||||
{
|
|
||||||
_thread->Kill(true);
|
|
||||||
Delete(_thread);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64 LoadingThread::GetID() const
|
|
||||||
{
|
|
||||||
return _thread ? _thread->GetID() : 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
void LoadingThread::NotifyExit()
|
|
||||||
{
|
|
||||||
Platform::InterlockedIncrement(&_exitFlag);
|
|
||||||
}
|
|
||||||
|
|
||||||
void LoadingThread::Join()
|
|
||||||
{
|
|
||||||
auto thread = _thread;
|
|
||||||
if (thread)
|
|
||||||
thread->Join();
|
|
||||||
}
|
|
||||||
|
|
||||||
bool LoadingThread::Start(const String& name)
|
|
||||||
{
|
|
||||||
ASSERT(_thread == nullptr && name.HasChars());
|
|
||||||
|
|
||||||
// Create new thread
|
|
||||||
auto thread = Thread::Create(this, name, ThreadPriority::Normal);
|
|
||||||
if (thread == nullptr)
|
|
||||||
return true;
|
|
||||||
|
|
||||||
_thread = thread;
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
void LoadingThread::Run(ContentLoadTask* job)
|
|
||||||
{
|
|
||||||
ASSERT(job);
|
|
||||||
|
|
||||||
job->Execute();
|
|
||||||
_totalTasksDoneCount++;
|
|
||||||
}
|
|
||||||
|
|
||||||
String LoadingThread::ToString() const
|
|
||||||
{
|
|
||||||
return String::Format(TEXT("Loading Thread {0}"), GetID());
|
|
||||||
}
|
|
||||||
|
|
||||||
int32 LoadingThread::Run()
|
|
||||||
{
|
|
||||||
#if USE_EDITOR && PLATFORM_WINDOWS
|
|
||||||
|
|
||||||
// Initialize COM
|
|
||||||
// TODO: maybe add sth to Thread::Create to indicate that thread will use COM stuff
|
|
||||||
const auto result = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
|
|
||||||
if (FAILED(result))
|
|
||||||
{
|
|
||||||
LOG(Error, "Failed to init COM for WIC texture importing! Result: {0:x}", static_cast<uint32>(result));
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
ContentLoadTask* task;
|
|
||||||
ThisThread = this;
|
|
||||||
|
|
||||||
while (HasExitFlagClear())
|
|
||||||
{
|
|
||||||
if (Tasks.try_dequeue(task))
|
|
||||||
{
|
|
||||||
Run(task);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
TasksMutex.Lock();
|
|
||||||
TasksSignal.Wait(TasksMutex);
|
|
||||||
TasksMutex.Unlock();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ThisThread = nullptr;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
void LoadingThread::Exit()
|
|
||||||
{
|
|
||||||
// Send info
|
|
||||||
ASSERT_LOW_LAYER(_thread);
|
|
||||||
LOG(Info, "Content thread '{0}' exited. Load calls: {1}", _thread->GetName(), _totalTasksDoneCount);
|
|
||||||
}
|
|
||||||
|
|
||||||
LoadingThread* ContentLoadingManager::GetCurrentLoadThread()
|
|
||||||
{
|
|
||||||
return ThisThread;
|
|
||||||
}
|
|
||||||
|
|
||||||
int32 ContentLoadingManager::GetTasksCount()
|
|
||||||
{
|
|
||||||
return Tasks.Count();
|
|
||||||
}
|
|
||||||
|
|
||||||
bool ContentLoadingManagerService::Init()
|
|
||||||
{
|
|
||||||
ASSERT(ContentLoadingManagerImpl::Threads.IsEmpty() && IsInMainThread());
|
|
||||||
|
|
||||||
// Calculate amount of loading threads to use
|
|
||||||
const CPUInfo cpuInfo = Platform::GetCPUInfo();
|
|
||||||
const int32 count = Math::Clamp(Math::CeilToInt(LOADING_THREAD_PER_LOGICAL_CORE * (float)cpuInfo.LogicalProcessorCount), 1, 12);
|
|
||||||
LOG(Info, "Creating {0} content loading threads...", count);
|
|
||||||
|
|
||||||
// Create loading threads
|
|
||||||
MainThread = New<LoadingThread>();
|
|
||||||
ThisThread = MainThread;
|
|
||||||
Threads.EnsureCapacity(count);
|
|
||||||
for (int32 i = 0; i < count; i++)
|
|
||||||
{
|
|
||||||
auto thread = New<LoadingThread>();
|
|
||||||
if (thread->Start(String::Format(TEXT("Load Thread {0}"), i)))
|
|
||||||
{
|
|
||||||
LOG(Fatal, "Cannot spawn content thread {0}/{1}", i, count);
|
|
||||||
Delete(thread);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
Threads.Add(thread);
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
void ContentLoadingManagerService::BeforeExit()
|
|
||||||
{
|
|
||||||
// Signal threads to end work soon
|
|
||||||
for (int32 i = 0; i < Threads.Count(); i++)
|
|
||||||
Threads[i]->NotifyExit();
|
|
||||||
TasksSignal.NotifyAll();
|
|
||||||
}
|
|
||||||
|
|
||||||
void ContentLoadingManagerService::Dispose()
|
|
||||||
{
|
|
||||||
// Exit all threads
|
|
||||||
for (int32 i = 0; i < Threads.Count(); i++)
|
|
||||||
Threads[i]->NotifyExit();
|
|
||||||
TasksSignal.NotifyAll();
|
|
||||||
for (int32 i = 0; i < Threads.Count(); i++)
|
|
||||||
Threads[i]->Join();
|
|
||||||
Threads.ClearDelete();
|
|
||||||
Delete(MainThread);
|
|
||||||
MainThread = nullptr;
|
|
||||||
ThisThread = nullptr;
|
|
||||||
|
|
||||||
// Cancel all remaining tasks (no chance to execute them)
|
|
||||||
Tasks.CancelAll();
|
|
||||||
}
|
|
||||||
|
|
||||||
String ContentLoadTask::ToString() const
|
|
||||||
{
|
|
||||||
return String::Format(TEXT("Content Load Task ({})"), (int32)GetState());
|
|
||||||
}
|
|
||||||
|
|
||||||
void ContentLoadTask::Enqueue()
|
|
||||||
{
|
|
||||||
Tasks.Add(this);
|
|
||||||
TasksSignal.NotifyOne();
|
|
||||||
}
|
|
||||||
|
|
||||||
bool ContentLoadTask::Run()
|
|
||||||
{
|
|
||||||
// Perform an operation
|
|
||||||
const auto result = run();
|
|
||||||
|
|
||||||
// Process result
|
|
||||||
const bool failed = result != Result::Ok;
|
|
||||||
if (failed)
|
|
||||||
{
|
|
||||||
LOG(Warning, "\'{0}\' failed with result: {1}", ToString(), ToString(result));
|
|
||||||
}
|
|
||||||
return failed;
|
|
||||||
}
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
// Copyright (c) 2012-2024 Wojciech Figat. All rights reserved.
|
|
||||||
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include "Engine/Threading/IRunnable.h"
|
|
||||||
|
|
||||||
class Asset;
|
|
||||||
class LoadingThread;
|
|
||||||
class ContentLoadTask;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Resources loading thread
|
|
||||||
/// </summary>
|
|
||||||
class LoadingThread : public IRunnable
|
|
||||||
{
|
|
||||||
protected:
|
|
||||||
volatile int64 _exitFlag;
|
|
||||||
Thread* _thread;
|
|
||||||
int32 _totalTasksDoneCount;
|
|
||||||
|
|
||||||
public:
|
|
||||||
/// <summary>
|
|
||||||
/// Init
|
|
||||||
/// </summary>
|
|
||||||
LoadingThread();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Destructor
|
|
||||||
/// </summary>
|
|
||||||
~LoadingThread();
|
|
||||||
|
|
||||||
public:
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the thread identifier.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>Thread ID</returns>
|
|
||||||
uint64 GetID() const;
|
|
||||||
|
|
||||||
public:
|
|
||||||
/// <summary>
|
|
||||||
/// Returns true if thread has empty exit flag, so it can continue it's work
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>True if exit flag is empty, otherwise false</returns>
|
|
||||||
FORCE_INLINE bool HasExitFlagClear()
|
|
||||||
{
|
|
||||||
return Platform::AtomicRead(&_exitFlag) == 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Set exit flag to true so thread must exit
|
|
||||||
/// </summary>
|
|
||||||
void NotifyExit();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Stops the calling thread execution until the loading thread ends its execution.
|
|
||||||
/// </summary>
|
|
||||||
void Join();
|
|
||||||
|
|
||||||
public:
|
|
||||||
/// <summary>
|
|
||||||
/// Starts thread execution.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="name">The thread name.</param>
|
|
||||||
/// <returns>True if cannot start, otherwise false</returns>
|
|
||||||
bool Start(const String& name);
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Runs the specified task.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="task">The task.</param>
|
|
||||||
void Run(ContentLoadTask* task);
|
|
||||||
|
|
||||||
public:
|
|
||||||
// [IRunnable]
|
|
||||||
String ToString() const override;
|
|
||||||
int32 Run() override;
|
|
||||||
void Exit() override;
|
|
||||||
};
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Content loading manager.
|
|
||||||
/// </summary>
|
|
||||||
class ContentLoadingManager
|
|
||||||
{
|
|
||||||
friend ContentLoadTask;
|
|
||||||
friend LoadingThread;
|
|
||||||
friend Asset;
|
|
||||||
|
|
||||||
public:
|
|
||||||
/// <summary>
|
|
||||||
/// Checks if current execution context is thread used to load assets.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>True if execution is in Load Thread, otherwise false.</returns>
|
|
||||||
FORCE_INLINE static bool IsInLoadThread()
|
|
||||||
{
|
|
||||||
return GetCurrentLoadThread() != nullptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets content loading thread handle if current thread is one of them.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>Current load thread or null if current thread is different.</returns>
|
|
||||||
static LoadingThread* GetCurrentLoadThread();
|
|
||||||
|
|
||||||
public:
|
|
||||||
/// <summary>
|
|
||||||
/// Gets amount of enqueued tasks to perform.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>The tasks count.</returns>
|
|
||||||
static int32 GetTasksCount();
|
|
||||||
};
|
|
||||||
50
Source/Engine/Content/Loading/LoadingThread.h
Normal file
50
Source/Engine/Content/Loading/LoadingThread.h
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
// Copyright (c) 2012-2024 Wojciech Figat. All rights reserved.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "Engine/Threading/IRunnable.h"
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Resources loading thread.
|
||||||
|
/// </summary>
|
||||||
|
class LoadingThread : public IRunnable
|
||||||
|
{
|
||||||
|
protected:
|
||||||
|
volatile int64 _exitFlag;
|
||||||
|
Thread* _thread;
|
||||||
|
int32 _totalTasksDoneCount;
|
||||||
|
|
||||||
|
public:
|
||||||
|
LoadingThread();
|
||||||
|
~LoadingThread();
|
||||||
|
|
||||||
|
public:
|
||||||
|
/// <summary>
|
||||||
|
/// Set exit flag to true so thread must exit
|
||||||
|
/// </summary>
|
||||||
|
void NotifyExit();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Stops the calling thread execution until the loading thread ends its execution.
|
||||||
|
/// </summary>
|
||||||
|
void Join();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Starts thread execution.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="name">The thread name.</param>
|
||||||
|
/// <returns>True if cannot start, otherwise false</returns>
|
||||||
|
bool Start(const String& name);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Runs the specified task.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="task">The task.</param>
|
||||||
|
void Run(class ContentLoadTask* task);
|
||||||
|
|
||||||
|
public:
|
||||||
|
// [IRunnable]
|
||||||
|
String ToString() const override;
|
||||||
|
int32 Run() override;
|
||||||
|
void Exit() override;
|
||||||
|
};
|
||||||
@@ -67,6 +67,7 @@ protected:
|
|||||||
#if TRACY_ENABLE
|
#if TRACY_ENABLE
|
||||||
ZoneScoped;
|
ZoneScoped;
|
||||||
ZoneName(*name, name.Length());
|
ZoneName(*name, name.Length());
|
||||||
|
ZoneValue(chunk->LocationInFile.Size / 1024); // Size in kB
|
||||||
#endif
|
#endif
|
||||||
if (ref->Storage->LoadAssetChunk(chunk))
|
if (ref->Storage->LoadAssetChunk(chunk))
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -4,7 +4,6 @@
|
|||||||
#include "Engine/Core/Log.h"
|
#include "Engine/Core/Log.h"
|
||||||
#include "Engine/Threading/Threading.h"
|
#include "Engine/Threading/Threading.h"
|
||||||
#include "Engine/Streaming/StreamingGroup.h"
|
#include "Engine/Streaming/StreamingGroup.h"
|
||||||
#include "Engine/Content/Loading/ContentLoadingManager.h"
|
|
||||||
#include "Engine/Graphics/PixelFormatExtensions.h"
|
#include "Engine/Graphics/PixelFormatExtensions.h"
|
||||||
#include "Engine/Graphics/GPUDevice.h"
|
#include "Engine/Graphics/GPUDevice.h"
|
||||||
#include "Engine/Graphics/RenderTools.h"
|
#include "Engine/Graphics/RenderTools.h"
|
||||||
|
|||||||
Reference in New Issue
Block a user