You're breathtaking!

Source/Engine/Content/Loading/ContentLoadTask.h

@@ -0,0 +1,90 @@
+// Copyright (c) 2012-2020 Wojciech Figat. All rights reserved.
+
+#pragma once
+
+#include "Engine/Threading/Task.h"
+
+class Asset;
+class LoadingThread;
+
+/// <summary>
+/// Describes content loading task object.
+/// </summary>
+class ContentLoadTask : public Task
+{
+    friend LoadingThread;
+
+public:
+
+    /// <summary>
+    /// Describes work type
+    /// </summary>
+    DECLARE_ENUM_3(Type, Custom, LoadAsset, LoadAssetData);
+
+    /// <summary>
+    /// Describes work result value
+    /// </summary>
+    DECLARE_ENUM_4(Result, Ok, AssetLoadError, MissingReferences, LoadDataError);
+
+private:
+
+    /// <summary>
+    /// Task type
+    /// </summary>
+    Type _type;
+
+protected:
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ContentLoadTask"/> class.
+    /// </summary>
+    /// <param name="type">The task type.</param>
+    ContentLoadTask(const Type type)
+        : _type(type)
+    {
+    }
+
+public:
+
+    /// <summary>
+    /// Gets a task type.
+    /// </summary>
+    /// <returns>The type.</returns>
+    FORCE_INLINE Type GetType() const
+    {
+        return _type;
+    }
+
+public:
+
+    /// <summary>
+    /// Checks if async task is loading given asset resource
+    /// </summary>
+    /// <param name="asset">Target asset to check</param>
+    /// <returns>True if is loading that asset, otherwise false</returns>
+    bool IsLoading(Asset* asset) const
+    {
+        return _type == Type::LoadAsset && HasReference((Object*)asset);
+    }
+
+protected:
+
+    virtual Result run() = 0;
+
+public:
+
+    // [Task]
+    String ToString() const override
+    {
+        return String::Format(TEXT("Content Load Task {0} ({1})"),
+                              ToString(GetType()),
+                              ::ToString(GetState())
+        );
+    }
+
+protected:
+
+    // [Task]
+    void Enqueue() override;
+    bool Run() override;
+};

Source/Engine/Content/Loading/ContentLoadingManager.cpp

@@ -0,0 +1,233 @@
+// Copyright (c) 2012-2020 Wojciech Figat. All rights reserved.
+
+#include "ContentLoadingManager.h"
+#include "ContentLoadTask.h"
+#include "Engine/Core/Log.h"
+#include "Engine/Core/Math/Math.h"
+#include "Engine/Platform/CPUInfo.h"
+#include "Engine/Platform/Thread.h"
+#include "Engine/Platform/ConditionVariable.h"
+#include "Engine/Content/Config.h"
+#include "Engine/Engine/EngineService.h"
+#include "Engine/Threading/Threading.h"
+#include "Engine/Threading/ConcurrentTaskQueue.h"
+#if USE_EDITOR && PLATFORM_WINDOWS
+#include "Engine/Platform/Win32/IncludeWindowsHeaders.h"
+#include <propidlbase.h>
+#endif
+
+namespace ContentLoadingManagerImpl
+{
+    THREADLOCAL LoadingThread* ThisThread = nullptr;
+    LoadingThread* MainThread = nullptr;
+    Array<LoadingThread*> Threads;
+    ConcurrentTaskQueue<ContentLoadTask> Tasks;
+    ConditionVariable TasksSignal;
+};
+
+using namespace ContentLoadingManagerImpl;
+
+class ContentLoadingManagerService : public EngineService
+{
+public:
+
+    ContentLoadingManagerService()
+        : EngineService(TEXT("Content Loading Manager"), -500)
+    {
+    }
+
+    bool Init() override;
+    void BeforeExit() override;
+    void Dispose() override;
+};
+
+ContentLoadingManagerService ContentLoadingManagerServiceInstance;
+
+LoadingThread::LoadingThread()
+    : _exitFlag(false)
+    , _thread(nullptr)
+    , _totalTasksDoneCount(0)
+{
+}
+
+LoadingThread::~LoadingThread()
+{
+    // Check if has thread attached
+    if (_thread != nullptr)
+    {
+        if (_thread->IsRunning())
+            _thread->Kill(true);
+        Delete(_thread);
+    }
+}
+
+uint64 LoadingThread::GetID() const
+{
+    return _thread ? _thread->GetID() : 0;
+}
+
+void LoadingThread::NotifyExit()
+{
+    Platform::InterlockedIncrement(&_exitFlag);
+}
+
+void LoadingThread::Join()
+{
+    auto thread = _thread;
+    if (thread)
+        thread->Join();
+}
+
+bool LoadingThread::Start(const String& name)
+{
+    ASSERT(_thread == nullptr && name.HasChars());
+
+    // Create new thread
+    auto thread = Thread::Create(this, name, ThreadPriority::Normal);
+    if (thread == nullptr)
+        return true;
+
+    _thread = thread;
+
+    return false;
+}
+
+void LoadingThread::Run(ContentLoadTask* job)
+{
+    ASSERT(job);
+
+    job->Execute();
+    _totalTasksDoneCount++;
+}
+
+String LoadingThread::ToString() const
+{
+    return String::Format(TEXT("Loading Thread {0}"), GetID());
+}
+
+int32 LoadingThread::Run()
+{
+#if USE_EDITOR && PLATFORM_WINDOWS
+
+    // Initialize COM
+    // TODO: maybe add sth to Thread::Create to indicate that thread will use COM stuff
+    const auto result = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
+    if (FAILED(result))
+    {
+        LOG(Error, "Failed to init COM for WIC texture importing! Result: {0:x}", static_cast<uint32>(result));
+        return -1;
+    }
+
+#endif
+
+    ContentLoadTask* task;
+    CriticalSection mutex;
+    ThisThread = this;
+
+    while (HasExitFlagClear())
+    {
+        if (Tasks.try_dequeue(task))
+        {
+            Run(task);
+        }
+        else
+        {
+            mutex.Lock();
+            TasksSignal.Wait(mutex);
+            mutex.Unlock();
+        }
+    }
+
+    ThisThread = nullptr;
+    return 0;
+}
+
+void LoadingThread::Exit()
+{
+    // Send info
+    ASSERT_LOW_LAYER(_thread);
+    LOG(Info, "Content thread '{0}' exited. Load calls: {1}", _thread->GetName(), _totalTasksDoneCount);
+}
+
+LoadingThread* ContentLoadingManager::GetCurrentLoadThread()
+{
+    return ThisThread;
+}
+
+int32 ContentLoadingManager::GetTasksCount()
+{
+    return Tasks.Count();
+}
+
+bool ContentLoadingManagerService::Init()
+{
+    ASSERT(ContentLoadingManagerImpl::Threads.IsEmpty() && IsInMainThread());
+
+    // Calculate amount of loading threads to use
+    const CPUInfo cpuInfo = Platform::GetCPUInfo();
+    const int32 count = static_cast<int32>(Math::Clamp(LOADING_THREAD_PER_PHYSICAL_CORE * cpuInfo.ProcessorCoreCount, 1.0f, 4.0f));
+    LOG(Info, "Creating {0} content loading threads...", count);
+
+    // Create loading threads
+    MainThread = New<LoadingThread>();
+    ThisThread = MainThread;
+    Threads.EnsureCapacity(count);
+    for (int32 i = 0; i < count; i++)
+    {
+        auto thread = New<LoadingThread>();
+        if (thread->Start(String::Format(TEXT("Load Thread {0}"), i)))
+        {
+            LOG(Fatal, "Cannot spawn content thread {0}/{1}", i, count);
+            Delete(thread);
+            return true;
+        }
+        Threads.Add(thread);
+    }
+
+    return false;
+}
+
+void ContentLoadingManagerService::BeforeExit()
+{
+    // Signal threads to end work soon
+    for (int32 i = 0; i < Threads.Count(); i++)
+        Threads[i]->NotifyExit();
+    TasksSignal.NotifyAll();
+}
+
+void ContentLoadingManagerService::Dispose()
+{
+    // Exit all threads
+    for (int32 i = 0; i < Threads.Count(); i++)
+        Threads[i]->NotifyExit();
+    TasksSignal.NotifyAll();
+    for (int32 i = 0; i < Threads.Count(); i++)
+        Threads[i]->Join();
+    Threads.ClearDelete();
+    Delete(MainThread);
+    MainThread = nullptr;
+    ThisThread = nullptr;
+
+    // Cancel all reaming tasks (no chance to execute them)
+    Tasks.CancelAll();
+}
+
+void ContentLoadTask::Enqueue()
+{
+    Tasks.Add(this);
+    TasksSignal.NotifyOne();
+}
+
+bool ContentLoadTask::Run()
+{
+    // Perform an operation
+    const auto result = run();
+
+    // Process result
+    const bool failed = result != Result::Ok;
+    if (failed)
+    {
+        LOG(Warning, "\'{0}\' failed with result: {1}", ToString(), ToString(result));
+    }
+    return failed;
+}

Source/Engine/Content/Loading/ContentLoadingManager.h

@@ -0,0 +1,119 @@
+// Copyright (c) 2012-2020 Wojciech Figat. All rights reserved.
+
+#pragma once
+
+#include "Engine/Threading/IRunnable.h"
+
+class Asset;
+class LoadingThread;
+class ContentLoadTask;
+
+/// <summary>
+/// Resources loading thread
+/// </summary>
+class LoadingThread : public IRunnable
+{
+protected:
+
+    volatile int64 _exitFlag;
+    Thread* _thread;
+    int32 _totalTasksDoneCount;
+
+public:
+
+    /// <summary>
+    /// Init
+    /// </summary>
+    LoadingThread();
+
+    /// <summary>
+    /// Destructor
+    /// </summary>
+    ~LoadingThread();
+
+public:
+
+    /// <summary>
+    /// Gets the thread identifier.
+    /// </summary>
+    /// <returns>Thread ID</returns>
+    uint64 GetID() const;
+
+public:
+
+    /// <summary>
+    /// Returns true if thread has empty exit flag, so it can continue it's work
+    /// </summary>
+    /// <returns>True if exit flag is empty, otherwise false</returns>
+    FORCE_INLINE bool HasExitFlagClear()
+    {
+        return Platform::AtomicRead(&_exitFlag) == 0;
+    }
+
+    /// <summary>
+    /// Set exit flag to true so thread must exit
+    /// </summary>
+    void NotifyExit();
+
+    /// <summary>
+    /// Stops the calling thread execution until the loading thread ends its execution.
+    /// </summary>
+    void Join();
+
+public:
+
+    /// <summary>
+    /// Starts thread execution.
+    /// </summary>
+    /// <param name="name">The thread name.</param>
+    /// <returns>True if cannot start, otherwise false</returns>
+    bool Start(const String& name);
+
+    /// <summary>
+    /// Runs the specified task.
+    /// </summary>
+    /// <param name="task">The task.</param>
+    void Run(ContentLoadTask* task);
+
+public:
+
+    // [IRunnable]
+    String ToString() const override;
+    int32 Run() override;
+    void Exit() override;
+};
+
+/// <summary>
+/// Content loading manager.
+/// </summary>
+class ContentLoadingManager
+{
+    friend ContentLoadTask;
+    friend LoadingThread;
+    friend Asset;
+
+public:
+
+    /// <summary>
+    /// Checks if current execution context is thread used to load assets.
+    /// </summary>
+    /// <returns>True if execution is in Load Thread, otherwise false.</returns>
+    FORCE_INLINE static bool IsInLoadThread()
+    {
+        return GetCurrentLoadThread() != nullptr;
+    }
+
+    /// <summary>
+    /// Gets content loading thread handle if current thread is one of them.
+    /// </summary>
+    /// <returns>Current load thread or null if current thread is different.</returns>
+    static LoadingThread* GetCurrentLoadThread();
+
+public:
+
+    /// <summary>
+    /// Gets amount of enqueued tasks to perform.
+    /// </summary>
+    /// <returns>The tasks count.</returns>
+    static int32 GetTasksCount();
+};

Source/Engine/Content/Loading/Tasks/LoadAssetDataTask.h

@@ -0,0 +1,90 @@
+// Copyright (c) 2012-2020 Wojciech Figat. All rights reserved.
+
+#pragma once
+
+#include "../ContentLoadTask.h"
+#include "Engine/Core/Log.h"
+#include "Engine/Content/AssetReference.h"
+#include "Engine/Content/BinaryAsset.h"
+#include "Engine/Content/WeakAssetReference.h"
+#include "Engine/Profiler/ProfilerCPU.h"
+
+/// <summary>
+/// Asset data loading task object.
+/// </summary>
+class LoadAssetDataTask : public ContentLoadTask
+{
+private:
+
+    WeakAssetReference<BinaryAsset> _asset; // Don't keep ref to the asset (so it can be unloaded if none using it, task will fail then)
+    AssetChunksFlag _chunks;
+    FlaxStorage::LockData _dataLock;
+
+public:
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="LoadAssetDataTask"/> class.
+    /// </summary>
+    /// <param name="asset">The asset to load.</param>
+    /// <param name="chunks">The chunks to load.</param>
+    LoadAssetDataTask(BinaryAsset* asset, AssetChunksFlag chunks)
+        : ContentLoadTask(Type::LoadAssetData)
+        , _asset(asset)
+        , _chunks(chunks)
+        , _dataLock(asset->Storage->Lock())
+    {
+    }
+
+public:
+
+    // [ContentLoadTask]
+    bool HasReference(Object* obj) const override
+    {
+        return obj == _asset;
+    }
+
+protected:
+
+    // [ContentLoadTask]
+    Result run() override
+    {
+        PROFILE_CPU();
+
+        AssetReference<BinaryAsset> ref = _asset.Get();
+        if (ref == nullptr)
+            return Result::MissingReferences;
+
+        // Load chunks
+        for (int32 i = 0; i < ASSET_FILE_DATA_CHUNKS; i++)
+        {
+            if (GET_CHUNK_FLAG(i) & _chunks)
+            {
+                const auto chunk = ref->GetChunk(i);
+                if (chunk != nullptr)
+                {
+                    // Check for cancel
+                    if (IsCancelRequested())
+                        return Result::Ok;
+
+                    // Load it
+                    if (ref->Storage->LoadAssetChunk(chunk))
+                    {
+                        LOG(Warning, "Cannot load asset \'{0}\' chunk {1}.", ref->ToString(), i);
+                        return Result::LoadDataError;
+                    }
+                }
+            }
+        }
+
+        return Result::Ok;
+    }
+
+    void OnEnd() override
+    {
+        _dataLock.Release();
+        _asset.Unlink();
+
+        // Base
+        ContentLoadTask::OnEnd();
+    }
+};

Source/Engine/Content/Loading/Tasks/LoadAssetTask.h

@@ -0,0 +1,76 @@
+// Copyright (c) 2012-2020 Wojciech Figat. All rights reserved.
+
+#pragma once
+
+#include "../ContentLoadTask.h"
+#include "Engine/Content/Asset.h"
+#include "Engine/Content/AssetReference.h"
+#include "Engine/Content/WeakAssetReference.h"
+#include "Engine/Profiler/ProfilerCPU.h"
+
+/// <summary>
+/// Asset loading task object.
+/// </summary>
+class LoadAssetTask : public ContentLoadTask
+{
+private:
+
+    WeakAssetReference<Asset> _asset;
+
+public:
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="LoadAssetTask"/> class.
+    /// </summary>
+    /// <param name="asset">The asset to load.</param>
+    LoadAssetTask(Asset* asset)
+        : ContentLoadTask(Type::LoadAsset)
+        , _asset(asset)
+    {
+    }
+
+public:
+
+    /// <summary>
+    /// Gets the asset.
+    /// </summary>
+    /// <returns>The asset.</returns>
+    FORCE_INLINE Asset* GetAsset() const
+    {
+        return _asset.Get();
+    }
+
+public:
+
+    // [ContentLoadTask]
+    bool HasReference(Object* obj) const override
+    {
+        return obj == _asset;
+    }
+
+protected:
+
+    // [ContentLoadTask]
+    Result run() override
+    {
+        PROFILE_CPU();
+
+        AssetReference<Asset> ref = _asset.Get();
+        if (ref == nullptr)
+            return Result::MissingReferences;
+
+        // Call loading
+        if (ref->onLoad(this))
+            return Result::AssetLoadError;
+
+        return Result::Ok;
+    }
+
+    void OnEnd() override
+    {
+        _asset.Unlink();
+
+        // Base
+        ContentLoadTask::OnEnd();
+    }
+};