GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Optimize ContentStorageManager with Task Graph to use async update on a Job System

Browse Source
This commit is contained in:
Wojtek Figat
2021-07-02 14:43:19 +02:00
parent 57285ee3b5
commit 18b35517aa
4 changed files with 94 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
Source/Engine/Content/Storage/ContentStorageManager.cpp
View File

@@ -4,9 +4,11 @@
#include "FlaxFile.h"
#include "FlaxPackage.h"
#include "Engine/Core/Log.h"
#include "Engine/Engine/Engine.h"
#include "Engine/Engine/EngineService.h"
#include "Engine/Platform/FileSystem.h"
#include "Engine/Profiler/ProfilerCPU.h"
#include "Engine/Threading/TaskGraph.h"
namespace
{
@@ -21,20 +23,31 @@ namespace
Dictionary<String, FlaxStorage*> StorageMap(2048);
}
class ContentStorageManagerService : public EngineService
class ContentStorageService : public EngineService
{
public:
ContentStorageManagerService()
: EngineService(TEXT("ContentStorageManager"), -800)
ContentStorageService()
: EngineService(TEXT("ContentStorage"), -800)
{
}
void Update() override;
bool Init() override;
void Dispose() override;
};
ContentStorageManagerService ContentStorageManagerServiceInstance;
class ContentStorageSystem : public TaskGraphSystem
{
public:
void Job(int32 index);
void Execute(TaskGraph* graph) override;
};
namespace
{
TaskGraphSystem* System = nullptr;
}
ContentStorageService ContentStorageServiceInstance;
TimeSpan ContentStorageManager::UnusedDataChunksLifetime = TimeSpan::FromSeconds(10);
@@ -201,12 +214,30 @@ void ContentStorageManager::GetStorage(Array<FlaxStorage*>& result)
result.Add(Files[i]);
}
void ContentStorageManagerService::Update()
bool ContentStorageService::Init()
{
PROFILE_CPU();
System = New<ContentStorageSystem>();
Engine::UpdateGraph->AddSystem(System);
return false;
}
void ContentStorageService::Dispose()
{
ScopeLock lock(Locker);
for (auto i = StorageMap.Begin(); i.IsNotEnd(); ++i)
i->Value->Dispose();
Files.ClearDelete();
Packages.ClearDelete();
StorageMap.Clear();
ASSERT(Files.IsEmpty() && Packages.IsEmpty());
SAFE_DELETE(System);
}
void ContentStorageSystem::Job(int32 index)
{
PROFILE_CPU_NAMED("ContentStorage.Job");
ScopeLock lock(Locker);
for (auto i = StorageMap.Begin(); i.IsNotEnd(); ++i)
{
auto storage = i->Value;
@@ -230,17 +261,14 @@ void ContentStorageManagerService::Update()
}
}
void ContentStorageManagerService::Dispose()
void ContentStorageSystem::Execute(TaskGraph* graph)
{
ScopeLock lock(Locker);
if (StorageMap.Count() == 0)
return;
// Cleanup
for (auto i = StorageMap.Begin(); i.IsNotEnd(); ++i)
i->Value->Dispose();
Files.ClearDelete();
Packages.ClearDelete();
StorageMap.Clear();
// Ensure that data has been disposed
ASSERT(Files.IsEmpty() && Packages.IsEmpty());
// Schedule work to update all storage containers in async
Function<void(int32)> job;
job.Bind<ContentStorageSystem, &ContentStorageSystem::Job>(this);
graph->DispatchJob(job, 1);
}

60
Source/Engine/Content/Storage/FlaxChunk.h
View File

@@ -3,7 +3,6 @@
#pragma once
#include "Engine/Core/Types/DataContainer.h"
#include "Engine/Core/Types/DateTime.h"
#include "../Config.h"
/// <summary>
@@ -78,12 +77,12 @@ public:
/// <summary>
/// The chunk flags.
/// </summary>
FlaxChunkFlags Flags;
FlaxChunkFlags Flags = FlaxChunkFlags::None;
/// <summary>
/// The last usage time (UTC).
/// The last usage time (atomic, ticks of DateTime in UTC).
/// </summary>
DateTime LastAccessTime;
int64 LastAccessTime = 0;
/// <summary>
/// The chunk data.
@@ -96,48 +95,9 @@ public:
/// Initializes a new instance of the <see cref="FlaxChunk"/> class.
/// </summary>
FlaxChunk()
: Flags(FlaxChunkFlags::None)
, LastAccessTime(0)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="FlaxChunk"/> class.
/// </summary>
/// <param name="location">The chunk location in the file.</param>
FlaxChunk(const Location& location)
: LocationInFile(location)
, Flags(FlaxChunkFlags::None)
, LastAccessTime(0)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="FlaxChunk" /> class.
/// </summary>
/// <param name="location">The chunk location in the file.</param>
/// <param name="flags">The flags.</param>
FlaxChunk(const Location& location, const FlaxChunkFlags flags)
: LocationInFile(location)
, Flags(flags)
, LastAccessTime(0)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="FlaxChunk"/> class.
/// </summary>
/// <param name="address">The address in the file.</param>
/// <param name="size">The size in bytes.</param>
FlaxChunk(uint32 address, uint32 size)
: LocationInFile(address, size)
, Flags(FlaxChunkFlags::None)
, LastAccessTime(0)
{
}
public:
/// <summary>
/// Finalizes an instance of the <see cref="FlaxChunk"/> class.
/// </summary>
@@ -150,7 +110,6 @@ public:
/// <summary>
/// Gets this chunk data pointer.
/// </summary>
/// <returns>Data</returns>
FORCE_INLINE byte* Get()
{
return Data.Get();
@@ -159,7 +118,6 @@ public:
/// <summary>
/// Gets this chunk data pointer.
/// </summary>
/// <returns>Data</returns>
FORCE_INLINE const byte* Get() const
{
return Data.Get();
@@ -168,7 +126,6 @@ public:
/// <summary>
/// Gets this chunk data pointer.
/// </summary>
/// <returns>Data</returns>
template<typename T>
FORCE_INLINE T* Get() const
{
@@ -178,18 +135,14 @@ public:
/// <summary>
/// Gets this chunk data size (in bytes).
/// </summary>
/// <returns>Data size</returns>
FORCE_INLINE int32 Size() const
{
return Data.Length();
}
public:
/// <summary>
/// Determines whether this chunk is loaded.
/// </summary>
/// <returns>True if this instance is loaded, otherwise false.</returns>
FORCE_INLINE bool IsLoaded() const
{
return Data.IsValid();
@@ -198,7 +151,6 @@ public:
/// <summary>
/// Determines whether this chunk is missing (no data loaded or assigned).
/// </summary>
/// <returns>True if this instance is missing, otherwise false.</returns>
FORCE_INLINE bool IsMissing() const
{
return Data.IsInvalid();
@@ -207,7 +159,6 @@ public:
/// <summary>
/// Determines whether this chunk exists in a file.
/// </summary>
/// <returns>True if this instance is in a file, otherwise false.</returns>
FORCE_INLINE bool ExistsInFile() const
{
return LocationInFile.Size > 0;
@@ -216,10 +167,7 @@ public:
/// <summary>
/// Registers the usage operation of chunk data.
/// </summary>
void RegisterUsage()
{
LastAccessTime = DateTime::NowUTC();
}
void RegisterUsage();
/// <summary>
/// Unloads this chunk data.

39
Source/Engine/Content/Storage/FlaxStorage.cpp
View File

@@ -22,6 +22,11 @@ String AssetHeader::ToString() const
return String::Format(TEXT("ID: {0}, TypeName: {1}, Chunks Count: {2}"), ID, TypeName, GetChunksCount());
}
void FlaxChunk::RegisterUsage()
{
Platform::AtomicStore(&LastAccessTime, DateTime::NowUTC().Ticks);
}
const int32 FlaxStorage::MagicCode = 1180124739;
FlaxStorage::LockData FlaxStorage::LockData::Invalid(nullptr);
@@ -231,7 +236,7 @@ FlaxStorage::LockData FlaxStorage::LockSafe()
bool FlaxStorage::ShouldDispose() const
{
return _refCount == 0 && DateTime::NowUTC() - _lastRefLostTime >= TimeSpan::FromMilliseconds(500) && Platform::AtomicRead((int64*)&_chunksLock) == 0;
return _refCount == 0 && Platform::AtomicRead((int64*)&_chunksLock) == 0 && DateTime::NowUTC() - _lastRefLostTime >= TimeSpan::FromMilliseconds(500);
}
uint32 FlaxStorage::GetMemoryUsage() const
@@ -319,9 +324,10 @@ bool FlaxStorage::Load()
LOG(Warning, "Empty chunk found.");
return true;
}
FlaxChunkFlags flags;
stream->ReadInt32(reinterpret_cast<int32*>(&flags));
AddChunk(New<FlaxChunk>(e, flags));
auto chunk = New<FlaxChunk>();
chunk->LocationInFile = e;
stream->ReadInt32(reinterpret_cast<int32*>(&chunk->Flags));
AddChunk(chunk);
}
break;
@@ -369,9 +375,10 @@ bool FlaxStorage::Load()
LOG(Warning, "Empty chunk found.");
return true;
}
FlaxChunkFlags flags;
stream->ReadInt32(reinterpret_cast<int32*>(&flags));
AddChunk(New<FlaxChunk>(e, flags));
auto chunk = New<FlaxChunk>();
chunk->LocationInFile = e;
stream->ReadInt32(reinterpret_cast<int32*>(&chunk->Flags));
AddChunk(chunk);
}
break;
@@ -403,7 +410,9 @@ bool FlaxStorage::Load()
LOG(Warning, "Empty chunk found.");
return true;
}
AddChunk(New<FlaxChunk>(e));
auto chunk = New<FlaxChunk>();
chunk->LocationInFile = e;
AddChunk(chunk);
}
break;
@@ -436,7 +445,9 @@ bool FlaxStorage::Load()
LOG(Warning, "Empty chunk found.");
return true;
}
AddChunk(New<FlaxChunk>(e));
auto chunk = New<FlaxChunk>();
chunk->LocationInFile = e;
AddChunk(chunk);
}
break;
@@ -473,7 +484,9 @@ bool FlaxStorage::Load()
LOG(Warning, "Empty chunk found.");
return true;
}
AddChunk(New<FlaxChunk>(e));
auto chunk = New<FlaxChunk>();
chunk->LocationInFile = e;
AddChunk(chunk);
}
break;
@@ -529,7 +542,9 @@ bool FlaxStorage::Load()
auto& oldChunk = chunks[i];
if (oldChunk.Size > 0)
{
AddChunk(New<FlaxChunk>(oldChunk.Adress, oldChunk.Size));
auto chunk = New<FlaxChunk>();
chunk->LocationInFile = FlaxChunk::Location(oldChunk.Adress, oldChunk.Size);
AddChunk(chunk);
}
}
@@ -1305,7 +1320,7 @@ void FlaxStorage::Tick()
for (int32 i = 0; i < _chunks.Count(); i++)
{
auto chunk = _chunks[i];
const bool wasUsed = (now - chunk->LastAccessTime) < ContentStorageManager::UnusedDataChunksLifetime;
const bool wasUsed = (now - DateTime(Platform::AtomicRead(&chunk->LastAccessTime))) < ContentStorageManager::UnusedDataChunksLifetime;
if (!wasUsed && chunk->IsLoaded())
{
chunk->Unload();

35
Source/Engine/Content/Storage/FlaxStorage.h
View File

@@ -142,9 +142,6 @@ public:
struct LockData
{
friend FlaxStorage;
public:
static LockData Invalid;
private:
@@ -166,8 +163,12 @@ public:
if (_storage)
_storage->LockChunks();
}
public:
LockData(LockData&& other) noexcept
: _storage(other._storage)
{
other._storage = nullptr;
}
~LockData()
{
@@ -184,12 +185,8 @@ public:
}
}
public:
// Assignment operator
LockData& operator=(const LockData& other)
{
// Protect against invalid self-assignment
if (this != &other)
{
if (_storage)
@@ -198,9 +195,18 @@ public:
if (_storage)
_storage->LockChunks();
}
return *this;
}
LockData& operator=(LockData&& other) noexcept
{
if (this == &other)
return *this;
_storage = other._storage;
other._storage = nullptr;
return *this;
}
};
/// <summary>
@@ -224,7 +230,6 @@ public:
/// <summary>
/// Gets the references count.
/// </summary>
/// <returns>Amount of references to that storage container.</returns>
FORCE_INLINE uint32 GetRefCount() const
{
return _refCount;
@@ -233,13 +238,11 @@ public:
/// <summary>
/// Checks if storage container should be disposed (it's not used anymore).
/// </summary>
/// <returns>True if should be disposed, otherwise false.</returns>
bool ShouldDispose() const;
/// <summary>
/// Gets the path.
/// </summary>
/// <returns>File path</returns>
FORCE_INLINE const String& GetPath() const
{
return _path;
@@ -248,7 +251,6 @@ public:
/// <summary>
/// Determines whether this instance is loaded.
/// </summary>
/// <returns>True if this instance is loaded, otherwise false.</returns>
FORCE_INLINE bool IsLoaded() const
{
return _version != 0;
@@ -257,7 +259,6 @@ public:
/// <summary>
/// Determines whether this instance is disposed.
/// </summary>
/// <returns>True if this instance is disposed, otherwise false.</returns>
FORCE_INLINE bool IsDisposed() const
{
return _version == 0;
@@ -266,19 +267,16 @@ public:
/// <summary>
/// Gets total memory used by chunks (in bytes).
/// </summary>
/// <returns>Memory usage in bytes.</returns>
uint32 GetMemoryUsage() const;
/// <summary>
/// Determines whether this storage container is a package.
/// </summary>
/// <returns>True if this container is a package, otherwise false.</returns>
virtual bool IsPackage() const = 0;
/// <summary>
/// Checks whenever storage container allows the data modifications.
/// </summary>
/// <returns>True if can write data to the storage, otherwise it's readonly.</returns>
virtual bool AllowDataModifications() const = 0;
/// <summary>
@@ -298,7 +296,6 @@ public:
/// <summary>
/// Gets amount of entries in the storage.
/// </summary>
/// <returns>Entries count.</returns>
virtual int32 GetEntriesCount() const = 0;
/// <summary>