GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for storing custom platform tools data in Game Cooker cache

Browse Source
This commit is contained in:
Wojtek Figat
2023-12-18 11:07:11 +01:00
parent c6c53baff2
commit e4df1fc756
4 changed files with 97 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Flax.flaxproj
View File

@@ -4,7 +4,7 @@
"Major": 1,
"Minor": 7,
"Revision": 1,
"Build": 6406
"Build": 6407
},
"Company": "Flax",
"Copyright": "Copyright (c) 2012-2023 Wojciech Figat. All rights reserved.",

52
Source/Editor/Cooker/PlatformTools.h
View File

@@ -8,6 +8,37 @@
class TextureBase;
/// <summary>
/// The game cooker cache interface.
/// </summary>
class FLAXENGINE_API IBuildCache
{
public:
/// <summary>
/// Removes all cached entries for assets that contain a given asset type. This forces rebuild for them.
/// </summary>
virtual void InvalidateCachePerType(const StringView& typeName) = 0;
/// <summary>
/// Removes all cached entries for assets that contain a given asset type. This forces rebuild for them.
/// </summary>
template<typename T>
FORCE_INLINE void InvalidateCachePerType()
{
InvalidateCachePerType(T::TypeName);
}
/// <summary>
/// Removes all cached entries for assets that contain a shader. This forces rebuild for them.
/// </summary>
void InvalidateCacheShaders();
/// <summary>
/// Removes all cached entries for assets that contain a texture. This forces rebuild for them.
/// </summary>
void InvalidateCacheTextures();
};
/// <summary>
/// The platform support tools base interface.
/// </summary>
@@ -76,6 +107,27 @@ public:
virtual bool IsNativeCodeFile(CookingData& data, const String& file);
public:
/// <summary>
/// Loads the build cache. Allows to invalidate any cached asset types based on the build settings for incremental builds (eg. invalidate textures/shaders).
/// </summary>
/// <param name="data">The cooking data.</param>
/// <param name="data">The build cache interface.</param>
/// <param name="data">The loaded cache data. Can be empty when starting a fresh build.</param>
virtual void LoadCache(CookingData& data, IBuildCache* cache, const Span<byte>& bytes)
{
}
/// <summary>
/// Saves the build cache. Allows to store any build settings to be used for cache invalidation on incremental builds.
/// </summary>
/// <param name="data">The cooking data.</param>
/// <param name="data">The build cache interface.</param>
/// <returns>Data to cache, will be restored during next incremental build.<returns>
virtual Array<byte> SaveCache(CookingData& data, IBuildCache* cache)
{
return Array<byte>();
}
/// <summary>
/// Called when game building starts.
/// </summary>

56
Source/Editor/Cooker/Steps/CookAssetsStep.cpp
View File

@@ -35,6 +35,7 @@
#include "Engine/Engine/Base/GameBase.h"
#include "Engine/Engine/Globals.h"
#include "Engine/Tools/TextureTool/TextureTool.h"
#include "Engine/Profiler/ProfilerCPU.h"
#include "Engine/Scripting/Enums.h"
#if PLATFORM_TOOLS_WINDOWS
#include "Engine/Platform/Windows/WindowsPlatformSettings.h"
@@ -49,6 +50,20 @@
Dictionary<String, CookAssetsStep::ProcessAssetFunc> CookAssetsStep::AssetProcessors;
void IBuildCache::InvalidateCacheShaders()
{
InvalidateCachePerType<Shader>();
InvalidateCachePerType<Material>();
InvalidateCachePerType<ParticleEmitter>();
}
void IBuildCache::InvalidateCacheTextures()
{
InvalidateCachePerType<Texture>();
InvalidateCachePerType<CubeTexture>();
InvalidateCachePerType<SpriteAtlas>();
}
bool CookAssetsStep::CacheEntry::IsValid(bool withDependencies)
{
AssetInfo assetInfo;
@@ -113,15 +128,13 @@ void CookAssetsStep::CacheData::InvalidateCachePerType(const StringView& typeNam
void CookAssetsStep::CacheData::Load(CookingData& data)
{
PROFILE_CPU();
HeaderFilePath = data.CacheDirectory / String::Format(TEXT("CookedHeader_{0}.bin"), FLAXENGINE_VERSION_BUILD);
CacheFolder = data.CacheDirectory / TEXT("Cooked");
Entries.Clear();
if (!FileSystem::DirectoryExists(CacheFolder))
{
FileSystem::CreateDirectory(CacheFolder);
}
if (!FileSystem::FileExists(HeaderFilePath))
{
LOG(Warning, "Missing incremental build cooking assets cache.");
@@ -143,9 +156,7 @@ void CookAssetsStep::CacheData::Load(CookingData& data)
return;
LOG(Info, "Loading incremental build cooking cache (entries count: {0})", entriesCount);
file->ReadBytes(&Settings, sizeof(Settings));
Entries.EnsureCapacity(Math::RoundUpToPowerOf2(static_cast<int32>(entriesCount * 3.0f)));
Array<Pair<String, DateTime>> fileDependencies;
@@ -179,6 +190,9 @@ void CookAssetsStep::CacheData::Load(CookingData& data)
e.FileDependencies = fileDependencies;
}
Array<byte> platformCache;
file->Read(platformCache);
int32 checkChar;
file->ReadInt32(&checkChar);
if (checkChar != 13)
@@ -187,6 +201,9 @@ void CookAssetsStep::CacheData::Load(CookingData& data)
Entries.Clear();
}
// Per-platform custom data loading (eg. to invalidate textures/shaders options)
data.Tools->LoadCache(data, this, ToSpan(platformCache));
const auto buildSettings = BuildSettings::Get();
const auto gameSettings = GameSettings::Get();
@@ -200,12 +217,12 @@ void CookAssetsStep::CacheData::Load(CookingData& data)
if (MATERIAL_GRAPH_VERSION != Settings.Global.MaterialGraphVersion)
{
LOG(Info, "{0} option has been modified.", TEXT("MaterialGraphVersion"));
InvalidateCachePerType(Material::TypeName);
InvalidateCachePerType<Material>();
}
if (PARTICLE_GPU_GRAPH_VERSION != Settings.Global.ParticleGraphVersion)
{
LOG(Info, "{0} option has been modified.", TEXT("ParticleGraphVersion"));
InvalidateCachePerType(ParticleEmitter::TypeName);
InvalidateCachePerType<ParticleEmitter>();
}
if (buildSettings->ShadersNoOptimize != Settings.Global.ShadersNoOptimize)
{
@@ -262,24 +279,24 @@ void CookAssetsStep::CacheData::Load(CookingData& data)
#endif
if (invalidateShaders)
{
InvalidateCachePerType(Shader::TypeName);
InvalidateCachePerType(Material::TypeName);
InvalidateCachePerType(ParticleEmitter::TypeName);
InvalidateCachePerType<Shader>();
InvalidateCachePerType<Material>();
InvalidateCachePerType<ParticleEmitter>();
}
// Invalidate textures if streaming settings gets modified
if (Settings.Global.StreamingSettingsAssetId != gameSettings->Streaming || (Entries.ContainsKey(gameSettings->Streaming) && !Entries[gameSettings->Streaming].IsValid()))
{
InvalidateCachePerType(Texture::TypeName);
InvalidateCachePerType(CubeTexture::TypeName);
InvalidateCachePerType(SpriteAtlas::TypeName);
InvalidateCachePerType<Texture>();
InvalidateCachePerType<CubeTexture>();
InvalidateCachePerType<SpriteAtlas>();
}
}
void CookAssetsStep::CacheData::Save()
void CookAssetsStep::CacheData::Save(CookingData& data)
{
PROFILE_CPU();
LOG(Info, "Saving incremental build cooking cache (entries count: {0})", Entries.Count());
auto file = FileWriteStream::Open(HeaderFilePath);
if (file == nullptr)
return;
@@ -302,6 +319,7 @@ void CookAssetsStep::CacheData::Save()
file->Write(f.Second);
}
}
file->Write(data.Tools->SaveCache(data, this));
file->WriteInt32(13);
}
@@ -961,6 +979,7 @@ public:
const int32 count = addedEntries.Count();
if (count == 0)
return false;
PROFILE_CPU();
// Get assets init data and load all chunks
Array<AssetInitData> assetsData;
@@ -1143,7 +1162,7 @@ bool CookAssetsStep::Perform(CookingData& data)
// Cook asset
if (Process(data, cache, assetRef.Get()))
{
cache.Save();
cache.Save(data);
return true;
}
data.Stats.CookedAssets++;
@@ -1151,12 +1170,12 @@ bool CookAssetsStep::Perform(CookingData& data)
// Auto save build cache after every few cooked assets (reduces next build time if cooking fails later)
if (data.Stats.CookedAssets % 50 == 0)
{
cache.Save();
cache.Save(data);
}
}
// Save build cache header
cache.Save();
cache.Save(data);
// Create build game header
{
@@ -1173,7 +1192,6 @@ bool CookAssetsStep::Perform(CookingData& data)
}
stream->WriteInt32(('x' + 'D') * 131); // think about it as '131 times xD'
stream->WriteInt32(FLAXENGINE_VERSION_BUILD);
Array<byte> bytes;

19
Source/Editor/Cooker/Steps/CookAssetsStep.h
View File

@@ -3,6 +3,7 @@
#pragma once
#include "Editor/Cooker/GameCooker.h"
#include "Editor/Cooker/PlatformTools.h"
#include "Engine/Core/Types/Pair.h"
#include "Engine/Core/Types/DateTime.h"
#include "Engine/Core/Collections/Dictionary.h"
@@ -56,7 +57,7 @@ public:
/// <summary>
/// Assets cooking cache data (incremental building feature).
/// </summary>
struct FLAXENGINE_API CacheData
struct FLAXENGINE_API CacheData : public IBuildCache
{
/// <summary>
/// The cache header file path.
@@ -136,16 +137,6 @@ public:
/// <returns>The added entry reference.</returns>
CacheEntry& CreateEntry(const Asset* asset, String& cachedFilePath);
/// <summary>
/// Removes all cached entries for assets that contain a shader. This forces rebuild for them.
/// </summary>
void InvalidateShaders();
/// <summary>
/// Removes all cached entries for assets that contain a texture. This forces rebuild for them.
/// </summary>
void InvalidateCachePerType(const StringView& typeName);
/// <summary>
/// Loads the cache for the given cooking data.
/// </summary>
@@ -155,7 +146,11 @@ public:
/// <summary>
/// Saves this cache (header file).
/// </summary>
void Save();
/// <param name="data">The data.</param>
void Save(CookingData& data);
using IBuildCache::InvalidateCachePerType;
void InvalidateCachePerType(const StringView& typeName) override;
};
struct FLAXENGINE_API AssetCookData