From cdff7708fb8e5b47060a36775b7075fdcf4ee9a0 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Tue, 12 Aug 2025 10:25:02 +0200 Subject: [PATCH] Add automatic periodic Vulkan Pipeline State Cache serialization --- .../GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp | 62 ++++++++++++++----- .../GraphicsDevice/Vulkan/GPUDeviceVulkan.h | 9 ++- .../Vulkan/GPUPipelineStateVulkan.cpp | 2 + 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp index 93819cfb5..9ff9ab77b 100644 --- a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp +++ b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp @@ -36,6 +36,7 @@ #include "Engine/Profiler/ProfilerCPU.h" #include "Engine/Profiler/ProfilerMemory.h" #include "Engine/Threading/Threading.h" +#include "Engine/Threading/ThreadPoolTask.h" #include "Engine/Scripting/Enums.h" #if !USE_EDITOR && (PLATFORM_WINDOWS || PLATFORM_LINUX) @@ -1540,29 +1541,50 @@ bool VulkanPlatformBase::SaveCache(const String& folder, const Char* filename, c #define CACHE_FOLDER Globals::ProductLocalFolder #endif -bool GPUDeviceVulkan::SavePipelineCache() +#if VULKAN_USE_PIPELINE_CACHE +bool SavePipelineCacheAsync() +{ + PROFILE_CPU(); + ((GPUDeviceVulkan*)GPUDevice::Instance)->SavePipelineCache(false, true); + return false; +} +#endif + +bool GPUDeviceVulkan::SavePipelineCache(bool async, bool cached) { #if VULKAN_USE_PIPELINE_CACHE - if (PipelineCache == VK_NULL_HANDLE || !vkGetPipelineCacheData) + if (PipelineCache == VK_NULL_HANDLE || !vkGetPipelineCacheData || PipelineCacheUsage == 0) return false; PROFILE_CPU(); PROFILE_MEM(Graphics); - // Query data size - size_t dataSize = 0; - VkResult result = vkGetPipelineCacheData(Device, PipelineCache, &dataSize, nullptr); - LOG_VULKAN_RESULT_WITH_RETURN(result); - if (dataSize <= 0) - return false; + if (!cached) + { + // Query data size + size_t dataSize = 0; + VkResult result = vkGetPipelineCacheData(Device, PipelineCache, &dataSize, nullptr); + LOG_VULKAN_RESULT_WITH_RETURN(result); + if (dataSize <= 0) + return false; - // Query data - Array data; - data.Resize((int32)dataSize); - result = vkGetPipelineCacheData(Device, PipelineCache, &dataSize, data.Get()); - LOG_VULKAN_RESULT_WITH_RETURN(result); + // Query data + PipelineCacheSaveData.Resize((int32)dataSize); + result = vkGetPipelineCacheData(Device, PipelineCache, &dataSize, PipelineCacheSaveData.Get()); + LOG_VULKAN_RESULT_WITH_RETURN(result); + } + + if (async) + { + // Kick off the async job that will save the cached bytes + Function action(SavePipelineCacheAsync); + return Task::StartNew(action) != nullptr; + } + + // Reset usage counter + PipelineCacheUsage = 0; // Save data - return VulkanPlatform::SaveCache(CACHE_FOLDER, TEXT("VulkanPipeline.cache"), data); + return VulkanPlatform::SaveCache(CACHE_FOLDER, TEXT("VulkanPipeline.cache"), PipelineCacheSaveData); #else return false; #endif @@ -2014,6 +2036,7 @@ bool GPUDeviceVulkan::Init() pipelineCacheCreateInfo.pInitialData = data.Count() > 0 ? data.Get() : nullptr; const VkResult result = vkCreatePipelineCache(Device, &pipelineCacheCreateInfo, nullptr, &PipelineCache); LOG_VULKAN_RESULT(result); + PipelineCacheSaveTime = Platform::GetTimeSeconds(); } #endif #if VULKAN_USE_VALIDATION_CACHE @@ -2067,6 +2090,17 @@ void GPUDeviceVulkan::DrawBegin() DeferredDeletionQueue.ReleaseResources(); StagingManager.ProcessPendingFree(); DescriptorPoolsManager->GC(); + +#if VULKAN_USE_PIPELINE_CACHE + // Serialize pipeline cache periodically for less PSO hitches on next app run + const double time = Platform::GetTimeSeconds(); + const double saveTimeFrequency = Engine::FrameCount < 60 * Math::Clamp(Engine::GetFramesPerSecond(), 30, 60) ? 10 : 180; // More frequent saves during the first 1min of gameplay + if (Engine::HasFocus && time - PipelineCacheSaveTime >= saveTimeFrequency) + { + SavePipelineCache(true); + PipelineCacheSaveTime = time; + } +#endif } void GPUDeviceVulkan::Dispose() diff --git a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h index b593315d8..a30dbda1c 100644 --- a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h +++ b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h @@ -502,6 +502,11 @@ public: /// The pipeline cache. /// VkPipelineCache PipelineCache = VK_NULL_HANDLE; +#if VULKAN_USE_PIPELINE_CACHE + uint32 PipelineCacheUsage = 0; + double PipelineCacheSaveTime = 0.0f; + Array PipelineCacheSaveData; +#endif #if VULKAN_USE_VALIDATION_CACHE /// @@ -562,7 +567,9 @@ public: /// /// Saves the pipeline cache. /// - bool SavePipelineCache(); + /// Enables async writing to file to reduce stuttering of main thread. + /// Uses cached results from the last call to vkGetPipelineCacheData, used to just save cached data when running in async. + bool SavePipelineCache(bool async = false, bool cached = false); #if VK_EXT_validation_cache /// diff --git a/Source/Engine/GraphicsDevice/Vulkan/GPUPipelineStateVulkan.cpp b/Source/Engine/GraphicsDevice/Vulkan/GPUPipelineStateVulkan.cpp index 927e4fbca..ef7520f68 100644 --- a/Source/Engine/GraphicsDevice/Vulkan/GPUPipelineStateVulkan.cpp +++ b/Source/Engine/GraphicsDevice/Vulkan/GPUPipelineStateVulkan.cpp @@ -113,6 +113,7 @@ ComputePipelineStateVulkan* GPUShaderProgramCSVulkan::GetOrCreateState() // Create pipeline object VkPipeline pipeline; VkResult result = vkCreateComputePipelines(_device->Device, _device->PipelineCache, 1, &desc, nullptr, &pipeline); + _device->PipelineCacheUsage++; LOG_VULKAN_RESULT(result); if (result != VK_SUCCESS) return nullptr; @@ -313,6 +314,7 @@ VkPipeline GPUPipelineStateVulkan::GetState(RenderPassVulkan* renderPass, GPUVer auto depthWrite = _descDepthStencil.depthWriteEnable; _descDepthStencil.depthWriteEnable &= renderPass->CanDepthWrite ? 1 : 0; const VkResult result = vkCreateGraphicsPipelines(_device->Device, _device->PipelineCache, 1, &_desc, nullptr, &pipeline); + _device->PipelineCacheUsage++; _descDepthStencil.depthWriteEnable = depthWrite; LOG_VULKAN_RESULT(result); if (result != VK_SUCCESS)