From 4486ec3b72938b7ab5d348f00d633c7a4f466fb6 Mon Sep 17 00:00:00 2001
From: Wojciech Figat <wojtek@figat.pl>
Date: Tue, 11 Jan 2022 10:49:03 +0100
Subject: [PATCH] Add Vulkan pipeline cache serialization

---
 .../GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp | 139 +++++++++++-------
 .../GraphicsDevice/Vulkan/GPUDeviceVulkan.h   |  10 +-
 2 files changed, 94 insertions(+), 55 deletions(-)

diff --git a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp
index 7360a25fd..0ccbf3417 100644
--- a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp
+++ b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp
@@ -1370,6 +1370,39 @@ PixelFormat GPUDeviceVulkan::GetClosestSupportedPixelFormat(PixelFormat format,
     return format;
 }
 
+void GetPipelineCachePath(String& path)
+{
+#if USE_EDITOR
+    path = Globals::ProjectCacheFolder / TEXT("VulkanPipeline.cache");
+#else
+    path = Globals::ProductLocalFolder / TEXT("VulkanPipeline.cache");
+#endif
+}
+
+bool GPUDeviceVulkan::SavePipelineCache()
+{
+    if (PipelineCache == VK_NULL_HANDLE || !vkGetPipelineCacheData)
+        return false;
+
+    // Query data size
+    size_t dataSize = 0;
+    VkResult result = vkGetPipelineCacheData(Device, PipelineCache, &dataSize, nullptr);
+    LOG_VULKAN_RESULT_WITH_RETURN(result);
+    if (dataSize <= 0)
+        return false;
+
+    // Query data
+    Array<byte> data;
+    data.Resize((int32)dataSize);
+    result = vkGetPipelineCacheData(Device, PipelineCache, &dataSize, data.Get());
+    LOG_VULKAN_RESULT_WITH_RETURN(result);
+
+    // Save data
+    String path;
+    GetPipelineCachePath(path);
+    return File::WriteAllBytes(path, data);
+}
+
 #if VK_EXT_validation_cache
 
 void GetValidationCachePath(String& path)
@@ -1381,54 +1414,8 @@ void GetValidationCachePath(String& path)
 #endif
 }
 
-void GPUDeviceVulkan::LoadValidationCache()
-{
-    Array<uint8> data;
-
-    String path;
-    GetValidationCachePath(path);
-
-    if (FileSystem::FileExists(path))
-    {
-        LOG(Info, "Trying to load Vulkan validation cache file {0}", path);
-        File::ReadAllBytes(path, data);
-
-        if (data.HasItems())
-        {
-            int32* dataPtr = (int32*)data.Get();
-            if (*dataPtr > 0)
-            {
-                dataPtr++;
-                const int32 version = *dataPtr++;
-                if (version == VK_PIPELINE_CACHE_HEADER_VERSION_ONE)
-                {
-                    dataPtr += VK_UUID_SIZE / sizeof(int32);
-                }
-                else
-                {
-                    LOG(Warning, "Bad validation cache file, version: {0}, expected: {1}", version, VK_PIPELINE_CACHE_HEADER_VERSION_ONE);
-                    data.Clear();
-                }
-            }
-            else
-            {
-                LOG(Warning, "Bad validation cache file, header size: {0}", *dataPtr);
-                data.Clear();
-            }
-        }
-    }
-
-    VkValidationCacheCreateInfoEXT validationCreateInfo;
-    RenderToolsVulkan::ZeroStruct(validationCreateInfo, VK_STRUCTURE_TYPE_VALIDATION_CACHE_CREATE_INFO_EXT);
-    validationCreateInfo.initialDataSize = data.Count();
-    validationCreateInfo.pInitialData = data.Count() > 0 ? data.Get() : nullptr;
-    const VkResult result = vkCreateValidationCacheEXT(Device, &validationCreateInfo, nullptr, &ValidationCache);
-    LOG_VULKAN_RESULT(result);
-}
-
 bool GPUDeviceVulkan::SaveValidationCache()
 {
-    // Skip if missing
     if (ValidationCache == VK_NULL_HANDLE || !vkGetValidationCacheDataEXT)
         return false;
 
@@ -1849,11 +1836,63 @@ bool GPUDeviceVulkan::Init()
     UniformBufferUploader = New<UniformBufferUploaderVulkan>(this);
     DescriptorPoolsManager = New<DescriptorPoolsManagerVulkan>(this);
     MainContext = New<GPUContextVulkan>(this, GraphicsQueue);
-    // TODO: create and load PipelineCache
+    if (vkCreatePipelineCache)
+    {
+        Array<uint8> data;
+        String path;
+        GetPipelineCachePath(path);
+        if (FileSystem::FileExists(path))
+        {
+            LOG(Info, "Trying to load Vulkan pipeline cache file {0}", path);
+            File::ReadAllBytes(path, data);
+        }
+        VkPipelineCacheCreateInfo pipelineCacheCreateInfo;
+        RenderToolsVulkan::ZeroStruct(pipelineCacheCreateInfo, VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO);
+        pipelineCacheCreateInfo.initialDataSize = data.Count();
+        pipelineCacheCreateInfo.pInitialData = data.Count() > 0 ? data.Get() : nullptr;
+        const VkResult result = vkCreatePipelineCache(Device, &pipelineCacheCreateInfo, nullptr, &PipelineCache);
+        LOG_VULKAN_RESULT(result);
+    }
 #if VK_EXT_validation_cache
     if (OptionalDeviceExtensions.HasEXTValidationCache && vkCreateValidationCacheEXT && vkDestroyValidationCacheEXT)
     {
-        LoadValidationCache();
+        Array<uint8> data;
+        String path;
+        GetValidationCachePath(path);
+        if (FileSystem::FileExists(path))
+        {
+            LOG(Info, "Trying to load Vulkan validation cache file {0}", path);
+            File::ReadAllBytes(path, data);
+            if (data.HasItems())
+            {
+                int32* dataPtr = (int32*)data.Get();
+                if (*dataPtr > 0)
+                {
+                    dataPtr++;
+                    const int32 version = *dataPtr++;
+                    if (version == VK_PIPELINE_CACHE_HEADER_VERSION_ONE)
+                    {
+                        dataPtr += VK_UUID_SIZE / sizeof(int32);
+                    }
+                    else
+                    {
+                        LOG(Warning, "Bad validation cache file, version: {0}, expected: {1}", version, VK_PIPELINE_CACHE_HEADER_VERSION_ONE);
+                        data.Clear();
+                    }
+                }
+                else
+                {
+                    LOG(Warning, "Bad validation cache file, header size: {0}", *dataPtr);
+                    data.Clear();
+                }
+            }
+        }
+        VkValidationCacheCreateInfoEXT validationCreateInfo;
+        RenderToolsVulkan::ZeroStruct(validationCreateInfo, VK_STRUCTURE_TYPE_VALIDATION_CACHE_CREATE_INFO_EXT);
+        validationCreateInfo.initialDataSize = data.Count();
+        validationCreateInfo.pInitialData = data.Count() > 0 ? data.Get() : nullptr;
+        const VkResult result = vkCreateValidationCacheEXT(Device, &validationCreateInfo, nullptr, &ValidationCache);
+        LOG_VULKAN_RESULT(result);
     }
 #endif
 
@@ -1911,6 +1950,8 @@ void GPUDeviceVulkan::Dispose()
     Allocator = VK_NULL_HANDLE;
     if (PipelineCache != VK_NULL_HANDLE)
     {
+        if (SavePipelineCache())
+            LOG(Warning, "Failed to save Vulkan pipeline cache");
         vkDestroyPipelineCache(Device, PipelineCache, nullptr);
         PipelineCache = VK_NULL_HANDLE;
     }
@@ -1918,9 +1959,7 @@ void GPUDeviceVulkan::Dispose()
     if (ValidationCache != VK_NULL_HANDLE)
     {
         if (SaveValidationCache())
-        {
             LOG(Warning, "Failed to save Vulkan validation cache");
-        }
         vkDestroyValidationCacheEXT(Device, ValidationCache, nullptr);
         ValidationCache = VK_NULL_HANDLE;
     }
diff --git a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h
index ea0d4edff..59b89b00c 100644
--- a/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h
+++ b/Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h
@@ -672,14 +672,14 @@ public:
     /// <param name="optimalTiling">If set to <c>true</c> the optimal tiling should be used, otherwise use linear tiling.</param>
     /// <returns>The output format.</returns>
     PixelFormat GetClosestSupportedPixelFormat(PixelFormat format, GPUTextureFlags flags, bool optimalTiling);
+    
+    /// <summary>
+    /// Saves the pipeline cache.
+    /// </summary>
+    bool SavePipelineCache();
 
 #if VK_EXT_validation_cache
 
-    /// <summary>
-    /// Loads the validation cache.
-    /// </summary>
-    void LoadValidationCache();
-
     /// <summary>
     /// Saves the validation cache.
     /// </summary>