Fix Vulkan synchronization between CPU and GPU to prevent running over frames in flight

Source/Engine/GraphicsDevice/Vulkan/CmdBufferVulkan.cpp

@@ -20,6 +20,15 @@ void CmdBufferVulkan::AddWaitSemaphore(VkPipelineStageFlags waitFlags, Semaphore
     _waitSemaphores.Add(waitSemaphore);
 }
 
+void CmdBufferVulkan::Wait(float timeInSecondsToWait)
+{
+    if (!IsSubmitted())
+        return;
+    const bool failed = _device->FenceManager.WaitForFence(_fence, (uint64)(timeInSecondsToWait * 1e9));
+    ASSERT(!failed);
+    RefreshFenceStatus();
+}
+
 void CmdBufferVulkan::Begin()
 {
     PROFILE_CPU();
@@ -186,9 +195,8 @@ CmdBufferVulkan::~CmdBufferVulkan()
     auto& fenceManager = _device->FenceManager;
     if (_state == State::Submitted)
     {
-        // Wait 60ms
-        const uint64 waitForCmdBufferInNanoSeconds = 60 * 1000 * 1000LL;
-        fenceManager.WaitAndReleaseFence(_fence, waitForCmdBufferInNanoSeconds);
+        // Wait
+        fenceManager.WaitAndReleaseFence(_fence);
     }
     else
     {
@@ -300,7 +308,6 @@ void CmdBufferManagerVulkan::SubmitActiveCmdBuffer(SemaphoreVulkan* signalSemaph
 
 void CmdBufferManagerVulkan::WaitForCmdBuffer(CmdBufferVulkan* cmdBuffer, float timeInSecondsToWait)
 {
-    PROFILE_CPU();
     ASSERT(cmdBuffer->IsSubmitted());
     const bool failed = _device->FenceManager.WaitForFence(cmdBuffer->GetFence(), (uint64)(timeInSecondsToWait * 1e9));
     ASSERT(!failed);

Source/Engine/GraphicsDevice/Vulkan/CmdBufferVulkan.h

@@ -119,6 +119,7 @@ public:
 
 public:
     void AddWaitSemaphore(VkPipelineStageFlags waitFlags, SemaphoreVulkan* waitSemaphore);
+    void Wait(float timeInSecondsToWait = 1.0f);
 
     void Begin();
     void End();

Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp

@@ -2251,8 +2251,13 @@ FenceVulkan* FenceManagerVulkan::AllocateFence(bool createSignaled)
 
 bool FenceManagerVulkan::WaitForFence(FenceVulkan* fence, uint64 timeInNanoseconds) const
 {
+    if (fence->IsSignaled)
+        return false;
+    PROFILE_CPU();
+    ZoneColor(TracyWaitZoneColor);
     ASSERT(_usedFences.Contains(fence));
-    ASSERT(!fence->IsSignaled);
+    if (timeInNanoseconds)
+        timeInNanoseconds = 1000ll * 1000ll * 1000LL; // 1s
     const VkResult result = vkWaitForFences(_device->Device, 1, &fence->Handle, true, timeInNanoseconds);
     LOG_VULKAN_RESULT(result);
     if (result == VK_SUCCESS)

Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h

@@ -106,7 +106,7 @@ public:
     }
 
     // Returns true if waiting timed out or failed, false otherwise.
-    bool WaitForFence(FenceVulkan* fence, uint64 timeInNanoseconds) const;
+    bool WaitForFence(FenceVulkan* fence, uint64 timeInNanoseconds = 0) const;
 
     void ResetFence(FenceVulkan* fence) const;
 
@@ -114,7 +114,7 @@ public:
     void ReleaseFence(FenceVulkan*& fence);
 
     // Sets the fence handle to null
-    void WaitAndReleaseFence(FenceVulkan*& fence, uint64 timeInNanoseconds);
+    void WaitAndReleaseFence(FenceVulkan*& fence, uint64 timeInNanoseconds = 0);
 
 private:
     // Returns true if fence was signaled, otherwise false.

Source/Engine/GraphicsDevice/Vulkan/GPUSwapChainVulkan.cpp

@@ -20,7 +20,7 @@ void BackBufferVulkan::Setup(GPUSwapChainVulkan* window, VkImage backbuffer, Pix
     initResource(VK_IMAGE_LAYOUT_UNDEFINED);
 
     Device = window->GetDevice();
-    Handle.Init(window->GetDevice(), this, backbuffer, 1, format, MSAALevel::None, extent, VK_IMAGE_VIEW_TYPE_2D);
+    Handle.Init(Device, this, backbuffer, 1, format, MSAALevel::None, extent, VK_IMAGE_VIEW_TYPE_2D);
     RenderingDoneSemaphore = New<SemaphoreVulkan>(Device);
     ImageAcquiredSemaphore = New<SemaphoreVulkan>(Device);
 }
@@ -30,6 +30,12 @@ void BackBufferVulkan::Release()
     Handle.Release();
     Delete(RenderingDoneSemaphore);
     Delete(ImageAcquiredSemaphore);
+    if (SubmitCmdBuffer)
+    {
+        SubmitCmdBuffer->Wait();
+        SubmitCmdBuffer = nullptr;
+    }
+    Device = nullptr;
 }
 
 GPUSwapChainVulkan::GPUSwapChainVulkan(GPUDeviceVulkan* device, Window* window)
@@ -133,6 +139,22 @@ GPUTextureView* GPUSwapChainVulkan::GetBackBufferView()
     return &_backBuffers[_acquiredImageIndex].Handle;
 }
 
+void GPUSwapChainVulkan::Begin(RenderTask* task)
+{
+    GPUSwapChain::Begin(task);
+
+    // Wait for the backbuffer to be available
+    if (_currentImageIndex != -1)
+    {
+        auto& backBuffer = _backBuffers[_currentImageIndex];
+        if (backBuffer.SubmitCmdBuffer)
+        {
+            backBuffer.SubmitCmdBuffer->Wait();
+            backBuffer.SubmitCmdBuffer = nullptr;
+        }
+    }
+}
+
 bool GPUSwapChainVulkan::Resize(int32 width, int32 height)
 {
     // Check if size won't change
@@ -540,6 +562,11 @@ void GPUSwapChainVulkan::Present(bool vsync)
     context->AddImageBarrier(backBuffer, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
     context->FlushBarriers();
 
+    // Cache a command buffer to wait on its fence before drawing to this backbuffer again
+    auto& acquiredBackBuffer = _backBuffers[_acquiredImageIndex];
+    ASSERT(acquiredBackBuffer.SubmitCmdBuffer == nullptr);
+    acquiredBackBuffer.SubmitCmdBuffer = context->GetCmdBufferManager()->GetActiveCmdBuffer();
+
     context->GetCmdBufferManager()->SubmitActiveCmdBuffer(_backBuffers[_acquiredImageIndex].RenderingDoneSemaphore);
 
     // Present the back buffer to the viewport window

Source/Engine/GraphicsDevice/Vulkan/GPUSwapChainVulkan.h

@@ -20,15 +20,20 @@ public:
     GPUDeviceVulkan* Device;
 
     /// <summary>
-    /// The image acquired semaphore handle.
+    /// The semaphore used to signal backbuffer image being ready to be used again for the commands rendering.
     /// </summary>
     SemaphoreVulkan* ImageAcquiredSemaphore;
 
     /// <summary>
-    /// The rendering done semaphore handle.
+    /// The semaphore used to signal the last command buffer rendering to be completed before presenting the image.
     /// </summary>
     SemaphoreVulkan* RenderingDoneSemaphore;
 
+    /// <summary>
+    /// The last command buffer used to submit commands for rendering to that backbuffer. Used to wait by CPU for the rendering completion before starting recording new commands (ensure that frame is not in flight).
+    /// </summary>
+    CmdBufferVulkan* SubmitCmdBuffer = nullptr;
+
     /// <summary>
     /// The render target surface handle.
     /// </summary>
@@ -109,6 +114,7 @@ public:
     bool IsFullscreen() override;
     void SetFullscreen(bool isFullscreen) override;
     GPUTextureView* GetBackBufferView() override;
+    void Begin(RenderTask* task) override;
     void Present(bool vsync) override;
     bool Resize(int32 width, int32 height) override;
     void CopyBackbuffer(GPUContext* context, GPUTexture* dst) override;

Source/Engine/GraphicsDevice/Vulkan/QueueVulkan.cpp

@@ -62,8 +62,7 @@ void QueueVulkan::Submit(CmdBufferVulkan* cmdBuffer, uint32 signalSemaphoresCoun
 	const bool WaitForIdleOnSubmit = false;
 	if (WaitForIdleOnSubmit)
 	{
-		// Use 200ms timeout
-		bool success = _device->FenceManager.WaitForFence(fence, 200 * 1000 * 1000);
+		bool success = _device->FenceManager.WaitForFence(fence);
 		ASSERT(success);
 		ASSERT(_device->FenceManager.IsFenceSignaled(fence));
 		cmdBuffer->GetOwner()->RefreshFenceStatus();