Add new GPU Query API that is lightweight and supports occlusion queries

Source/Engine/GraphicsDevice/Vulkan/CmdBufferVulkan.cpp

@@ -6,7 +6,7 @@
 #include "RenderToolsVulkan.h"
 #include "QueueVulkan.h"
 #include "GPUContextVulkan.h"
-#if VULKAN_USE_QUERIES
+#if VULKAN_USE_TIMER_QUERIES
 #include "GPUTimerQueryVulkan.h"
 #endif
 #include "DescriptorSetVulkan.h"
@@ -243,6 +243,7 @@ void CmdBufferPoolVulkan::RefreshFenceStatus(const CmdBufferVulkan* skipCmdBuffe
 
 CmdBufferManagerVulkan::CmdBufferManagerVulkan(GPUDeviceVulkan* device, GPUContextVulkan* context)
     : _device(device)
+    , _context(context)
     , _pool(device)
     , _queue(context->GetQueue())
     , _activeCmdBuffer(nullptr)
@@ -259,12 +260,28 @@ void CmdBufferManagerVulkan::SubmitActiveCmdBuffer(SemaphoreVulkan* signalSemaph
         if (_activeCmdBuffer->IsInsideRenderPass())
             _activeCmdBuffer->EndRenderPass();
 
-#if VULKAN_USE_QUERIES
-        // Pause all active queries
-        for (int32 i = 0; i < _queriesInProgress.Count(); i++)
+#if VULKAN_USE_TIMER_QUERIES && GPU_VULKAN_PAUSE_QUERIES
+        // Pause all active timer queries
+        auto queries = _activeTimerQueries.Get();
+#if GPU_VULKAN_QUERY_NEW
+        for (int32 i = 0; i < _activeTimerQueries.Count(); i++)
         {
-            _queriesInProgress.Get()[i]->Interrupt(_activeCmdBuffer);
+            GPUQueryVulkan query;
+            query.Raw = queries[i];
+
+            // End active query to get time from start until submission
+            auto pool = _device->QueryPools[query.PoolIndex];
+            vkCmdWriteTimestamp(_activeCmdBuffer->GetHandle(), VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, pool->GetHandle(), query.SecondQueryIndex);
+            pool->MarkQueryAsStarted(query.SecondQueryIndex);
+            // TODO: somehow handle ending this query properly by stopping split query instead
+            //_context->EndQuery(query.Raw);
+
+            // TODO: reimplement timer queries pause/resume to be more exact?
         }
+#else
+        for (int32 i = 0; i < _activeTimerQueries.Count(); i++)
+            queries->Interrupt(_activeCmdBuffer);
+#endif
 #endif
 
         _activeCmdBuffer->End();
@@ -317,27 +334,37 @@ void CmdBufferManagerVulkan::PrepareForNewActiveCommandBuffer()
 
     _activeCmdBuffer->Begin();
 
-#if VULKAN_USE_QUERIES
-    // Resume any paused queries with the new command buffer
-    for (int32 i = 0; i < _queriesInProgress.Count(); i++)
+#if VULKAN_USE_TIMER_QUERIES && GPU_VULKAN_PAUSE_QUERIES
+    // Resume any paused timer queries with the new command buffer
+    auto queries = _activeTimerQueries.Get();
+#if GPU_VULKAN_QUERY_NEW
+    for (int32 i = 0; i < _activeTimerQueries.Count(); i++)
     {
-        _queriesInProgress.Get()[i]->Resume(_activeCmdBuffer);
+        GPUQueryVulkan query;
+        query.Raw = queries[i];
+        //_activeTimerQueries.Get()[i]->Resume(_activeCmdBuffer);
+    }
+#else
+    for (int32 i = 0; i < _activeTimerQueries.Count(); i++)
+    {
+        queries->Resume(_activeCmdBuffer);
     }
 #endif
+#endif
 }
 
-void CmdBufferManagerVulkan::OnQueryBegin(GPUTimerQueryVulkan* query)
+#if GPU_VULKAN_QUERY_NEW && GPU_VULKAN_PAUSE_QUERIES
+
+void CmdBufferManagerVulkan::OnTimerQueryBegin(QueryType query)
 {
-#if VULKAN_USE_QUERIES
-    _queriesInProgress.Add(query);
-#endif
+    _activeTimerQueries.Add(query);
 }
 
-void CmdBufferManagerVulkan::OnQueryEnd(GPUTimerQueryVulkan* query)
+void CmdBufferManagerVulkan::OnTimerQueryEnd(QueryType query)
 {
-#if VULKAN_USE_QUERIES
-    _queriesInProgress.Remove(query);
-#endif
+    _activeTimerQueries.Remove(query);
 }
 
 #endif
+
+#endif

Source/Engine/GraphicsDevice/Vulkan/CmdBufferVulkan.h

@@ -168,10 +168,18 @@ class CmdBufferManagerVulkan
 {
 private:
     GPUDeviceVulkan* _device;
+    GPUContextVulkan* _context;
     CmdBufferPoolVulkan _pool;
     QueueVulkan* _queue;
     CmdBufferVulkan* _activeCmdBuffer;
-    Array<GPUTimerQueryVulkan*> _queriesInProgress;
+#if VULKAN_USE_TIMER_QUERIES && GPU_VULKAN_PAUSE_QUERIES
+#if GPU_VULKAN_QUERY_NEW
+    typedef uint64 QueryType;
+#else
+    typedef GPUTimerQueryVulkan* QueryType;
+#endif
+    Array<QueryType> _activeTimerQueries;
+#endif
 
 public:
     CmdBufferManagerVulkan(GPUDeviceVulkan* device, GPUContextVulkan* context);
@@ -192,11 +200,6 @@ public:
         return _activeCmdBuffer != nullptr;
     }
 
-    FORCE_INLINE bool HasQueriesInProgress() const
-    {
-        return _queriesInProgress.Count() != 0;
-    }
-
     FORCE_INLINE CmdBufferVulkan* GetCmdBuffer()
     {
         if (!_activeCmdBuffer)
@@ -207,14 +210,16 @@ public:
 public:
     void SubmitActiveCmdBuffer(SemaphoreVulkan* signalSemaphore = nullptr);
     void WaitForCmdBuffer(CmdBufferVulkan* cmdBuffer, float timeInSecondsToWait = 1.0f);
-    void RefreshFenceStatus(CmdBufferVulkan* skipCmdBuffer = nullptr)
+    void RefreshFenceStatus(const CmdBufferVulkan* skipCmdBuffer = nullptr)
     {
         _pool.RefreshFenceStatus(skipCmdBuffer);
     }
     void PrepareForNewActiveCommandBuffer();
 
-    void OnQueryBegin(GPUTimerQueryVulkan* query);
-    void OnQueryEnd(GPUTimerQueryVulkan* query);
+#if VULKAN_USE_TIMER_QUERIES && GPU_VULKAN_PAUSE_QUERIES
+    void OnTimerQueryBegin(QueryType query);
+    void OnTimerQueryEnd(QueryType query);
+#endif
 };
 
 #endif

Source/Engine/GraphicsDevice/Vulkan/Config.h

@@ -45,8 +45,14 @@
 #endif
 #endif
 
-#ifndef VULKAN_USE_QUERIES
-#define VULKAN_USE_QUERIES 1
+#ifndef VULKAN_USE_TIMER_QUERIES
+#define VULKAN_USE_TIMER_QUERIES 1
 #endif
 
+// Toggles GPUTimerQueryVulkan to use BeginQuery/EndQuery via GPuContext rather than old custom implementation
+#define GPU_VULKAN_QUERY_NEW 1
+
+// Toggles pausing and resuming all GPU timer queries when command buffer is being flushed (for more exact timings)
+#define GPU_VULKAN_PAUSE_QUERIES 0
+
 #endif

Source/Engine/GraphicsDevice/Vulkan/GPUContextVulkan.cpp

@@ -1300,6 +1300,72 @@ void GPUContextVulkan::DrawIndexedInstancedIndirect(GPUBuffer* bufferForArgs, ui
     RENDER_STAT_DRAW_CALL(0, 0);
 }
 
+uint64 GPUContextVulkan::BeginQuery(GPUQueryType type)
+{
+    // Check if timer queries are supported
+    if (type == GPUQueryType::Timer && _device->PhysicalDeviceLimits.timestampComputeAndGraphics != VK_TRUE)
+        return 0;
+
+    // Allocate query
+    auto poolIndex = _device->GetOrCreateQueryPool(type);
+    auto pool = _device->QueryPools[poolIndex];
+    uint32 index = 0;
+    const auto cmdBuffer = _cmdBufferManager->GetCmdBuffer();
+    if (!pool->AcquireQuery(cmdBuffer, index))
+        return 0;
+    GPUQueryVulkan query;
+    query.PoolIndex = (uint16)poolIndex;
+    query.QueryIndex = (uint16)index;
+    query.SecondQueryIndex = 0;
+    query.Dummy = 1; // Ensure Raw is never 0, even for the first query
+
+    // Begin query
+    switch (type)
+    {
+    case GPUQueryType::Timer:
+        // Timer queries need 2 slots (begin + end)
+        pool->AcquireQuery(cmdBuffer, index);
+        query.SecondQueryIndex = (uint16)index;
+
+        vkCmdWriteTimestamp(cmdBuffer->GetHandle(), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, pool->GetHandle(), query.QueryIndex);
+#if GPU_VULKAN_PAUSE_QUERIES
+        _cmdBufferManager->OnTimerQueryBegin(query.Raw);
+#endif
+        break;
+    case GPUQueryType::Occlusion:
+        vkCmdBeginQuery(cmdBuffer->GetHandle(), pool->GetHandle(), query.QueryIndex, 0);
+        break;
+    }
+    pool->MarkQueryAsStarted(query.QueryIndex);
+
+    return query.Raw;
+}
+
+void GPUContextVulkan::EndQuery(uint64 queryID)
+{
+    if (!queryID)
+        return;
+    GPUQueryVulkan query;
+    query.Raw = queryID;
+    auto pool = _device->QueryPools[query.PoolIndex];
+
+    // End query
+    const auto cmdBuffer = _cmdBufferManager->GetCmdBuffer();
+    switch (pool->Type)
+    {
+    case GPUQueryType::Timer:
+        vkCmdWriteTimestamp(cmdBuffer->GetHandle(), VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, pool->GetHandle(), query.SecondQueryIndex);
+        pool->MarkQueryAsStarted(query.SecondQueryIndex);
+#if GPU_VULKAN_PAUSE_QUERIES
+        _cmdBufferManager->OnTimerQueryEnd(query.Raw);
+#endif
+        break;
+    case GPUQueryType::Occlusion:
+        vkCmdEndQuery(cmdBuffer->GetHandle(), pool->GetHandle(), query.QueryIndex);
+        break;
+    }
+}
+
 void GPUContextVulkan::SetViewport(const Viewport& viewport)
 {
     vkCmdSetViewport(_cmdBufferManager->GetCmdBuffer()->GetHandle(), 0, 1, (VkViewport*)&viewport);

Source/Engine/GraphicsDevice/Vulkan/GPUContextVulkan.h

@@ -189,6 +189,8 @@ public:
     void DrawIndexedInstanced(uint32 indicesCount, uint32 instanceCount, int32 startInstance, int32 startVertex, int32 startIndex) override;
     void DrawInstancedIndirect(GPUBuffer* bufferForArgs, uint32 offsetForArgs) override;
     void DrawIndexedInstancedIndirect(GPUBuffer* bufferForArgs, uint32 offsetForArgs) override;
+    uint64 BeginQuery(GPUQueryType type) override;
+    void EndQuery(uint64 queryID) override;
     void SetViewport(const Viewport& viewport) override;
     void SetScissor(const Rectangle& scissorRect) override;
     GPUPipelineState* GetState() const override;

Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.cpp

@@ -627,14 +627,14 @@ RenderPassVulkan::~RenderPassVulkan()
     Device->DeferredDeletionQueue.EnqueueResource(DeferredDeletionQueueVulkan::Type::RenderPass, Handle);
 }
 
-QueryPoolVulkan::QueryPoolVulkan(GPUDeviceVulkan* device, int32 capacity, VkQueryType type)
+QueryPoolVulkan::QueryPoolVulkan(GPUDeviceVulkan* device, int32 capacity, GPUQueryType type)
     : _device(device)
     , _handle(VK_NULL_HANDLE)
-    , _type(type)
+    , Type(type)
 {
     VkQueryPoolCreateInfo createInfo;
     RenderToolsVulkan::ZeroStruct(createInfo, VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO);
-    createInfo.queryType = type;
+    createInfo.queryType = type == GPUQueryType::Occlusion ? VK_QUERY_TYPE_OCCLUSION : VK_QUERY_TYPE_TIMESTAMP;
     createInfo.queryCount = capacity;
     VALIDATE_VULKAN_RESULT(vkCreateQueryPool(device->Device, &createInfo, nullptr, &_handle));
 
@@ -667,7 +667,7 @@ void QueryPoolVulkan::Reset(CmdBufferVulkan* cmdBuffer)
 
 #endif
 
-BufferedQueryPoolVulkan::BufferedQueryPoolVulkan(GPUDeviceVulkan* device, int32 capacity, VkQueryType type)
+BufferedQueryPoolVulkan::BufferedQueryPoolVulkan(GPUDeviceVulkan* device, int32 capacity, GPUQueryType type)
     : QueryPoolVulkan(device, capacity, type)
     , _lastBeginIndex(0)
 {
@@ -720,6 +720,16 @@ void BufferedQueryPoolVulkan::ReleaseQuery(uint32 queryIndex)
             _lastBeginIndex = (uint32)queryIndex;
         }
     }
+    if (_usedQueryBits[word] == 0)
+    {
+        // Check if pool got empty and reset the pointer back to start
+        for (int32 wordIndex = 0; wordIndex < _usedQueryBits.Count(); wordIndex++)
+        {
+            if (_usedQueryBits[wordIndex])
+                return;
+        }
+        _lastBeginIndex = 0;
+    }
 }
 
 void BufferedQueryPoolVulkan::MarkQueryAsStarted(uint32 queryIndex)
@@ -729,7 +739,7 @@ void BufferedQueryPoolVulkan::MarkQueryAsStarted(uint32 queryIndex)
     _startedQueryBits[word] = _startedQueryBits[word] | bit;
 }
 
-bool BufferedQueryPoolVulkan::GetResults(GPUContextVulkan* context, uint32 index, uint64& result)
+bool BufferedQueryPoolVulkan::GetResults(uint32 index, uint64& result)
 {
     const uint64 bit = (uint64)(index % 64);
     const uint64 bitMask = (uint64)1 << bit;
@@ -1228,22 +1238,20 @@ GPUDeviceVulkan::~GPUDeviceVulkan()
     GPUDeviceVulkan::Dispose();
 }
 
-BufferedQueryPoolVulkan* GPUDeviceVulkan::FindAvailableQueryPool(VkQueryType queryType)
+int32 GPUDeviceVulkan::GetOrCreateQueryPool(GPUQueryType type)
 {
-    auto& pools = queryType == VK_QUERY_TYPE_OCCLUSION ? OcclusionQueryPools : TimestampQueryPools;
-
-    // Try to use pool with available space inside
-    for (int32 i = 0; i < pools.Count(); i++)
+    auto pools = QueryPools.Get();
+    for (int32 i = 0; i < QueryPools.Count(); i++)
     {
-        auto pool = pools.Get()[i];
-        if (pool->HasRoom())
-            return pool;
+        auto pool = pools[i];
+        if (pool->Type == type && pool->HasRoom())
+            return i;
     }
 
-    // Create new pool
-    const auto pool = New<BufferedQueryPoolVulkan>(this, queryType == VK_QUERY_TYPE_OCCLUSION ? 4096 : 1024, queryType);
-    pools.Add(pool);
-    return pool;
+    PROFILE_CPU_NAMED("Create Create Pool");
+    auto pool = New<BufferedQueryPoolVulkan>(this, type == GPUQueryType::Occlusion ? 4096 : 1024, type);
+    QueryPools.Add(pool);
+    return QueryPools.Count() - 1;
 }
 
 RenderPassVulkan* GPUDeviceVulkan::GetOrCreateRenderPass(RenderTargetLayoutVulkan& layout)
@@ -1752,6 +1760,10 @@ bool GPUDeviceVulkan::Init()
         limits.MaximumTexture3DSize = PhysicalDeviceLimits.maxImageDimension3D;
         limits.MaximumTextureCubeSize = PhysicalDeviceLimits.maxImageDimensionCube;
         limits.MaximumSamplerAnisotropy = PhysicalDeviceLimits.maxSamplerAnisotropy;
+        if (PhysicalDeviceLimits.timestampComputeAndGraphics != VK_TRUE)
+        {
+            LOG(Warning, "Timer Queries are unsupported by this device");
+        }
 
         for (int32 i = 0; i < static_cast<int32>(PixelFormat::MAX); i++)
         {
@@ -1982,6 +1994,16 @@ void GPUDeviceVulkan::DrawBegin()
     // Base
     GPUDevice::DrawBegin();
 
+    // Put back used queries to the pool
+    for (auto& query : QueriesToRelease)
+    {
+        auto pool = QueryPools[query.PoolIndex];
+        pool->ReleaseQuery(query.QueryIndex);
+        if (pool->Type == GPUQueryType::Timer)
+            pool->ReleaseQuery(query.SecondQueryIndex);
+    }
+    QueriesToRelease.Clear();
+
     // Flush resources
     DeferredDeletionQueue.ReleaseResources();
     DescriptorPoolsManager->GC();
@@ -2022,8 +2044,7 @@ void GPUDeviceVulkan::Dispose()
     _layouts.ClearDelete();
     HelperResources.Dispose();
     UploadBuffer.Dispose();
-    TimestampQueryPools.ClearDelete();
-    OcclusionQueryPools.ClearDelete();
+    QueryPools.ClearDelete();
     SAFE_DELETE_GPU_RESOURCE(UniformBufferUploader);
     Delete(DescriptorPoolsManager);
     SAFE_DELETE(MainContext);
@@ -2084,6 +2105,61 @@ void GPUDeviceVulkan::WaitForGPU()
     }
 }
 
+bool GPUDeviceVulkan::GetQueryResult(uint64 queryID, uint64& result, bool wait)
+{
+    if (!queryID)
+        return false;
+    GPUQueryVulkan query;
+    query.Raw = queryID;
+    auto pool = QueryPools[query.PoolIndex];
+
+RETRY:
+    bool hasData;
+    uint64 resultSecondary;
+    switch (pool->Type)
+    {
+    case GPUQueryType::Timer:
+        hasData = pool->GetResults(query.QueryIndex, result) && pool->GetResults(query.SecondQueryIndex, resultSecondary);
+#if VULKAN_USE_TIMER_QUERIES && GPU_VULKAN_PAUSE_QUERIES
+        if (hasData)
+        {
+            // Check if dependant queries have completed (timer queries can be split when active command buffer get submitted) 
+            // TODO: impl this
+        }
+#endif
+        if (hasData)
+        {
+            if (resultSecondary >= result)
+            {
+                // Convert GPU timestamps to nanoseconds and then to microseconds
+                double nanoseconds = double(resultSecondary - result) * double(PhysicalDeviceLimits.timestampPeriod);
+                result = (uint64)(nanoseconds * 0.001);
+            }
+            else
+                result = 0;
+        }
+        break;
+    case GPUQueryType::Occlusion:
+        hasData = pool->GetResults(query.QueryIndex, result);
+        break;
+    }
+
+    if (!hasData && wait)
+    {
+        // Wait until data is ready
+        Platform::Yield();
+        goto RETRY;
+    }
+
+    if (hasData)
+    {
+        // Auto-release query on the next frame
+        QueriesToRelease.Add(query);
+    }
+
+    return hasData;
+}
+
 GPUTexture* GPUDeviceVulkan::CreateTexture(const StringView& name)
 {
     PROFILE_MEM(GraphicsTextures);

Source/Engine/GraphicsDevice/Vulkan/GPUDeviceVulkan.h

@@ -28,6 +28,24 @@ class GPUDeviceVulkan;
 class UniformBufferUploaderVulkan;
 class DescriptorPoolsManagerVulkan;
 
+/// <summary>
+/// GPU query ID packed into 64-bits.
+/// </summary>
+struct GPUQueryVulkan
+{
+    union
+    {
+        struct
+        {
+            uint16 PoolIndex;
+            uint16 QueryIndex;
+            uint16 SecondQueryIndex;
+            uint16 Dummy;
+        };
+        uint64 Raw;
+    };
+};
+
 class SemaphoreVulkan
 {
 private:
@@ -261,16 +279,17 @@ protected:
     GPUDeviceVulkan* _device;
     VkQueryPool _handle;
 
-    const VkQueryType _type;
 #if VULKAN_RESET_QUERY_POOLS
     Array<Range> _resetRanges;
 #endif
 
 public:
-    QueryPoolVulkan(GPUDeviceVulkan* device, int32 capacity, VkQueryType type);
+    QueryPoolVulkan(GPUDeviceVulkan* device, int32 capacity, GPUQueryType type);
     ~QueryPoolVulkan();
 
 public:
+    const GPUQueryType Type;
+
     inline VkQueryPool GetHandle() const
     {
         return _handle;
@@ -294,11 +313,11 @@ private:
     int32 _lastBeginIndex;
 
 public:
-    BufferedQueryPoolVulkan(GPUDeviceVulkan* device, int32 capacity, VkQueryType type);
+    BufferedQueryPoolVulkan(GPUDeviceVulkan* device, int32 capacity, GPUQueryType type);
     bool AcquireQuery(CmdBufferVulkan* cmdBuffer, uint32& resultIndex);
     void ReleaseQuery(uint32 queryIndex);
     void MarkQueryAsStarted(uint32 queryIndex);
-    bool GetResults(GPUContextVulkan* context, uint32 index, uint64& result);
+    bool GetResults(uint32 index, uint64& result);
     bool HasRoom() const;
 };
 
@@ -498,14 +517,13 @@ public:
     VkPhysicalDeviceFeatures PhysicalDeviceFeatures;
     VkPhysicalDeviceVulkan12Features PhysicalDeviceFeatures12;
 
-    Array<BufferedQueryPoolVulkan*> TimestampQueryPools;
-    Array<BufferedQueryPoolVulkan*> OcclusionQueryPools;
-
+    Array<BufferedQueryPoolVulkan*> QueryPools;
+    Array<GPUQueryVulkan> QueriesToRelease;
 #if VULKAN_RESET_QUERY_POOLS
     Array<QueryPoolVulkan*> QueriesToReset;
 #endif
 
-    BufferedQueryPoolVulkan* FindAvailableQueryPool(VkQueryType queryType);
+    int32 GetOrCreateQueryPool(GPUQueryType type);
     RenderPassVulkan* GetOrCreateRenderPass(RenderTargetLayoutVulkan& layout);
     FramebufferVulkan* GetOrCreateFramebuffer(FramebufferVulkan::Key& key, VkExtent2D& extent, uint32 layers);
     PipelineLayoutVulkan* GetOrCreateLayout(DescriptorSetLayoutInfoVulkan& key);
@@ -553,6 +571,7 @@ public:
     void DrawBegin() override;
     void Dispose() override;
     void WaitForGPU() override;
+    bool GetQueryResult(uint64 queryID, uint64& result, bool wait = false) override;
     GPUTexture* CreateTexture(const StringView& name) override;
     GPUShader* CreateShader(const StringView& name) override;
     GPUPipelineState* CreatePipelineState() override;

Source/Engine/GraphicsDevice/Vulkan/GPUTimerQueryVulkan.cpp

@@ -11,6 +11,78 @@ GPUTimerQueryVulkan::GPUTimerQueryVulkan(GPUDeviceVulkan* device)
 {
 }
 
+#if !VULKAN_USE_TIMER_QUERIES
+
+void GPUTimerQueryVulkan::OnReleaseGPU()
+{
+}
+
+void GPUTimerQueryVulkan::Begin()
+{
+}
+
+void GPUTimerQueryVulkan::End()
+{
+}
+
+bool GPUTimerQueryVulkan::HasResult()
+{
+    return true;
+}
+
+float GPUTimerQueryVulkan::GetResult()
+{
+    return 0;
+}
+
+#elif GPU_VULKAN_QUERY_NEW
+
+void GPUTimerQueryVulkan::OnReleaseGPU()
+{
+    _hasResult = false;
+    _endCalled = false;
+    _timeDelta = 0.0f;
+}
+
+void GPUTimerQueryVulkan::Begin()
+{
+    const auto context = _device->GetMainContext();
+    _query = context->BeginQuery(GPUQueryType::Timer);
+    _hasResult = false;
+    _endCalled = false;
+}
+
+void GPUTimerQueryVulkan::End()
+{
+    if (_endCalled)
+        return;
+    const auto context = _device->GetMainContext();
+    context->EndQuery(_query);
+    _endCalled = true;
+}
+
+bool GPUTimerQueryVulkan::HasResult()
+{
+    if (!_endCalled)
+        return false;
+    if (_hasResult)
+        return true;
+    uint64 result;
+    return _device->GetQueryResult(_query, result, false);
+}
+
+float GPUTimerQueryVulkan::GetResult()
+{
+    if (_hasResult)
+        return _timeDelta;
+    uint64 result;
+    _timeDelta = _device->GetQueryResult(_query, result, true) ? (float)((double)result / 1000.0) : 0.0f;
+    _hasResult = true;
+    return _timeDelta;
+}
+
+#else
+
 void GPUTimerQueryVulkan::Interrupt(CmdBufferVulkan* cmdBuffer)
 {
     if (!_interrupted)
@@ -38,8 +110,7 @@ bool GPUTimerQueryVulkan::GetResult(Query& query)
 {
     if (query.Pool)
     {
-        const auto context = (GPUContextVulkan*)_device->GetMainContext();
-        if (query.Pool->GetResults(context, query.Index, query.Result))
+        if (query.Pool->GetResults(query.Index, query.Result))
         {
             // Release query
             query.Pool->ReleaseQuery(query.Index);
@@ -58,7 +129,7 @@ bool GPUTimerQueryVulkan::GetResult(Query& query)
 
 void GPUTimerQueryVulkan::WriteTimestamp(CmdBufferVulkan* cmdBuffer, Query& query, VkPipelineStageFlagBits stage) const
 {
-    auto pool = _device->FindAvailableQueryPool(VK_QUERY_TYPE_TIMESTAMP);
+    auto pool = _device->QueryPools[_device->GetOrCreateQueryPool(GPUQueryType::Timer)];
     uint32 index;
     if (pool->AcquireQuery(cmdBuffer, index))
     {
@@ -76,7 +147,6 @@ void GPUTimerQueryVulkan::WriteTimestamp(CmdBufferVulkan* cmdBuffer, Query& quer
 
 bool GPUTimerQueryVulkan::TryGetResult()
 {
-#if VULKAN_USE_QUERIES
     // Try get queries value (if not already)
     for (int32 i = 0; i < _queries.Count(); i++)
     {
@@ -115,20 +185,12 @@ bool GPUTimerQueryVulkan::TryGetResult()
             e.End.Pool->ReleaseQuery(e.End.Index);
     }
     _queries.Clear();
-#else
-    _timeDelta = 0.0f;
-    _hasResult = true;
-#endif
     return true;
 }
 
 bool GPUTimerQueryVulkan::UseQueries()
 {
-#if VULKAN_USE_QUERIES
     return _device->PhysicalDeviceLimits.timestampComputeAndGraphics == VK_TRUE;
-#else
-    return false;
-#endif
 }
 
 void GPUTimerQueryVulkan::OnReleaseGPU()
@@ -150,7 +212,6 @@ void GPUTimerQueryVulkan::OnReleaseGPU()
 
 void GPUTimerQueryVulkan::Begin()
 {
-#if VULKAN_USE_QUERIES
     if (UseQueries())
     {
         const auto context = (GPUContextVulkan*)_device->GetMainContext();
@@ -162,12 +223,11 @@ void GPUTimerQueryVulkan::Begin()
         _queryIndex = 0;
         _interrupted = false;
         WriteTimestamp(cmdBuffer, e.Begin, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
-        context->GetCmdBufferManager()->OnQueryBegin(this);
+        context->GetCmdBufferManager()->OnTimerQueryBegin(this);
 
         ASSERT(_queries.IsEmpty());
         _queries.Add(e);
     }
-#endif
 
     _hasResult = false;
     _endCalled = false;
@@ -178,7 +238,6 @@ void GPUTimerQueryVulkan::End()
     if (_endCalled)
         return;
 
-#if VULKAN_USE_QUERIES
     if (UseQueries())
     {
         const auto context = (GPUContextVulkan*)_device->GetMainContext();
@@ -188,9 +247,8 @@ void GPUTimerQueryVulkan::End()
         {
             WriteTimestamp(cmdBuffer, _queries[_queryIndex].End, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
         }
-        context->GetCmdBufferManager()->OnQueryEnd(this);
+        context->GetCmdBufferManager()->OnTimerQueryEnd(this);
     }
-#endif
 
     _endCalled = true;
 }
@@ -213,3 +271,5 @@ float GPUTimerQueryVulkan::GetResult()
 }
 
 #endif
+
+#endif

Source/Engine/GraphicsDevice/Vulkan/GPUTimerQueryVulkan.h

@@ -13,6 +13,13 @@
 class GPUTimerQueryVulkan : public GPUResourceVulkan<GPUTimerQuery>
 {
 private:
+#if !VULKAN_USE_TIMER_QUERIES
+#elif GPU_VULKAN_QUERY_NEW
+    bool _hasResult = false;
+    bool _endCalled = false;
+    float _timeDelta = 0.0f;
+    uint64 _query = 0;
+#else
     struct Query
     {
         BufferedQueryPoolVulkan* Pool;
@@ -32,6 +39,7 @@ private:
     float _timeDelta = 0.0f;
     int32 _queryIndex;
     Array<QueryPair, InlinedAllocation<8>> _queries;
+#endif
 
 public:
     /// <summary>
@@ -40,6 +48,7 @@ public:
     /// <param name="device">The graphics device.</param>
     GPUTimerQueryVulkan(GPUDeviceVulkan* device);
 
+#if !GPU_VULKAN_QUERY_NEW
 public:
     /// <summary>
     /// Interrupts an in-progress query, allowing the command buffer to submitted. Interrupted queries must be resumed using Resume().
@@ -58,6 +67,7 @@ private:
     void WriteTimestamp(CmdBufferVulkan* cmdBuffer, Query& query, VkPipelineStageFlagBits stage) const;
     bool TryGetResult();
     bool UseQueries();
+#endif
 
 public:
     // [GPUTimerQuery]

Source/Engine/GraphicsDevice/Vulkan/Mac/MacVulkanPlatform.h

@@ -9,7 +9,7 @@
 #define VULKAN_BACK_BUFFERS_COUNT 3
 
 // General/Validation Error:0 VK_ERROR_INITIALIZATION_FAILED: Could not create MTLCounterSampleBuffer for query pool of type VK_QUERY_TYPE_TIMESTAMP. Reverting to emulated behavior. (Error code 0): Cannot allocate sample buffer
-#define VULKAN_USE_QUERIES 0
+#define VULKAN_USE_TIMER_QUERIES 0
 
 /// <summary>
 /// The implementation for the Vulkan API support for Mac platform.

Source/Engine/GraphicsDevice/Vulkan/iOS/iOSVulkanPlatform.h

@@ -9,7 +9,7 @@
 #define VULKAN_BACK_BUFFERS_COUNT 3
 
 // General/Validation Error:0 VK_ERROR_INITIALIZATION_FAILED: Could not create MTLCounterSampleBuffer for query pool of type VK_QUERY_TYPE_TIMESTAMP. Reverting to emulated behavior. (Error code 0): Cannot allocate sample buffer
-#define VULKAN_USE_QUERIES 0
+#define VULKAN_USE_TIMER_QUERIES 0
 
 /// <summary>
 /// The implementation for the Vulkan API support for iOS platform.