Add stencil support in PSO on DX11

Source/Engine/GraphicsDevice/DirectX/DX11/GPUContextDX11.cpp

@@ -559,15 +559,12 @@ void GPUContextDX11::SetState(GPUPipelineState* state)
 #endif
         GPUShaderProgramPSDX11* ps = nullptr;
         D3D11_PRIMITIVE_TOPOLOGY primitiveTopology = D3D11_PRIMITIVE_TOPOLOGY_UNDEFINED;
-
         if (state)
         {
             ASSERT(_currentState->IsValid());
-
             blendState = _currentState->BlendState;
             rasterizerState = _device->RasterizerStates[_currentState->RasterizerStateIndex];
-            depthStencilState = _device->DepthStencilStates[_currentState->DepthStencilStateIndex];
+            depthStencilState = _currentState->DepthStencilState;
-
             ASSERT(_currentState->VS != nullptr);
             vs = _currentState->VS;
 #if GPU_ALLOW_TESSELLATION_SHADERS
@@ -578,7 +575,6 @@ void GPUContextDX11::SetState(GPUPipelineState* state)
             gs = _currentState->GS;
 #endif
             ps = _currentState->PS;
-
             primitiveTopology = _currentState->PrimitiveTopology;
         }
 

Source/Engine/GraphicsDevice/DirectX/DX11/GPUDeviceDX11.cpp

@@ -24,6 +24,56 @@
 #define DX11_FORCE_USE_DX10 0
 #define DX11_FORCE_USE_DX10_1 0
 
+static D3D11_COMPARISON_FUNC ToDX11(ComparisonFunc value)
+{
+    switch (value)
+    {
+    case ComparisonFunc::Never:
+        return D3D11_COMPARISON_NEVER;
+    case ComparisonFunc::Less:
+        return D3D11_COMPARISON_LESS;
+    case ComparisonFunc::Equal:
+        return D3D11_COMPARISON_EQUAL;
+    case ComparisonFunc::LessEqual:
+        return D3D11_COMPARISON_LESS_EQUAL;
+    case ComparisonFunc::Greater:
+        return D3D11_COMPARISON_GREATER;
+    case ComparisonFunc::NotEqual:
+        return D3D11_COMPARISON_NOT_EQUAL;
+    case ComparisonFunc::GreaterEqual:
+        return D3D11_COMPARISON_GREATER_EQUAL;
+    case ComparisonFunc::Always:
+        return D3D11_COMPARISON_ALWAYS;
+    default:
+        return (D3D11_COMPARISON_FUNC)-1;
+    }
+}
+
+static D3D11_STENCIL_OP ToDX11(StencilOperation value)
+{
+    switch (value)
+    {
+    case StencilOperation::Keep:
+        return D3D11_STENCIL_OP_KEEP;
+    case StencilOperation::Zero:
+        return D3D11_STENCIL_OP_ZERO;
+    case StencilOperation::Replace:
+        return D3D11_STENCIL_OP_REPLACE;
+    case StencilOperation::IncrementSaturated:
+        return D3D11_STENCIL_OP_INCR_SAT;
+    case StencilOperation::DecrementSaturated:
+        return D3D11_STENCIL_OP_DECR_SAT;
+    case StencilOperation::Invert:
+        return D3D11_STENCIL_OP_INVERT;
+    case StencilOperation::Increment:
+        return D3D11_STENCIL_OP_INCR;
+    case StencilOperation::Decrement:
+        return D3D11_STENCIL_OP_DECR;
+    default:
+        return (D3D11_STENCIL_OP)-1;
+    }
+}
+
 static bool TryCreateDevice(IDXGIAdapter* adapter, D3D_FEATURE_LEVEL maxFeatureLevel, D3D_FEATURE_LEVEL* featureLevel)
 {
     ID3D11Device* device = nullptr;
@@ -251,22 +301,66 @@ GPUDeviceDX11::GPUDeviceDX11(IDXGIFactory* dxgiFactory, GPUAdapterDX* adapter)
     , _factoryDXGI(dxgiFactory)
 {
     Platform::MemoryClear(RasterizerStates, sizeof(RasterizerStates));
-    Platform::MemoryClear(DepthStencilStates, sizeof(DepthStencilStates));
+}
+
+ID3D11DepthStencilState* GPUDeviceDX11::GetDepthStencilState(const void* descriptionPtr)
+{
+    const GPUPipelineState::Description& description = *(const GPUPipelineState::Description*)descriptionPtr;
+    DepthStencilMode key;
+    Platform::MemoryClear(&key, sizeof(key)); // Ensure to clear any padding bytes for raw memory compare/hashing
+    key.DepthEnable = description.DepthEnable ? 1 : 0;
+    key.DepthWriteEnable = description.DepthWriteEnable ? 1 : 0;
+    key.DepthClipEnable = description.DepthClipEnable ? 1 : 0;
+    key.StencilEnable = description.StencilEnable ? 1 : 0;
+    key.StencilReadMask = description.StencilReadMask;
+    key.StencilWriteMask = description.StencilWriteMask;
+    key.DepthFunc = description.DepthFunc;
+    key.StencilFunc = description.StencilFunc;
+    key.StencilFailOp = description.StencilFailOp;
+    key.StencilDepthFailOp = description.StencilDepthFailOp;
+    key.StencilPassOp = description.StencilPassOp;
+
+    // Use lookup
+    ID3D11DepthStencilState* state = nullptr;
+    if (DepthStencilStates.TryGet(key, state))
+        return state;
+    
+    // Try again but with lock to prevent race condition with double-adding the same thing
+    ScopeLock lock(StatesWriteLocker);
+    if (DepthStencilStates.TryGet(key, state))
+        return state;
+
+    // Prepare description
+    D3D11_DEPTH_STENCIL_DESC desc;
+	desc.DepthEnable = !!description.DepthEnable;
+	desc.DepthWriteMask = description.DepthWriteEnable ? D3D11_DEPTH_WRITE_MASK_ALL : D3D11_DEPTH_WRITE_MASK_ZERO;
+	desc.DepthFunc = ToDX11(description.DepthFunc);
+    desc.StencilEnable = !!description.StencilEnable;
+    desc.StencilReadMask = description.StencilReadMask;
+    desc.StencilWriteMask = description.StencilWriteMask;
+    desc.FrontFace.StencilFailOp = ToDX11(description.StencilFailOp);
+    desc.FrontFace.StencilDepthFailOp = ToDX11(description.StencilDepthFailOp);
+    desc.FrontFace.StencilPassOp = ToDX11(description.StencilPassOp);
+    desc.FrontFace.StencilFunc = ToDX11(description.StencilFunc);
+    desc.BackFace = desc.FrontFace;
+
+    // Create object and cache it
+    VALIDATE_DIRECTX_CALL(_device->CreateDepthStencilState(&desc, &state));
+    DepthStencilStates.Add(key, state);
+    return state;
 }
 
 ID3D11BlendState* GPUDeviceDX11::GetBlendState(const BlendingMode& blending)
 {
     // Use lookup
-    ID3D11BlendState* blendState = nullptr;
+    ID3D11BlendState* state = nullptr;
-    if (BlendStates.TryGet(blending, blendState))
+    if (BlendStates.TryGet(blending, state))
-        return blendState;
+        return state;
     
-    // Make it safe
+    // Try again but with lock to prevent race condition with double-adding the same thing
-    ScopeLock lock(BlendStatesWriteLocker);
+    ScopeLock lock(StatesWriteLocker);
-
+    if (BlendStates.TryGet(blending, state))
-    // Try again to prevent race condition with double-adding the same thing
+        return state;
-    if (BlendStates.TryGet(blending, blendState))
-        return blendState;
 
     // Prepare description
     D3D11_BLEND_DESC desc;
@@ -285,13 +379,10 @@ ID3D11BlendState* GPUDeviceDX11::GetBlendState(const BlendingMode& blending)
         desc.RenderTarget[i] = desc.RenderTarget[0];
 #endif
 
-    // Create object
+    // Create object and cache it
-    VALIDATE_DIRECTX_CALL(_device->CreateBlendState(&desc, &blendState));
+    VALIDATE_DIRECTX_CALL(_device->CreateBlendState(&desc, &state));
-
+    BlendStates.Add(blending, state);
-    // Cache blend state
+    return state;
-    BlendStates.Add(blending, blendState);
-
-    return blendState;
 }
 
 bool GPUDeviceDX11::Init()
@@ -560,31 +651,6 @@ bool GPUDeviceDX11::Init()
 #undef CREATE_RASTERIZER_STATE
     }
 
-    // Depth Stencil States
-    {
-        D3D11_DEPTH_STENCIL_DESC dsDesc;
-        dsDesc.StencilEnable = FALSE;
-        dsDesc.StencilReadMask = D3D11_DEFAULT_STENCIL_READ_MASK;
-        dsDesc.StencilWriteMask = D3D11_DEFAULT_STENCIL_WRITE_MASK;
-        const D3D11_DEPTH_STENCILOP_DESC defaultStencilOp = { D3D11_STENCIL_OP_KEEP, D3D11_STENCIL_OP_KEEP, D3D11_STENCIL_OP_KEEP, D3D11_COMPARISON_ALWAYS };
-        dsDesc.FrontFace = defaultStencilOp;
-        dsDesc.BackFace = defaultStencilOp;
-        int32 index;
-#define CREATE_DEPTH_STENCIL_STATE(depthEnable, depthWrite) \
-			dsDesc.DepthEnable = depthEnable; \
-			dsDesc.DepthWriteMask = depthWrite ? D3D11_DEPTH_WRITE_MASK_ALL : D3D11_DEPTH_WRITE_MASK_ZERO; \
-			for(int32 depthFunc = 1; depthFunc <= 8; depthFunc++) { \
-			dsDesc.DepthFunc = (D3D11_COMPARISON_FUNC)depthFunc; \
-			index = (int32)depthFunc + (depthEnable ? 0 : 9) + (depthWrite ? 0 : 18); \
-			HRESULT result = _device->CreateDepthStencilState(&dsDesc, &DepthStencilStates[index]); \
-			LOG_DIRECTX_RESULT_WITH_RETURN(result, true); }
-        CREATE_DEPTH_STENCIL_STATE(false, false);
-        CREATE_DEPTH_STENCIL_STATE(false, true);
-        CREATE_DEPTH_STENCIL_STATE(true, true);
-        CREATE_DEPTH_STENCIL_STATE(true, false);
-#undef CREATE_DEPTH_STENCIL_STATE
-    }
-
     _state = DeviceState::Ready;
     return GPUDeviceDX::Init();
 }
@@ -624,20 +690,15 @@ void GPUDeviceDX11::Dispose()
     SAFE_RELEASE(_samplerPointWrap);
     SAFE_RELEASE(_samplerShadow);
     SAFE_RELEASE(_samplerShadowLinear);
-    //
     for (auto i = BlendStates.Begin(); i.IsNotEnd(); ++i)
-    {
         i->Value->Release();
-    }
+    for (auto i = DepthStencilStates.Begin(); i.IsNotEnd(); ++i)
+        i->Value->Release();
     BlendStates.Clear();
     for (uint32 i = 0; i < ARRAY_COUNT(RasterizerStates); i++)
     {
         SAFE_RELEASE(RasterizerStates[i]);
     }
-    for (uint32 i = 0; i < ARRAY_COUNT(DepthStencilStates); i++)
-    {
-        SAFE_RELEASE(DepthStencilStates[i]);
-    }
 
     // Clear DirectX stuff
     SAFE_DELETE(_mainContext);

Source/Engine/GraphicsDevice/DirectX/DX11/GPUDeviceDX11.h

@@ -11,6 +11,7 @@
 #if GRAPHICS_API_DIRECTX11
 
 class Engine;
+enum class StencilOperation : byte;
 class GPUContextDX11;
 class GPUSwapChainDX11;
 
@@ -23,6 +24,34 @@ class GPUDeviceDX11 : public GPUDeviceDX
     friend GPUSwapChainDX11;
 private:
 
+    struct DepthStencilMode
+    {
+        int8 DepthEnable : 1;
+        int8 DepthWriteEnable : 1;
+        int8 DepthClipEnable : 1;
+        int8 StencilEnable : 1;
+        uint8 StencilReadMask;
+        uint8 StencilWriteMask;
+        ComparisonFunc DepthFunc;
+        ComparisonFunc StencilFunc;
+        StencilOperation StencilFailOp;
+        StencilOperation StencilDepthFailOp;
+        StencilOperation StencilPassOp;
+
+        bool operator==(const DepthStencilMode& other) const
+        {
+            return Platform::MemoryCompare(this, &other, sizeof(DepthStencilMode));
+        }
+
+        friend uint32 GetHash(const DepthStencilMode& key)
+        {
+            uint32 hash = 0;
+            for (int32 i = 0; i < sizeof(DepthStencilMode) / 4; i++)
+                CombineHash(hash, ((uint32*)&key)[i]);
+            return hash;
+        }
+    };
+
     // Private Stuff
     ID3D11Device* _device = nullptr;
     ID3D11DeviceContext* _imContext = nullptr;
@@ -40,10 +69,10 @@ private:
     ID3D11SamplerState* _samplerShadowLinear = nullptr;
 
     // Shared data for pipeline states
-    CriticalSection BlendStatesWriteLocker;
+    CriticalSection StatesWriteLocker;
     Dictionary<BlendingMode, ID3D11BlendState*> BlendStates;
+    Dictionary<DepthStencilMode, ID3D11DepthStencilState*> DepthStencilStates;
     ID3D11RasterizerState* RasterizerStates[3 * 2 * 2]; // Index =  CullMode[0-2] + Wireframe[0?3] + DepthClipEnable[0?6]
-    ID3D11DepthStencilState* DepthStencilStates[9 * 2 * 2]; // Index = ComparisonFunc[0-8] + DepthTestEnable[0?9] + DepthWriteEnable[0?18]
 
 public:
     static GPUDevice* Create();
@@ -75,6 +104,7 @@ public:
         return _mainContext;
     }
 
+    ID3D11DepthStencilState* GetDepthStencilState(const void* descriptionPtr);
     ID3D11BlendState* GetBlendState(const BlendingMode& blending);
 
 public:

Source/Engine/GraphicsDevice/DirectX/DX11/GPUPipelineStateDX11.cpp

@@ -6,6 +6,7 @@
 
 void GPUPipelineStateDX11::OnReleaseGPU()
 {
+    DepthStencilState = nullptr;
     BlendState = nullptr;
     VS = nullptr;
 #if GPU_ALLOW_TESSELLATION_SHADERS
@@ -58,8 +59,8 @@ bool GPUPipelineStateDX11::Init(const Description& desc)
 #endif
 
     // States
-    DepthStencilStateIndex = static_cast<int32>(desc.DepthFunc) + (desc.DepthEnable ? 0 : 9) + (desc.DepthWriteEnable ? 0 : 18);
     RasterizerStateIndex = static_cast<int32>(desc.CullMode) + (desc.Wireframe ? 0 : 3) + (desc.DepthClipEnable ? 0 : 6);
+    DepthStencilState = _device->GetDepthStencilState(&desc);
     BlendState = _device->GetBlendState(desc.BlendMode);
 
     // Calculate approx. memory usage (just to set sth)

Source/Engine/GraphicsDevice/DirectX/DX11/GPUPipelineStateDX11.h

@@ -15,8 +15,8 @@ class GPUPipelineStateDX11 : public GPUResourceDX11<GPUPipelineState>
 {
 public:
 
-    int32 DepthStencilStateIndex;
     int32 RasterizerStateIndex;
+    ID3D11DepthStencilState* DepthStencilState = nullptr;
     ID3D11BlendState* BlendState = nullptr;
     GPUShaderProgramVSDX11* VS = nullptr;
 #if GPU_ALLOW_TESSELLATION_SHADERS