Add **render layer mask to decals**

#967 #3080

Content/Editor/MaterialTemplates/Decal.shader

@@ -6,6 +6,7 @@
 @3
 
 #include "./Flax/Common.hlsl"
+#include "./Flax/Stencil.hlsl"
 #include "./Flax/MaterialCommon.hlsl"
 #include "./Flax/GBufferCommon.hlsl"
 @7
@@ -14,10 +15,13 @@ META_CB_BEGIN(0, Data)
 float4x4 WorldMatrix;
 float4x4 InvWorld;
 float4x4 SvPositionToWorld;
+float3 Padding0;
+uint RenderLayersMask;
 @1META_CB_END
 
 // Use depth buffer for per-pixel decal layering
 Texture2D DepthBuffer : register(t0);
+Texture2D<uint2> StencilBuffer : register(t1);
 
 // Material shader resources
 @2
@@ -200,6 +204,14 @@ void PS_Decal(
 #endif
 	)
 {
+	// Stencil masking
+	uint stencilObjectLayer = STENCIL_BUFFER_OBJECT_LAYER(STENCIL_BUFFER_LOAD(StencilBuffer, SvPosition.xy));
+	if ((RenderLayersMask & (1 << stencilObjectLayer)) == 0)
+	{
+		clip(-1);
+		return;
+	}
+
 	float2 screenUV = SvPosition.xy * ScreenSize.zw;
 	SvPosition.z = SAMPLE_RT(DepthBuffer, screenUV).r;
 

Source/Engine/Graphics/Materials/DecalMaterialShader.cpp

@@ -17,6 +17,8 @@ PACK_STRUCT(struct DecalMaterialShaderData {
     Matrix WorldMatrix;
     Matrix InvWorld;
     Matrix SvPositionToWorld;
+    Float3 Padding0;
+    uint32 RenderLayersMask;
     });
 
 DrawPass DecalMaterialShader::GetDrawModes() const
@@ -50,6 +52,7 @@ void DecalMaterialShader::Bind(BindParameters& params)
     GPUTexture* depthBuffer = params.RenderContext.Buffers->DepthBuffer;
     GPUTextureView* depthBufferView = EnumHasAnyFlags(depthBuffer->Flags(), GPUTextureFlags::ReadOnlyDepthView) ? depthBuffer->ViewReadOnlyDepth() : depthBuffer->View();
     context->BindSR(0, depthBufferView);
+    context->BindSR(1, depthBuffer->ViewStencil());
 
     // Setup material constants
     {
@@ -68,6 +71,7 @@ void DecalMaterialShader::Bind(BindParameters& params)
             -1.0f, 1.0f, 0, 1);
         const Matrix svPositionToWorld = offsetMatrix * view.IVP;
         Matrix::Transpose(svPositionToWorld, materialData->SvPositionToWorld);
+        materialData->RenderLayersMask = (uint32)drawCall.SortKey; // Provided by GBufferPass::DrawDecals
     }
 
     // Bind constants

Source/Engine/Graphics/Materials/MaterialShader.h

@@ -10,7 +10,7 @@
 /// <summary>
 /// Current materials shader version.
 /// </summary>
-#define MATERIAL_GRAPH_VERSION 177
+#define MATERIAL_GRAPH_VERSION 178
 
 class Material;
 class GPUShader;

Source/Engine/Level/Actors/Decal.cpp

@@ -87,6 +87,7 @@ void Decal::Draw(RenderContext& renderContext)
         transform.Scale *= _size;
         renderContext.View.GetWorldMatrix(transform, data.World);
         data.SortOrder = SortOrder;
+        data.RenderLayersMask = RenderLayersMask;
         data.Material = material;
         renderContext.List->Decals.Add(data);
     }
@@ -102,6 +103,7 @@ void Decal::Serialize(SerializeStream& stream, const void* otherObj)
     SERIALIZE(Material);
     SERIALIZE_MEMBER(Size, _size);
     SERIALIZE(SortOrder);
+    SERIALIZE(RenderLayersMask);
     SERIALIZE(DrawMinScreenSize);
 }
 
@@ -113,6 +115,7 @@ void Decal::Deserialize(DeserializeStream& stream, ISerializeModifier* modifier)
     DESERIALIZE(Material);
     DESERIALIZE_MEMBER(Size, _size);
     DESERIALIZE(SortOrder);
+    DESERIALIZE(RenderLayersMask);
     DESERIALIZE(DrawMinScreenSize);
 
     _bounds.Extents = _size * 0.5f;

Source/Engine/Level/Actors/Decal.h

@@ -3,6 +3,7 @@
 #pragma once
 
 #include "../Actor.h"
+#include "Engine/Core/Types/LayersMask.h"
 #include "Engine/Core/Math/Matrix.h"
 #include "Engine/Core/Math/OrientedBoundingBox.h"
 #include "Engine/Content/Assets/MaterialBase.h"
@@ -33,6 +34,12 @@ public:
     API_FIELD(Attributes="EditorOrder(20), EditorDisplay(\"Decal\")")
     int32 SortOrder = 0;
 
+    /// <summary>
+    /// The layers mask used for render composition. Can be used to include or exclude specific actor layers from the drawing on top of them.
+    /// </summary>
+    API_FIELD(Attributes = "EditorOrder(25), EditorDisplay(\"Decal\")")
+    LayersMask RenderLayersMask;
+
     /// <summary>
     /// The minimum screen size for the decal drawing. If the decal size on the screen is smaller than this value then decal will be culled. Set it to higher value to make culling more aggressive.
     /// </summary>

Source/Engine/Renderer/GBufferPass.cpp

@@ -518,6 +518,7 @@ void GBufferPass::DrawDecals(RenderContext& renderContext, GPUTextureView* light
 
         // Draw decal
         drawCall.World = decal.World;
+        drawCall.SortKey = (uint64)decal.RenderLayersMask;
         decal.Material->Bind(bindParams);
         // TODO: use hardware instancing
         context->DrawIndexed(drawCall.Draw.IndicesCount);

Source/Engine/Renderer/RenderList.h

@@ -167,6 +167,7 @@ struct RenderDecalData
     Matrix World;
     MaterialBase* Material;
     int32 SortOrder;
+    uint32 RenderLayersMask;
 };
 
 /// <summary>

Source/Engine/Tools/MaterialGenerator/MaterialGenerator.cpp

@@ -490,7 +490,7 @@ bool MaterialGenerator::Generate(WriteStream& source, MaterialInfo& materialInfo
             srv = 3; // Objects + Skinning Bones + Prev Bones
             break;
         case MaterialDomain::Decal:
-            srv = 1; // Depth buffer
+            srv = 2; // Depth buffer + Stencil buffer
             break;
         case MaterialDomain::Terrain:
             srv = 3; // Heightmap + 2 splatmaps