Fix custom 2D clipping mask to properly inherit parent container clip mask

2371

Source/Engine/Render2D/Render2D.cpp

@@ -841,10 +841,11 @@ void Render2D::PushClip(const Rectangle& clipRect)
 {
     RENDER2D_CHECK_RENDERING_STATE;
 
+    const auto& mask = ClipLayersStack.Peek();
     RotatedRectangle clipRectTransformed;
     ApplyTransform(clipRect, clipRectTransformed);
-    const Rectangle bounds = Rectangle::Shared(clipRectTransformed.ToBoundingRect(), ClipLayersStack.Peek().Bounds);
+    const Rectangle bounds = Rectangle::Shared(clipRectTransformed.ToBoundingRect(), mask.Bounds);
-    ClipLayersStack.Push({ clipRectTransformed, bounds });
+    ClipLayersStack.Push({ RotatedRectangle::Shared(clipRectTransformed, mask.Bounds), bounds });
 
     OnClipScissors();
 }

Source/Engine/Render2D/RotatedRectangle.h

@@ -10,7 +10,6 @@
 struct RotatedRectangle
 {
 public:
-
     /// <summary>
     /// The transformed top left corner.
     /// </summary>
@@ -27,7 +26,6 @@ public:
     Float2 ExtentY;
 
 public:
-
     /// <summary>
     /// Initializes a new instance of the <see cref="RotatedRectangle"/> struct.
     /// </summary>
@@ -60,7 +58,6 @@ public:
     }
 
 public:
-
     /// <summary>
     /// Convert rotated rectangle to the axis-aligned rectangle that builds rotated rectangle bounding box.
     /// </summary>
@@ -95,8 +92,24 @@ public:
         return false;
     }
 
-public:
+    /// <summary>
+    /// Calculates a rectangle that contains the shared part of both rectangles.
+    /// </summary>
+    /// <param name="a">The first rectangle.</param>
+    /// <param name="b">The second rectangle.</param>
+    /// <returns>Rectangle that contains shared part of a and b rectangles.</returns>
+    static RotatedRectangle Shared(const RotatedRectangle& a, const Rectangle& b)
+    {
+        // Clip the rotated rectangle bounds within the given AABB
+        RotatedRectangle result = a;
+        result.TopLeft = Float2::Max(a.TopLeft, b.GetTopLeft());
+        // TODO: do a little better math below (in case of actually rotated rectangle)
+        result.ExtentX.X = Math::Min(result.TopLeft.X + result.ExtentX.X, b.GetRight()) - result.TopLeft.X;
+        result.ExtentY.Y = Math::Min(result.TopLeft.Y + result.ExtentY.Y, b.GetBottom()) - result.TopLeft.Y;
+        return result;
+    }
 
+public:
     bool operator ==(const RotatedRectangle& other) const
     {
         return

Source/Shaders/GUICommon.hlsl

@@ -43,13 +43,9 @@ float4 PointInParallelogram(float2 p, float2 a, float4 bc)
 void PerformClipping(float2 clipOrigin, float2 windowPos, float4 clipExtents)
 {
 #if CLIPPING_ENABLE
-
+    // Clip pixels which are outside the clipping rect
-    // Clip pixels which are outside of the clipping rect
     float4 clipTest = PointInParallelogram(windowPos, clipOrigin, clipExtents);
-
-    // Clip pixels which are outside of the clipping rect
     clip(clipTest);
-
 #endif
 }