Refactor to ViewportRubberBandSelector class and use actor node virtual methods.

Source/Editor/Gizmo/ViewportRubberBandSelector.cs

@@ -0,0 +1,209 @@
+// Copyright (c) 2012-2024 Wojciech Figat. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using FlaxEditor;
+using FlaxEditor.Gizmo;
+using FlaxEditor.SceneGraph;
+using FlaxEngine.GUI;
+
+namespace FlaxEngine.Gizmo;
+
+/// <summary>
+/// Class for adding viewport rubber band selection.
+/// </summary>
+public class ViewportRubberBandSelector
+{
+    private bool _isRubberBandSpanning;
+    private bool _tryStartRubberBand;
+    private Float2 _cachedStartingMousePosition;
+    private Rectangle _rubberBandRect;
+    private Rectangle _lastRubberBandRect;
+
+    private IGizmoOwner _owner;
+
+    /// <summary>
+    /// Constructs a rubber band selector with a designated gizmo owner.
+    /// </summary>
+    /// <param name="owner">The gizmo owner.</param>
+    public ViewportRubberBandSelector(IGizmoOwner owner)
+    {
+        _owner = owner;
+    }
+
+    /// <summary>
+    /// Triggers the start of a rubber band selection.
+    /// </summary>
+    public void TryStartingRubberBandSelection()
+    {
+        if (!_isRubberBandSpanning && !_owner.Gizmos.Active.IsControllingMouse && !_owner.IsRightMouseButtonDown)
+        {
+            _tryStartRubberBand = true;
+        }
+    }
+
+    /// <summary>
+    /// Release the rubber band selection.
+    /// </summary>
+    /// <returns>Returns true if rubber band is currently spanning</returns>
+    public bool ReleaseRubberBandSelection()
+    {
+        if (_tryStartRubberBand)
+        {
+            _tryStartRubberBand = false;
+        }
+        
+        if (_isRubberBandSpanning)
+        {
+            _isRubberBandSpanning = false;
+            return true;
+        }
+        return false;
+    }
+    
+    /// <summary>
+    /// Tries to create a rubber band selection.
+    /// </summary>
+    /// <param name="canStart">Whether the creation can start.</param>
+    /// <param name="mousePosition">The current mouse position.</param>
+    /// <param name="viewFrustum">The view frustum.</param>
+    public void TryCreateRubberBand(bool canStart, Float2 mousePosition, BoundingFrustum viewFrustum)
+    {
+        if (_isRubberBandSpanning && !canStart)
+        {
+            _isRubberBandSpanning = false;
+            return;
+        }
+        
+        if (_tryStartRubberBand && (Mathf.Abs(_owner.MouseDelta.X) > 0.1f || Mathf.Abs(_owner.MouseDelta.Y) > 0.1f) && canStart)
+        {
+            _isRubberBandSpanning = true;
+            _cachedStartingMousePosition = mousePosition;
+            _rubberBandRect = new Rectangle(_cachedStartingMousePosition, Float2.Zero);
+            _tryStartRubberBand = false;
+        }
+        else if (_isRubberBandSpanning && !_owner.Gizmos.Active.IsControllingMouse && !_owner.IsRightMouseButtonDown)
+        {
+            _rubberBandRect.Width = mousePosition.X - _cachedStartingMousePosition.X;
+            _rubberBandRect.Height = mousePosition.Y - _cachedStartingMousePosition.Y;
+
+            if (_lastRubberBandRect != _rubberBandRect)
+            {
+                // Select rubberbanded rect actor nodes
+                var adjustedRect = _rubberBandRect;
+                _lastRubberBandRect = _rubberBandRect;
+                if (adjustedRect.Width < 0 || adjustedRect.Height < 0)
+                {
+                    // make sure we have a well-formed rectangle i.e. size is positive and X/Y is upper left corner
+                    var size = adjustedRect.Size;
+                    adjustedRect.X = Mathf.Min(adjustedRect.X, adjustedRect.X + adjustedRect.Width);
+                    adjustedRect.Y = Mathf.Min(adjustedRect.Y, adjustedRect.Y + adjustedRect.Height);
+                    size.X = Mathf.Abs(size.X);
+                    size.Y = Mathf.Abs(size.Y);
+                    adjustedRect.Size = size;
+                }
+
+                // Get hits from graph nodes.
+                List<SceneGraphNode> hits = new List<SceneGraphNode>();
+                var nodes = _owner.SceneGraphRoot.GetAllChildActorNodes();
+                foreach (var node in nodes)
+                {
+                    // Check for custom can select code
+                    if (!node.CanSelectActorNodeWithSelector())
+                        continue;
+ 
+                    var a = node.Actor;
+                    // Skip actor if outside of view frustum
+                    var actorBox = a.EditorBox;
+                    if (viewFrustum.Contains(actorBox) == ContainmentType.Disjoint)
+                        continue;
+
+                    // Get valid selection points
+                    var points = node.GetActorSelectionPoints();
+                    bool containsAllPoints = points.Length != 0;
+                    foreach (var point in points)
+                    {
+                        _owner.Viewport.ProjectPoint(point, out var loc);
+                        if (!adjustedRect.Contains(loc))
+                        {
+                            containsAllPoints = false;
+                            break;
+                        }
+                    }
+                    if (containsAllPoints)
+                    {
+                        if (a.HasPrefabLink)
+                            hits.Add(_owner.SceneGraphRoot.Find(a.GetPrefabRoot()));
+                        else
+                            hits.Add(node);
+                    }
+                }
+
+                var editor = Editor.Instance;
+                if (_owner.IsControlDown)
+                {
+                    var newSelection = new List<SceneGraphNode>();
+                    var currentSelection = editor.SceneEditing.Selection;
+                    newSelection.AddRange(currentSelection);
+                    foreach (var hit in hits)
+                    {
+                        if (currentSelection.Contains(hit))
+                            newSelection.Remove(hit);
+                        else
+                            newSelection.Add(hit);
+                    }
+                    _owner.Select(newSelection);
+                }
+                else if (Input.GetKey(KeyboardKeys.Shift))
+                {
+                    var newSelection = new List<SceneGraphNode>();
+                    var currentSelection = editor.SceneEditing.Selection;
+                    newSelection.AddRange(hits);
+                    newSelection.AddRange(currentSelection);
+                    _owner.Select(newSelection);
+                }
+                else
+                {
+                    _owner.Select(hits);
+                }
+            }
+            
+        }
+    }
+
+    /// <summary>
+    /// Used to draw the rubber band. Begins render 2D.
+    /// </summary>
+    /// <param name="context">The GPU Context.</param>
+    /// <param name="target">The GPU texture target.</param>
+    /// <param name="targetDepth">The GPU texture target depth.</param>
+    public void Draw(GPUContext context, GPUTexture target, GPUTexture targetDepth)
+    {
+        // Draw RubberBand for rect selection
+        if (!_isRubberBandSpanning)
+            return;
+        Render2D.Begin(context, target, targetDepth);
+        Draw2D();
+        Render2D.End();
+    }
+
+    /// <summary>
+    /// Used to draw the rubber band. Use if already rendering 2D context.
+    /// </summary>
+    public void Draw2D()
+    {
+        if (!_isRubberBandSpanning)
+            return;
+        Render2D.FillRectangle(_rubberBandRect, Style.Current.Selection);
+        Render2D.DrawRectangle(_rubberBandRect, Style.Current.SelectionBorder);
+    }
+
+    /// <summary>
+    /// Immediately stops the rubber band.
+    /// </summary>
+    public void StopRubberBand()
+    {
+        _isRubberBandSpanning = false;
+        _tryStartRubberBand = false;
+    }
+}

Source/Editor/SceneGraph/ActorNode.cs

@@ -182,6 +182,51 @@ namespace FlaxEditor.SceneGraph
             return null;
         }
 
+        /// <summary>
+        /// Get all nested actor nodes under this actor node.
+        /// </summary>
+        /// <returns>An array of ActorNodes</returns>
+        public ActorNode[] GetAllChildActorNodes()
+        {
+            // Check itself
+            if (ChildNodes == null || ChildNodes.Count == 0)
+                return [];
+
+            // Check deeper
+            var nodes = new List<ActorNode>();
+            for (int i = 0; i < ChildNodes.Count; i++)
+            {
+                if (ChildNodes[i] is ActorNode node)
+                {
+                    nodes.Add(node);
+                    var childNodes = node.GetAllChildActorNodes();
+                    if (childNodes.Length > 0)
+                    {
+                        nodes.AddRange(childNodes);
+                    }
+                }
+            }
+            return nodes.ToArray();
+        }
+
+        /// <summary>
+        /// Whether an actor node can be selected with a selector.
+        /// </summary>
+        /// <returns>True if the actor node can be selected</returns>
+        public virtual bool CanSelectActorNodeWithSelector()
+        {
+            return Actor && Actor.HideFlags is not (HideFlags.DontSelect or HideFlags.FullyHidden) && Actor is not EmptyActor && IsActive;
+        }
+
+        /// <summary>
+        /// The selection points used to check if an actor node can be selected.
+        /// </summary>
+        /// <returns>The points to use if the actor can be selected.</returns>
+        public virtual Vector3[] GetActorSelectionPoints()
+        {
+            return Actor.EditorBox.GetCorners();
+        }
+
         /// <summary>
         /// Gets a value indicating whether this actor can be used to create prefab from it (as a root).
         /// </summary>

Source/Editor/SceneGraph/Actors/CameraNode.cs

@@ -58,5 +58,11 @@ namespace FlaxEditor.SceneGraph.Actors
 
             return Camera.Internal_IntersectsItselfEditor(FlaxEngine.Object.GetUnmanagedPtr(_actor), ref ray.Ray, out distance);
         }
+
+        /// <inheritdoc />
+        public override Vector3[] GetActorSelectionPoints()
+        {
+            return [Actor.Position];
+        }
     }
 }

Source/Editor/SceneGraph/Actors/SceneNode.cs

@@ -33,6 +33,12 @@ namespace FlaxEditor.SceneGraph.Actors
             }
         }
 
+        /// <inheritdoc />
+        public override bool CanSelectActorNodeWithSelector()
+        {
+            return false;
+        }
+
         /// <summary>
         /// Gets the scene.
         /// </summary>

Source/Editor/SceneGraph/Actors/StaticModelNode.cs

@@ -88,6 +88,30 @@ namespace FlaxEditor.SceneGraph.Actors
             contextMenu.AddButton("Add collider", () => OnAddMeshCollider(window)).Enabled = ((StaticModel)Actor).Model != null;
         }
 
+        /// <inheritdoc />
+        public override Vector3[] GetActorSelectionPoints()
+        {
+            if (Actor is not StaticModel sm || !sm.Model)
+                return base.GetActorSelectionPoints();
+
+            // Check collision proxy points for more accurate selection.
+            var vecPoints = new List<Vector3>();
+            var m = sm.Model.LODs[0];
+            foreach (var mesh in m.Meshes)
+            {
+                var points = mesh.GetCollisionProxyPoints();
+                foreach (var point in points)
+                {
+                    vecPoints.Add(Actor.Transform.LocalToWorld(point));
+                }
+            }
+            
+            // Fall back to base actor editor box if no points from collision proxy.
+            if (vecPoints.Count == 0)
+                return base.GetActorSelectionPoints();
+            return vecPoints.ToArray();
+        }
+
         private void OnAddMeshCollider(EditorWindow window)
         {
             // Allow collider to be added to evey static model selection

Source/Editor/SceneGraph/Actors/UICanvasNode.cs

@@ -78,5 +78,11 @@ namespace FlaxEditor.SceneGraph.Actors
             if (Actor is UICanvas uiCanvas && uiCanvas.Is3D)
                 DebugDraw.DrawWireBox(uiCanvas.Bounds, Color.BlueViolet);
         }
+
+        /// <inheritdoc />
+        public override bool CanSelectActorNodeWithSelector()
+        {
+            return Actor is UICanvas uiCanvas && uiCanvas.Is3D;
+        }
     }
 }

Source/Editor/SceneGraph/Actors/UIControlNode.cs

@@ -40,5 +40,31 @@ namespace FlaxEditor.SceneGraph.Actors
                     control.PerformLayout();
             }
         }
+
+        /// <inheritdoc />
+        public override bool CanSelectActorNodeWithSelector()
+        {
+            // Check if control and skip if canvas is 2D
+            if (Actor is not UIControl uiControl)
+                return false;
+            UICanvas canvas = null;
+            var controlParent = uiControl.Parent;
+            while (controlParent != null && controlParent is not Scene)
+            {
+                if (controlParent is UICanvas uiCanvas)
+                {
+                    canvas = uiCanvas;
+                    break;
+                }
+                controlParent = controlParent.Parent;
+            }
+
+            if (canvas != null)
+            {
+                if (canvas.Is2D)
+                    return false;
+            }
+            return true;
+        }
     }
 }

Source/Editor/Viewport/MainEditorGizmoViewport.cs

@@ -11,6 +11,7 @@ using FlaxEditor.Tools;
 using FlaxEditor.Viewport.Modes;
 using FlaxEditor.Windows;
 using FlaxEngine;
+using FlaxEngine.Gizmo;
 using FlaxEngine.GUI;
 using FlaxEngine.Tools;
 
@@ -111,11 +112,7 @@ namespace FlaxEditor.Viewport
         private readonly ViewportDebugDrawData _debugDrawData = new ViewportDebugDrawData(32);
         private EditorSpritesRenderer _editorSpritesRenderer;
 
-        private bool _isRubberBandSpanning;
+        private ViewportRubberBandSelector _rubberBandSelector;
-        private bool _tryStartRubberBand;
-        private Float2 _cachedStartingMousePosition;
-        private Rectangle _rubberBandRect;
-        private Rectangle _lastRubberBandRect;
 
         /// <summary>
         /// Drag and drop handlers
@@ -223,6 +220,9 @@ namespace FlaxEditor.Viewport
             TransformGizmo.Duplicate += _editor.SceneEditing.Duplicate;
             Gizmos.Active = TransformGizmo;
             
+            // Add rubber band selector
+            _rubberBandSelector = new ViewportRubberBandSelector(this);
+
             // Add grid
             Grid = new GridGizmo(this);
             Grid.EnabledChanged += gizmo => _showGridButton.Icon = gizmo.Enabled ? Style.Current.CheckBoxTick : SpriteHandle.Invalid;
@@ -378,13 +378,7 @@ namespace FlaxEditor.Viewport
             }
             
             // Draw RubberBand for rect selection
-            if (_isRubberBandSpanning)
+            _rubberBandSelector.Draw(context, target, targetDepth);
-            {
-                Render2D.Begin(context, target, targetDepth);
-                Render2D.FillRectangle(_rubberBandRect, Style.Current.Selection);
-                Render2D.DrawRectangle(_rubberBandRect, Style.Current.SelectionBorder);
-                Render2D.End();
-            }
             
             // Draw selected objects debug shapes and visuals
             if (DrawDebugDraw && (renderContext.View.Flags & ViewFlags.DebugDraw) == ViewFlags.DebugDraw)
@@ -500,16 +494,14 @@ namespace FlaxEditor.Viewport
         public override void OnLostFocus()
         {
             base.OnLostFocus();
-            _isRubberBandSpanning = false;
+            _rubberBandSelector.StopRubberBand();
-            _tryStartRubberBand = false;
         }
 
         /// <inheritdoc />
         public override void OnMouseLeave()
         {
             base.OnMouseLeave();
-            _isRubberBandSpanning = false;
+            _rubberBandSelector.StopRubberBand();
-            _tryStartRubberBand = false;
         }
 
         /// <summary>
@@ -616,161 +608,8 @@ namespace FlaxEditor.Viewport
             base.OnMouseMove(location);
 
             // Dont allow rubber band selection when gizmo is controlling mouse, vertex painting mode, or cloth painting is enabled
-            if (_isRubberBandSpanning && ((Gizmos.Active.IsControllingMouse || Gizmos.Active is VertexPaintingGizmo || Gizmos.Active is ClothPaintingGizmo) || IsControllingMouse || IsRightMouseButtonDown))
+            _rubberBandSelector.TryCreateRubberBand(!((Gizmos.Active.IsControllingMouse || Gizmos.Active is VertexPaintingGizmo || Gizmos.Active is ClothPaintingGizmo) || IsControllingMouse || IsRightMouseButtonDown), 
-            {
+                                                   _viewMousePos, ViewFrustum);
-                _isRubberBandSpanning = false;
-            }
-
-            if (_tryStartRubberBand && (Mathf.Abs(MouseDelta.X) > 0.1f || Mathf.Abs(MouseDelta.Y) > 0.1f) && !_isRubberBandSpanning && !Gizmos.Active.IsControllingMouse && !IsControllingMouse && !IsRightMouseButtonDown)
-            {
-                _isRubberBandSpanning = true;
-                _cachedStartingMousePosition = _viewMousePos;
-                _rubberBandRect = new Rectangle(_cachedStartingMousePosition, Float2.Zero);
-            }
-            else if (_isRubberBandSpanning && !Gizmos.Active.IsControllingMouse && !IsControllingMouse && !IsRightMouseButtonDown)
-            {
-                _rubberBandRect.Width = _viewMousePos.X - _cachedStartingMousePosition.X;
-                _rubberBandRect.Height = _viewMousePos.Y - _cachedStartingMousePosition.Y;
-
-                if (_lastRubberBandRect != _rubberBandRect)
-                {
-                    // Select rubberbanded rect actor nodes
-                    var adjustedRect = _rubberBandRect;
-                    _lastRubberBandRect = _rubberBandRect;
-                    if (adjustedRect.Width < 0 || adjustedRect.Height < 0)
-                    {
-                        // make sure we have a well-formed rectangle i.e. size is positive and X/Y is upper left corner
-                        var size = adjustedRect.Size;
-                        adjustedRect.X = Mathf.Min(adjustedRect.X, adjustedRect.X + adjustedRect.Width);
-                        adjustedRect.Y = Mathf.Min(adjustedRect.Y, adjustedRect.Y + adjustedRect.Height);
-                        size.X = Mathf.Abs(size.X);
-                        size.Y = Mathf.Abs(size.Y);
-                        adjustedRect.Size = size;
-                    }
-
-                    List<SceneGraphNode> hits = new List<SceneGraphNode>();
-                    var allActors = Level.GetActors<Actor>(true);
-                    foreach (var a in allActors)
-                    {
-                        if (a.HideFlags is HideFlags.DontSelect or HideFlags.FullyHidden || a is EmptyActor || a is Scene || !a.IsActive)
-                            continue;
-
-                        var actorBox = a.EditorBox;
-                        if (ViewFrustum.Contains(actorBox) == ContainmentType.Disjoint)
-                            continue;
-
-                        // Check if control and skip if canvas is 2D
-                        if (a is UIControl control)
-                        {
-                            UICanvas canvas = null;
-                            var controlParent = control.Parent;
-                            while (controlParent != null && controlParent is not Scene)
-                            {
-                                if (controlParent is UICanvas uiCanvas)
-                                {
-                                    canvas = uiCanvas;
-                                    break;
-                                }
-                                controlParent = controlParent.Parent;
-                            }
-
-                            if (canvas != null)
-                            {
-                                if (canvas.Is2D)
-                                    continue;
-                            }
-                        }
-                        else if (a is UICanvas uiCanvas)
-                        {
-                            if (uiCanvas.Is2D)
-                                continue;
-                        }
-
-                        var containsAllPoints = true;
-                        var fallBackToBox = false;
-                        if (a is StaticModel sm)
-                        {
-                            if (sm.Model)
-                            {
-                                var m = sm.Model.LODs[0];
-                                foreach (var mesh in m.Meshes)
-                                {
-                                    var points = mesh.GetCollisionProxyPoints();
-                                    if (points.Length == 0)
-                                    {
-                                        fallBackToBox = true;
-                                        break;
-                                    }
-                                    foreach (var point in points)
-                                    {
-                                        Viewport.ProjectPoint(a.Transform.LocalToWorld(point), out var loc);
-                                        if (!adjustedRect.Contains(loc))
-                                        {
-                                            containsAllPoints = false;
-                                            break;
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                        else
-                        {
-                            fallBackToBox = true;
-                        }
-
-                        if (fallBackToBox)
-                        {
-                            // Check if all corners are in box to select it.
-                            var corners = actorBox.GetCorners();
-                            foreach (var c in corners)
-                            {
-                                Viewport.ProjectPoint(c, out var loc);
-                                if (!adjustedRect.Contains(loc))
-                                {
-                                    containsAllPoints = false;
-                                    break;
-                                }
-                            }
-                        }
-                        
-                        if (containsAllPoints)
-                        {
-                            if (a.HasPrefabLink)
-                                hits.Add(SceneGraphRoot.Find(a.GetPrefabRoot()));
-                            else
-                                hits.Add(SceneGraphRoot.Find(a));
-                        }
-                    }
-                    
-                    if (IsControlDown)
-                    {
-                        var newSelection = new List<SceneGraphNode>();
-                        var currentSelection = _editor.SceneEditing.Selection;
-                        newSelection.AddRange(currentSelection);
-                        foreach (var hit in hits)
-                        {
-                            if (currentSelection.Contains(hit))
-                                newSelection.Remove(hit);
-                            else
-                                newSelection.Add(hit);
-                        }
-                        Select(newSelection);
-                    }
-                    else if (((WindowRootControl)Root).GetKey(KeyboardKeys.Shift))
-                    {
-                        var newSelection = new List<SceneGraphNode>();
-                        var currentSelection = _editor.SceneEditing.Selection;
-                        newSelection.AddRange(hits);
-                        newSelection.AddRange(currentSelection);
-                        Select(newSelection);
-                    }
-                    else
-                    {
-                        Select(hits);
-                    }
-                }
-                
-            }
         }
 
         /// <inheritdoc />
@@ -778,10 +617,7 @@ namespace FlaxEditor.Viewport
         {
             base.OnLeftMouseButtonDown();
             
-            if (!_isRubberBandSpanning && !Gizmos.Active.IsControllingMouse && !IsControllingMouse && !IsRightMouseButtonDown)
+            _rubberBandSelector.TryStartingRubberBandSelection();
-            {
-                _tryStartRubberBand = true;
-            }
         }
 
         /// <inheritdoc />
@@ -791,17 +627,8 @@ namespace FlaxEditor.Viewport
             if (_prevInput.IsControllingMouse || !Bounds.Contains(ref _viewMousePos))
                 return;
 
-            if (_tryStartRubberBand)
+            // Select rubberbanded rect actor nodes or pick with gizmo
-            {
+            if (!_rubberBandSelector.ReleaseRubberBandSelection())
-                _tryStartRubberBand = false;
-            }
-            
-            // Select rubberbanded rect actor nodes
-            if (_isRubberBandSpanning)
-            {
-                _isRubberBandSpanning = false;
-            }
-            else
             {
                 // Try to pick something with the current gizmo
                 Gizmos.Active?.Pick();

Source/Engine/Graphics/Mesh.cs

@@ -638,7 +638,7 @@ namespace FlaxEngine
         /// Gets the collision proxy points for the mesh.
         /// </summary>
         /// <returns>The triangle points in the collision proxy.</returns>
-        public Float3[] GetCollisionProxyPoints()
+        internal Float3[] GetCollisionProxyPoints()
         {
             return Internal_GetCollisionProxyPoints(__unmanagedPtr, out _);
         }