From b73bd519becd1ece63fe6d0476e443b6a6f30670 Mon Sep 17 00:00:00 2001
From: Wojtek Figat <wojtek@figat.pl>
Date: Sat, 31 Dec 2022 14:43:50 +0100
Subject: [PATCH] Add navmesh update on terrain sculpt undo/redo and deffer
 navmesh update when sculpting action ends to prevent stalls

---
 Source/Editor/Tools/Terrain/Sculpt/Mode.cs    | 11 +---
 .../Terrain/Undo/EditTerrainMapAction.cs      | 62 ++++++++++++++-----
 2 files changed, 47 insertions(+), 26 deletions(-)

diff --git a/Source/Editor/Tools/Terrain/Sculpt/Mode.cs b/Source/Editor/Tools/Terrain/Sculpt/Mode.cs
index a8b8cf53a..4297408c8 100644
--- a/Source/Editor/Tools/Terrain/Sculpt/Mode.cs
+++ b/Source/Editor/Tools/Terrain/Sculpt/Mode.cs
@@ -147,17 +147,8 @@ namespace FlaxEditor.Tools.Terrain.Sculpt
                 Apply(ref p);
             }
 
-            var editorOptions = Editor.Instance.Options.Options;
-            bool isPlayMode = Editor.Instance.StateMachine.IsPlayMode;
-
             // Auto NavMesh rebuild
-            if (!isPlayMode && editorOptions.General.AutoRebuildNavMesh)
-            {
-                if (terrain.Scene && terrain.HasStaticFlag(StaticFlags.Navigation))
-                {
-                    Navigation.BuildNavMesh(terrain.Scene, brushBounds, editorOptions.General.AutoRebuildNavMeshTimeoutMs);
-                }
-            }
+            gizmo.CurrentEditUndoAction.AddDirtyBounds(ref brushBounds);
         }
 
         /// <summary>
diff --git a/Source/Editor/Tools/Terrain/Undo/EditTerrainMapAction.cs b/Source/Editor/Tools/Terrain/Undo/EditTerrainMapAction.cs
index baede8a73..c82b49b7e 100644
--- a/Source/Editor/Tools/Terrain/Undo/EditTerrainMapAction.cs
+++ b/Source/Editor/Tools/Terrain/Undo/EditTerrainMapAction.cs
@@ -41,33 +41,24 @@ namespace FlaxEditor.Tools.Terrain.Undo
             public object Tag;
         }
 
-        /// <summary>
-        /// The terrain (actor Id).
-        /// </summary>
         [Serialize]
         protected readonly Guid _terrain;
 
-        /// <summary>
-        /// The heightmap length (vertex count).
-        /// </summary>
         [Serialize]
         protected readonly int _heightmapLength;
 
-        /// <summary>
-        /// The heightmap data size (in bytes).
-        /// </summary>
         [Serialize]
         protected readonly int _heightmapDataSize;
 
-        /// <summary>
-        /// The terrain patches
-        /// </summary>
         [Serialize]
         protected readonly List<PatchData> _patches;
 
-        /// <summary>
-        /// Gets a value indicating whether this action has any modification to the terrain (recorded patches changes).
-        /// </summary>
+        [Serialize]
+        protected readonly List<BoundingBox> _navmeshBoundsModifications;
+
+        [Serialize]
+        protected readonly float _dirtyNavMeshTimeoutMs;
+
         [NoSerialize]
         public bool HasAnyModification => _patches.Count > 0;
 
@@ -98,6 +89,27 @@ namespace FlaxEditor.Tools.Terrain.Undo
             var heightmapSize = chunkSize * FlaxEngine.Terrain.PatchEdgeChunksCount + 1;
             _heightmapLength = heightmapSize * heightmapSize;
             _heightmapDataSize = _heightmapLength * stride;
+
+            // Auto NavMesh rebuild
+            var editorOptions = Editor.Instance.Options.Options;
+            bool isPlayMode = Editor.Instance.StateMachine.IsPlayMode;
+            if (!isPlayMode && editorOptions.General.AutoRebuildNavMesh)
+            {
+                if (terrain.Scene && terrain.HasStaticFlag(StaticFlags.Navigation))
+                {
+                    _navmeshBoundsModifications = new List<BoundingBox>();
+                    _dirtyNavMeshTimeoutMs = editorOptions.General.AutoRebuildNavMeshTimeoutMs;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Adds modified bounds to the undo actor modifications list.
+        /// </summary>
+        /// <param name="bounds">The world-space bounds.</param>
+        public void AddDirtyBounds(ref BoundingBox bounds)
+        {
+            _navmeshBoundsModifications?.Add(bounds);
         }
 
         /// <summary>
@@ -155,7 +167,15 @@ namespace FlaxEditor.Tools.Terrain.Undo
                 _patches[i] = patch;
             }
 
-            Editor.Instance.Scene.MarkSceneEdited(Terrain.Scene);
+            // Update navmesh
+            var scene = Terrain.Scene;
+            if (_navmeshBoundsModifications != null)
+            {
+                foreach (var bounds in _navmeshBoundsModifications)
+                    Navigation.BuildNavMesh(scene, bounds, _dirtyNavMeshTimeoutMs);
+            }
+
+            Editor.Instance.Scene.MarkSceneEdited(scene);
         }
 
         /// <inheritdoc />
@@ -183,10 +203,12 @@ namespace FlaxEditor.Tools.Terrain.Undo
                 Marshal.FreeHGlobal(_patches[i].After);
             }
             _patches.Clear();
+            _navmeshBoundsModifications?.Clear();
         }
 
         private void Set(Func<PatchData, IntPtr> dataGetter)
         {
+            // Update patches
             for (int i = 0; i < _patches.Count; i++)
             {
                 var patch = _patches[i];
@@ -194,6 +216,14 @@ namespace FlaxEditor.Tools.Terrain.Undo
                 SetData(ref patch.PatchCoord, data, patch.Tag);
             }
 
+            // Update navmesh
+            var scene = Terrain.Scene;
+            if (_navmeshBoundsModifications != null)
+            {
+                foreach (var bounds in _navmeshBoundsModifications)
+                    Navigation.BuildNavMesh(scene, bounds, _dirtyNavMeshTimeoutMs);
+            }
+
             Editor.Instance.Scene.MarkSceneEdited(Terrain.Scene);
         }