Improve cursor clipping logic (fix after alt+tab and confine to game window in Editor)

#691

Source/Editor/Editor.cs

@@ -1292,43 +1292,29 @@ namespace FlaxEditor
             return false;
         }
 
-        internal void Internal_ScreenToGameViewport(ref Float2 pos)
+        internal void Internal_ScreenToGameViewport(out Float2 pos)
         {
-            if (Windows.GameWin != null && Windows.GameWin.ContainsFocus)
+            pos = Float2.Zero;
+            //if (Windows.GameWin != null && Windows.GameWin.ContainsFocus)
             {
-                var win = Windows.GameWin.Root;
+                var win = Windows.GameWin?.Root;
                 if (win?.RootWindow is WindowRootControl root && root.Window && root.Window.IsFocused)
                 {
                     pos = Float2.Round(Windows.GameWin.Viewport.PointFromScreen(pos) * root.DpiScale);
                 }
-                else
-                {
-                    pos = Float2.Minimum;
-                }
-            }
-            else
-            {
-                pos = Float2.Minimum;
             }
         }
 
-        internal void Internal_GameViewportToScreen(ref Float2 pos)
+        internal void Internal_GameViewportToScreen(out Float2 pos)
         {
-            if (Windows.GameWin != null && Windows.GameWin.ContainsFocus)
+            pos = Float2.Zero;
+            //if (Windows.GameWin != null && Windows.GameWin.ContainsFocus)
             {
-                var win = Windows.GameWin.Root;
+                var win = Windows.GameWin?.Root;
                 if (win?.RootWindow is WindowRootControl root && root.Window && root.Window.IsFocused)
                 {
                     pos = Float2.Round(Windows.GameWin.Viewport.PointToScreen(pos / root.DpiScale));
                 }
-                else
-                {
-                    pos = Float2.Minimum;
-                }
-            }
-            else
-            {
-                pos = Float2.Minimum;
             }
         }
 
@@ -1345,7 +1331,7 @@ namespace FlaxEditor
 
         internal void Internal_GetGameWindowSize(out Float2 resultAsRef)
         {
-            resultAsRef = Float2.Zero;
+            resultAsRef = new Float2(1280, 720);
             var gameWin = Windows.GameWin;
             if (gameWin != null)
             {

Source/Editor/Windows/GameWindow.cs

@@ -25,6 +25,7 @@ namespace FlaxEditor.Windows
         private float _gameStartTime;
         private GUI.Docking.DockState _maximizeRestoreDockState;
         private GUI.Docking.DockPanel _maximizeRestoreDockTo;
+        private CursorLockMode _cursorLockMode = CursorLockMode.None;
 
         /// <summary>
         /// Gets the viewport.
@@ -464,6 +465,8 @@ namespace FlaxEditor.Windows
                 if (Editor.Windows.PropertiesWin.IsDocked)
                     Editor.Windows.PropertiesWin.Focus();
                 Screen.CursorVisible = true;
+                if (Screen.CursorLock == CursorLockMode.Clipped)
+                    Screen.CursorLock = CursorLockMode.None;
             }
         }
 
@@ -522,11 +525,18 @@ namespace FlaxEditor.Windows
         {
             base.OnStartContainsFocus();
 
-            // Center mouse in play mode
+            if (Editor.StateMachine.IsPlayMode && !Editor.StateMachine.PlayingState.IsPaused)
-            if (CenterMouseOnFocus && Editor.StateMachine.IsPlayMode && !Editor.StateMachine.PlayingState.IsPaused)
             {
-                var center = PointToWindow(Size * 0.5f);
+                // Center mouse in play mode
-                Root.MousePosition = center;
+                if (CenterMouseOnFocus)
+                {
+                    var center = PointToWindow(Size * 0.5f);
+                    Root.MousePosition = center;
+                }
+
+                // Restore lock mode
+                if (_cursorLockMode != CursorLockMode.None)
+                    Screen.CursorLock = _cursorLockMode;
             }
         }
 
@@ -537,6 +547,9 @@ namespace FlaxEditor.Windows
 
             // Restore cursor visibility (could be hidden by the game)
             Screen.CursorVisible = true;
+
+            // Cache lock mode
+            _cursorLockMode = Screen.CursorLock;
         }
 
         /// <inheritdoc />

Source/Engine/Engine/Screen.cpp

@@ -19,7 +19,6 @@ static CursorLockMode CursorLock = CursorLockMode::None;
 class ScreenService : public EngineService
 {
 public:
-
     ScreenService()
         : EngineService(TEXT("Screen"), 120)
     {
@@ -89,7 +88,7 @@ Float2 Screen::GameViewportToScreen(const Float2& viewportPos)
 bool Screen::GetCursorVisible()
 {
 #if USE_EDITOR
-    const auto win = Editor::Managed->GetGameWindow();
+    const auto win = Editor::Managed->GetGameWindow(true);
 #else
 	const auto win = Engine::MainWindow;
 #endif
@@ -99,7 +98,7 @@ bool Screen::GetCursorVisible()
 void Screen::SetCursorVisible(const bool value)
 {
 #if USE_EDITOR
-    const auto win = Editor::Managed->GetGameWindow();
+    const auto win = Editor::Managed->GetGameWindow(true);
 #else
 	const auto win = Engine::MainWindow;
 #endif
@@ -117,13 +116,18 @@ CursorLockMode Screen::GetCursorLock()
 void Screen::SetCursorLock(CursorLockMode mode)
 {
 #if USE_EDITOR
-    const auto win = Editor::Managed->GetGameWindow();
+    const auto win = Editor::Managed->GetGameWindow(true);
 #else
     const auto win = Engine::MainWindow;
 #endif
     if (win && mode == CursorLockMode::Clipped)
     {
-        win->StartClippingCursor(win->GetClientBounds());
+#if USE_EDITOR
+        Rectangle bounds(Editor::Managed->GameViewportToScreen(Float2::Zero), Editor::Managed->GetGameWindowSize());
+#else
+        Rectangle bounds = win->GetClientBounds();
+#endif
+        win->StartClippingCursor(bounds);
     }
     else if (win && CursorLock == CursorLockMode::Clipped)
     {

Source/Engine/Platform/Base/WindowBase.cpp

@@ -537,6 +537,8 @@ void WindowBase::Hide()
 {
     if (!_visible)
         return;
+    EndClippingCursor();
+    EndTrackingMouse();
     _visible = false;
     _showAfterFirstPaint = _settings.ShowAfterFirstPaint;
     Hidden();
@@ -560,7 +562,6 @@ void WindowBase::Close(ClosingReason reason)
     }
 
     // Close
-    EndTrackingMouse();
     Hide();
     OnClosed();
 }

Source/Engine/Platform/Base/WindowBase.h

@@ -271,10 +271,9 @@ API_INJECT_CODE(cpp, "#include \"Engine/Platform/Window.h\"");
 API_CLASS(NoSpawn, NoConstructor, Sealed, Name="Window")
 class FLAXENGINE_API WindowBase : public ScriptingObject
 {
-DECLARE_SCRIPTING_TYPE_NO_SPAWN(WindowBase);
+    DECLARE_SCRIPTING_TYPE_NO_SPAWN(WindowBase);
     friend GPUSwapChain;
 protected:
-
     bool _visible, _minimized, _maximized, _isClosing, _showAfterFirstPaint, _focused;
     GPUSwapChain* _swapChain;
     CreateWindowSettings _settings;
@@ -294,7 +293,6 @@ protected:
     virtual ~WindowBase();
 
 public:
-
     /// <summary>
     /// The rendering task for that window.
     /// </summary>
@@ -336,7 +334,6 @@ public:
     Action Draw;
 
 public:
-
     // Returns true if that window is the main Engine window (works in both editor and game mode)
     bool IsMain() const;
 
@@ -405,7 +402,6 @@ public:
     API_PROPERTY() virtual void* GetNativePtr() const = 0;
 
 public:
-
     /// <summary>
     /// Performs the UI update.
     /// </summary>
@@ -471,11 +467,9 @@ public:
     /// <summary>
     /// Checks if window is foreground (the window with which the user is currently working).
     /// </summary>
-    /// <returns>True if window is foreground, otherwise false.</returns>
     API_PROPERTY() virtual bool IsForegroundWindow() const;
 
 public:
-
     /// <summary>
     /// Gets the client bounds of the window (client area not including border).
     /// </summary>
@@ -585,8 +579,8 @@ public:
     {
         return Platform::CustomDpiScale * _dpiScale;
     }
-public:
 
+public:
     /// <summary>
     /// Gets the window title.
     /// </summary>
@@ -652,7 +646,6 @@ public:
     }
 
 public:
-
     /// <summary>
     /// Starts drag and drop operation
     /// </summary>
@@ -674,7 +667,6 @@ public:
     /// <summary>
     /// Gets the mouse tracking offset.
     /// </summary>
-    /// <returns>The mouse screen offset.</returns>
     API_PROPERTY() Float2 GetTrackingMouseOffset() const
     {
         return _trackingMouseOffset;
@@ -698,7 +690,7 @@ public:
     /// <summary>
     /// Starts the cursor clipping.
     /// </summary>
-    /// <param name="bounds">The bounds that the cursor will be confined to.</param>
+    /// <param name="bounds">The screen-space bounds that the cursor will be confined to.</param>
     API_FUNCTION() virtual void StartClippingCursor(const Rectangle& bounds)
     {
     }
@@ -721,7 +713,6 @@ public:
     /// <summary>
     /// Gets the mouse cursor.
     /// </summary>
-    /// <returns>The cursor type</returns>
     API_PROPERTY() FORCE_INLINE CursorType GetCursor() const
     {
         return _cursor;
@@ -755,7 +746,6 @@ public:
     API_PROPERTY() void SetRenderingEnabled(bool value);
 
 public:
-
     typedef Delegate<Char> CharDelegate;
     typedef Delegate<KeyboardKeys> KeyboardDelegate;
     typedef Delegate<const Float2&> MouseDelegate;
@@ -882,7 +872,6 @@ public:
     void OnClosing(ClosingReason reason, bool& cancel);
 
 public:
-
     /// <summary>
     /// Gets the text entered during the current frame (Unicode).
     /// </summary>
@@ -911,11 +900,9 @@ public:
     API_FUNCTION() bool GetKeyUp(KeyboardKeys key) const;
 
 public:
-
     /// <summary>
     /// Gets the mouse position in window coordinates.
     /// </summary>
-    /// <returns>Mouse cursor coordinates</returns>
     API_PROPERTY() Float2 GetMousePosition() const;
 
     /// <summary>
@@ -933,7 +920,6 @@ public:
     /// <summary>
     /// Gets the mouse wheel change during the last frame.
     /// </summary>
-    /// <returns>Mouse wheel value delta</returns>
     API_PROPERTY() float GetMouseScrollDelta() const;
 
     /// <summary>
@@ -958,7 +944,6 @@ public:
     API_FUNCTION() bool GetMouseButtonUp(MouseButton button) const;
 
 public:
-
     void OnShow();
     void OnResize(int32 width, int32 height);
     void OnClosed();
@@ -966,14 +951,12 @@ public:
     void OnLostFocus();
 
 private:
-
     void OnMainRenderTaskDelete(class ScriptingObject* obj)
     {
         RenderTask = nullptr;
     }
 
 public:
-
     // [ScriptingObject]
     String ToString() const override;
     void OnDeleteObject() override;

Source/Engine/Platform/Windows/WindowsWindow.cpp

@@ -519,7 +519,6 @@ void WindowsWindow::SetOpacity(const float opacity)
 void WindowsWindow::Focus()
 {
     ASSERT(HasHWND());
-
     if (GetFocus() != _handle)
     {
         SetFocus(_handle);
@@ -566,20 +565,18 @@ void WindowsWindow::EndTrackingMouse()
 
 void WindowsWindow::StartClippingCursor(const Rectangle& bounds)
 {
-    ASSERT(HasHWND());
+    _isClippingCursor = true;
-
+    *(RECT*)_clipCursorRect = {
-    if (!_isClippingCursor)
+        (LONG)bounds.GetUpperLeft().X,
-    {
+        (LONG)bounds.GetUpperLeft().Y,
-        _isClippingCursor = true;
+        (LONG)bounds.GetBottomRight().X,
-    }
+        (LONG)bounds.GetBottomRight().Y
-
-    const RECT lpRect = { 
-        bounds.GetUpperLeft().X,
-        bounds.GetUpperLeft().Y,
-        bounds.GetBottomRight().X,
-        bounds.GetBottomRight().Y
     };
-    ClipCursor(&lpRect);
+    if (IsFocused())
+    {
+        _clipCursorSet = true;
+        ClipCursor((RECT*)_clipCursorRect);
+    }
 }
 
 void WindowsWindow::EndClippingCursor()
@@ -587,8 +584,8 @@ void WindowsWindow::EndClippingCursor()
     if (_isClippingCursor)
     {
         _isClippingCursor = false;
-
+        _clipCursorSet = false;
-        ClipCursor(NULL);
+        ClipCursor(nullptr);
     }
 }
 
@@ -1094,8 +1091,18 @@ LRESULT WindowsWindow::WndProc(UINT msg, WPARAM wParam, LPARAM lParam)
         break;
     case WM_SETFOCUS:
         OnGotFocus();
+        if (_isClippingCursor && !_clipCursorSet)
+        {
+            _clipCursorSet = true;
+            ClipCursor((RECT*)_clipCursorRect);
+        }
         break;
     case WM_KILLFOCUS:
+        if (_clipCursorSet)
+        {
+            _clipCursorSet = false;
+            ClipCursor(nullptr);
+        }
         OnLostFocus();
         break;
     case WM_ACTIVATEAPP:

Source/Engine/Platform/Windows/WindowsWindow.h

@@ -27,9 +27,11 @@ private:
     bool _isResizing = false;
     bool _isSwitchingFullScreen = false;
     bool _trackingMouse = false;
+    bool _clipCursorSet = false;
     bool _isDuringMaximize = false;
     Windows::HANDLE _monitor = nullptr;
     Float2 _clientSize;
+    Windows::LONG _clipCursorRect[4];
     int32 _regionWidth = 0, _regionHeight = 0;
 
 public: