Add ReplicationModes to NetworkTransform with interpolation and prediction modes

Source/Engine/Networking/Components/NetworkTransform.cpp

@@ -1,26 +1,62 @@
 // Copyright (c) 2012-2022 Wojciech Figat. All rights reserved.
 
+// Interpolation and prediction logic based on https://www.gabrielgambetta.com/client-server-game-architecture.html
+
 #include "NetworkTransform.h"
 #include "Engine/Core/Math/Transform.h"
+#include "Engine/Engine/Time.h"
 #include "Engine/Level/Actor.h"
+#include "Engine/Networking/NetworkManager.h"
+#include "Engine/Networking/NetworkPeer.h"
 #include "Engine/Networking/NetworkReplicator.h"
 #include "Engine/Networking/NetworkStream.h"
+#include "Engine/Networking/NetworkStats.h"
+#include "Engine/Networking/INetworkDriver.h"
+#include "Engine/Networking/NetworkRpc.h"
 
 PACK_STRUCT(struct Data
     {
     uint8 LocalSpace : 1;
-    NetworkTransform::SyncModes SyncMode : 9;
+    uint8 HasSequenceIndex : 1;
+    NetworkTransform::ReplicationComponents Components : 9;
     });
 
-static_assert((int32)NetworkTransform::SyncModes::All + 1 == 512, "Invalid SyncModes bit count for Data.");
+static_assert((int32)NetworkTransform::ReplicationComponents::All + 1 == 512, "Invalid ReplicationComponents bit count for Data.");
+
+namespace
+{
+    // Percentage of local error that is acceptable (eg. 4 frames error)
+    constexpr float Precision = 4.0f;
+
+    template<typename T>
+    FORCE_INLINE bool IsWithinPrecision(const Vector3Base<T>& currentDelta, const Vector3Base<T>& targetDelta)
+    {
+        const T targetDeltaMax = targetDelta.GetAbsolute().MaxValue();
+        return targetDeltaMax > (T)ZeroTolerance && currentDelta.GetAbsolute().MaxValue() < targetDeltaMax * (T)Precision;
+    }
+}
 
 NetworkTransform::NetworkTransform(const SpawnParams& params)
     : Script(params)
 {
+    // TODO: don't tick when using Default mode or with OwnedAuthoritative role to optimize cpu perf OR introduce TaskGraphSystem to batch NetworkTransform updates over Job System
+    _tickUpdate = 1;
+}
+
+void NetworkTransform::SetSequenceIndex(uint16 value)
+{
+    NETWORK_RPC_IMPL(NetworkTransform, SetSequenceIndex, value);
+    _currentSequenceIndex = value;
 }
 
 void NetworkTransform::OnEnable()
 {
+    // Initialize state
+    _bufferHasDeltas = false;
+    _currentSequenceIndex = 0.0f;
+    _lastFrameTransform = GetActor() ? GetActor()->GetTransform() : Transform::Identity;
+    _buffer.Clear();
+
     // Register for replication
     NetworkReplicator::AddObject(this);
 }
@@ -29,6 +65,81 @@ void NetworkTransform::OnDisable()
 {
     // Unregister from replication
     NetworkReplicator::RemoveObject(this);
+
+    _buffer.Resize(0);
+}
+
+void NetworkTransform::OnUpdate()
+{
+    // TODO: cache role in Deserialize to improve cpu perf
+    const NetworkObjectRole role = NetworkReplicator::GetObjectRole(this);
+    if (role == NetworkObjectRole::OwnedAuthoritative)
+        return; // Ignore itself
+    if (Mode == ReplicationModes::Default)
+    {
+        // Transform replicated in Deserialize
+    }
+    else if (role == NetworkObjectRole::ReplicatedSimulated && Mode == ReplicationModes::Prediction)
+    {
+        // Compute delta of the actor transformation simulated locally
+        const Transform thisFrameTransform = GetActor() ? GetActor()->GetTransform() : Transform::Identity;
+        Transform delta = thisFrameTransform - _lastFrameTransform;
+
+        if (!delta.IsIdentity())
+        {
+            // Move to the next input sequence number
+            _currentSequenceIndex++;
+
+            // Add delta to buffer to re-apply after receiving authoritative transform value
+            if (!_bufferHasDeltas)
+            {
+                _buffer.Clear();
+                _bufferHasDeltas = true;
+            }
+            delta.Orientation = thisFrameTransform.Orientation; // Store absolute orientation value to prevent jittering when blending rotation deltas
+            _buffer.Add({ 0.0f, _currentSequenceIndex, delta, });
+
+            // Inform server about sequence number change (add offset to lead before server data)
+            SetSequenceIndex(_currentSequenceIndex - 1);
+        }
+        _lastFrameTransform = thisFrameTransform;
+    }
+    else
+    {
+        float lag = 0.0f;
+        // TODO: use lag from last used NetworkStream context
+        if (NetworkManager::Peer && NetworkManager::Peer->NetworkDriver)
+        {
+            // Use lag from the RTT between server and the client
+            const auto stats = NetworkManager::Peer->NetworkDriver->GetStats();
+            lag = stats.RTT / 2000.0f;
+        }
+        else
+        {
+            // Default lag is based on the network manager update rate
+            const float fps = NetworkManager::NetworkFPS;
+            lag = 1.0f / fps;
+        }
+
+        // Find the two authoritative positions surrounding the rendering timestamp
+        const float now = Time::Update.UnscaledTime.GetTotalSeconds();
+        const float gameTime = now - lag;
+
+        // Drop older positions
+        while (_buffer.Count() >= 2 && _buffer[1].Timestamp <= gameTime)
+            _buffer.RemoveAtKeepOrder(0);
+
+        // Interpolate between the two surrounding authoritative positions
+        if (_buffer.Count() >= 2 && _buffer[0].Timestamp <= gameTime && gameTime <= _buffer[1].Timestamp)
+        {
+            const auto& b0 = _buffer[0];
+            const auto& b1 = _buffer[1];
+            Transform transform;
+            const float alpha = (gameTime - b0.Timestamp) / (b1.Timestamp - b0.Timestamp);
+            Transform::Lerp(b0.Value, b1.Value, alpha, transform);
+            Set(transform);
+        }
+    }
 }
 
 void NetworkTransform::Serialize(NetworkStream* stream)
@@ -41,58 +152,62 @@ void NetworkTransform::Serialize(NetworkStream* stream)
         transform = Transform::Identity;
 
     // Encode data
+    const NetworkObjectRole role = NetworkReplicator::GetObjectRole(this);
     Data data;
     data.LocalSpace = LocalSpace;
-    data.SyncMode = SyncMode;
+    data.HasSequenceIndex = Mode == ReplicationModes::Prediction;
+    data.Components = Components;
     stream->Write(data);
-    if ((data.SyncMode & SyncModes::All) == (int)SyncModes::All)
+    if ((data.Components & ReplicationComponents::All) == (int)ReplicationComponents::All)
     {
         stream->Write(transform);
     }
     else
     {
-        if ((data.SyncMode & SyncModes::Position) == (int)SyncModes::Position)
+        if ((data.Components & ReplicationComponents::Position) == (int)ReplicationComponents::Position)
         {
             stream->Write(transform.Translation);
         }
-        else if (data.SyncMode & SyncModes::Position)
+        else if (data.Components & ReplicationComponents::Position)
         {
-            if (data.SyncMode & SyncModes::PositionX)
+            if (data.Components & ReplicationComponents::PositionX)
                 stream->Write(transform.Translation.X);
-            if (data.SyncMode & SyncModes::PositionY)
+            if (data.Components & ReplicationComponents::PositionY)
                 stream->Write(transform.Translation.X);
-            if (data.SyncMode & SyncModes::PositionZ)
+            if (data.Components & ReplicationComponents::PositionZ)
                 stream->Write(transform.Translation.X);
         }
-        if ((data.SyncMode & SyncModes::Scale) == (int)SyncModes::Scale)
+        if ((data.Components & ReplicationComponents::Scale) == (int)ReplicationComponents::Scale)
         {
             stream->Write(transform.Scale);
         }
-        else if (data.SyncMode & SyncModes::Scale)
+        else if (data.Components & ReplicationComponents::Scale)
         {
-            if (data.SyncMode & SyncModes::ScaleX)
+            if (data.Components & ReplicationComponents::ScaleX)
                 stream->Write(transform.Scale.X);
-            if (data.SyncMode & SyncModes::ScaleY)
+            if (data.Components & ReplicationComponents::ScaleY)
                 stream->Write(transform.Scale.X);
-            if (data.SyncMode & SyncModes::ScaleZ)
+            if (data.Components & ReplicationComponents::ScaleZ)
                 stream->Write(transform.Scale.X);
         }
-        if ((data.SyncMode & SyncModes::Rotation) == (int)SyncModes::Rotation)
+        if ((data.Components & ReplicationComponents::Rotation) == (int)ReplicationComponents::Rotation)
         {
             const Float3 rotation = transform.Orientation.GetEuler();
             stream->Write(rotation);
         }
-        else if (data.SyncMode & SyncModes::Rotation)
+        else if (data.Components & ReplicationComponents::Rotation)
         {
             const Float3 rotation = transform.Orientation.GetEuler();
-            if (data.SyncMode & SyncModes::RotationX)
+            if (data.Components & ReplicationComponents::RotationX)
                 stream->Write(rotation.X);
-            if (data.SyncMode & SyncModes::RotationY)
+            if (data.Components & ReplicationComponents::RotationY)
                 stream->Write(rotation.Y);
-            if (data.SyncMode & SyncModes::RotationZ)
+            if (data.Components & ReplicationComponents::RotationZ)
                 stream->Write(rotation.Z);
         }
     }
+    if (data.HasSequenceIndex)
+        stream->Write(_currentSequenceIndex);
 }
 
 void NetworkTransform::Deserialize(NetworkStream* stream)
@@ -103,65 +218,133 @@ void NetworkTransform::Deserialize(NetworkStream* stream)
         transform = LocalSpace ? parent->GetLocalTransform() : parent->GetTransform();
     else
         transform = Transform::Identity;
+    Transform transformLocal = transform;
 
     // Decode data
     Data data;
     stream->Read(data);
-    if ((data.SyncMode & SyncModes::All) == (int)SyncModes::All)
+    if ((data.Components & ReplicationComponents::All) == (int)ReplicationComponents::All)
     {
         stream->Read(transform);
     }
     else
     {
-        if ((data.SyncMode & SyncModes::Position) == (int)SyncModes::Position)
+        if ((data.Components & ReplicationComponents::Position) == (int)ReplicationComponents::Position)
         {
             stream->Read(transform.Translation);
         }
-        else if (data.SyncMode & SyncModes::Position)
+        else if (data.Components & ReplicationComponents::Position)
         {
-            if (data.SyncMode & SyncModes::PositionX)
+            if (data.Components & ReplicationComponents::PositionX)
                 stream->Read(transform.Translation.X);
-            if (data.SyncMode & SyncModes::PositionY)
+            if (data.Components & ReplicationComponents::PositionY)
                 stream->Read(transform.Translation.X);
-            if (data.SyncMode & SyncModes::PositionZ)
+            if (data.Components & ReplicationComponents::PositionZ)
                 stream->Read(transform.Translation.X);
         }
-        if ((data.SyncMode & SyncModes::Scale) == (int)SyncModes::Scale)
+        if ((data.Components & ReplicationComponents::Scale) == (int)ReplicationComponents::Scale)
         {
             stream->Read(transform.Scale);
         }
-        else if (data.SyncMode & SyncModes::Scale)
+        else if (data.Components & ReplicationComponents::Scale)
         {
-            if (data.SyncMode & SyncModes::ScaleX)
+            if (data.Components & ReplicationComponents::ScaleX)
                 stream->Read(transform.Scale.X);
-            if (data.SyncMode & SyncModes::ScaleY)
+            if (data.Components & ReplicationComponents::ScaleY)
                 stream->Read(transform.Scale.X);
-            if (data.SyncMode & SyncModes::ScaleZ)
+            if (data.Components & ReplicationComponents::ScaleZ)
                 stream->Read(transform.Scale.X);
         }
-        if ((data.SyncMode & SyncModes::Rotation) == (int)SyncModes::Rotation)
+        if ((data.Components & ReplicationComponents::Rotation) == (int)ReplicationComponents::Rotation)
         {
             Float3 rotation;
             stream->Read(rotation);
             transform.Orientation = Quaternion::Euler(rotation);
         }
-        else if (data.SyncMode & SyncModes::Rotation)
+        else if (data.Components & ReplicationComponents::Rotation)
         {
             Float3 rotation = transform.Orientation.GetEuler();
-            if (data.SyncMode & SyncModes::RotationX)
+            if (data.Components & ReplicationComponents::RotationX)
                 stream->Read(rotation.X);
-            if (data.SyncMode & SyncModes::RotationY)
+            if (data.Components & ReplicationComponents::RotationY)
                 stream->Read(rotation.Y);
-            if (data.SyncMode & SyncModes::RotationZ)
+            if (data.Components & ReplicationComponents::RotationZ)
                 stream->Read(rotation.Z);
             transform.Orientation = Quaternion::Euler(rotation);
         }
     }
+    uint16 sequenceIndex = 0;
+    if (data.HasSequenceIndex)
+        stream->Read(sequenceIndex);
+    if (data.LocalSpace != LocalSpace)
+        return; // TODO: convert transform space if server-client have different values set
 
-    // Set transform
+    const NetworkObjectRole role = NetworkReplicator::GetObjectRole(this);
+    if (role == NetworkObjectRole::OwnedAuthoritative)
+        return; // Ignore itself
+    if (Mode == ReplicationModes::Default)
+    {
+        // Immediate set
+        Set(transform);
+    }
+    else if (role == NetworkObjectRole::ReplicatedSimulated && Mode == ReplicationModes::Prediction)
+    {
+        const Transform transformAuthoritative = transform;
+        const Transform transformDeltaBefore = transformAuthoritative - transformLocal;
+
+        // Remove any transform deltas from local simulation that happened before the incoming authoritative transform data
+        if (!_bufferHasDeltas)
+        {
+            _buffer.Clear();
+            _bufferHasDeltas = true;
+        }
+        // TODO: items are added in order to do batch removal
+        for (int32 i = 0; i < _buffer.Count() && _buffer[i].SequenceIndex < sequenceIndex; i++)
+        {
+            _buffer.RemoveAtKeepOrder(i);
+        }
+
+        // Use received authoritative actor transformation but re-apply all deltas not yet processed by the server due to lag (reconciliation)
+        for (auto& e : _buffer)
+        {
+            transform.Translation = transform.Translation + e.Value.Translation;
+            transform.Scale = transform.Scale * e.Value.Scale;
+        }
+        // TODO: use euler angles or similar to cache/reapply rotation deltas (Quaternion jitters)
+        transform.Orientation = transformLocal.Orientation;
+
+        // If local simulation is very close to the authoritative server value then ignore slight error (based relative delta threshold)
+        const Transform transformDeltaAfter = transformAuthoritative - transformLocal;
+        const Transform transformDeltaDelta = transformDeltaAfter - transformDeltaBefore;
+        if (IsWithinPrecision(transformDeltaDelta.Translation, transformDeltaAfter.Translation) &&
+            IsWithinPrecision(transformDeltaDelta.Scale, transformDeltaAfter.Scale)
+        )
+        {
+            return;
+        }
+
+        // Set to the incoming value with applied local deltas
+        Set(transform);
+        _lastFrameTransform = transform;
+    }
+    else
+    {
+        // Add to the interpolation buffer
+        const float now = Time::Update.UnscaledTime.GetTotalSeconds();
+        _buffer.Add({ now, 0, transform });
+        if (_bufferHasDeltas)
+        {
+            _buffer.Clear();
+            _bufferHasDeltas = false;
+        }
+    }
+}
+
+void NetworkTransform::Set(const Transform& transform)
+{
     if (auto* parent = GetParent())
     {
-        if (data.LocalSpace)
+        if (LocalSpace)
             parent->SetLocalTransform(transform);
         else
             parent->SetTransform(transform);

Source/Engine/Networking/Components/NetworkTransform.h

@@ -3,20 +3,22 @@
 #pragma once
 
 #include "Engine/Scripting/Script.h"
+#include "Engine/Core/Math/Transform.h"
 #include "Engine/Networking/INetworkSerializable.h"
 
 /// <summary>
 /// Actor script component that synchronizes the Transform over the network.
 /// </summary>
+/// <remarks>Interpolation and prediction logic based on https://www.gabrielgambetta.com/client-server-game-architecture.html.</remarks>
 API_CLASS(Namespace="FlaxEngine.Networking") class FLAXENGINE_API NetworkTransform : public Script, public INetworkSerializable
 {
     API_AUTO_SERIALIZATION();
     DECLARE_SCRIPTING_TYPE(NetworkTransform);
 
     /// <summary>
-    /// Actor transform synchronization modes (flags).
+    /// Actor transform replication components (flags).
     /// </summary>
-    API_ENUM(Attributes="Flags") enum class SyncModes
+    API_ENUM(Attributes="Flags") enum class ReplicationComponents
     {
         // No sync.
         None = 0,
@@ -52,27 +54,66 @@ API_CLASS(Namespace="FlaxEngine.Networking") class FLAXENGINE_API NetworkTransfo
         All = Position | Scale | Rotation,
     };
 
+    /// <summary>
+    /// Actor transform replication modes.
+    /// </summary>
+    API_ENUM() enum class ReplicationModes
+    {
+        // The transform replicated from the owner (raw replication data messages that might result in sudden object jumps when moving).
+        Default,
+        // The transform replicated from the owner with local interpolation between received data to provide smoother movement.
+        Interpolation,
+        // The transform replicated from the owner but with local prediction (eg. player character that has local simulation but is validated against authoritative server).
+        Prediction,
+    };
+
+private:
+    struct BufferedItem
+    {
+        float Timestamp;
+        uint16 SequenceIndex;
+        Transform Value;
+    };
+
+    bool _bufferHasDeltas;
+    uint16 _currentSequenceIndex = 0;
+    Transform _lastFrameTransform;
+    Array<BufferedItem> _buffer;
+
 public:
     /// <summary>
     /// If checked, actor transform will be synchronized in local space of the parent actor (otherwise in world space).
     /// </summary>
     API_FIELD(Attributes="EditorOrder(10)")
     bool LocalSpace = false;
-
+    
     /// <summary>
-    /// Actor transform synchronization mode (flags).
+    /// Actor transform replication components (flags).
     /// </summary>
     API_FIELD(Attributes="EditorOrder(20)")
-    SyncModes SyncMode = SyncModes::All;
+    ReplicationComponents Components = ReplicationComponents::All;
 
+    /// <summary>
+    /// Actor transform replication mode.
+    /// </summary>
+    API_FIELD(Attributes="EditorOrder(30)")
+    ReplicationModes Mode = ReplicationModes::Default;
+
+private:
+    API_FUNCTION(Hidden, NetworkRpc=Server) void SetSequenceIndex(uint16 value);
+    
 public:
     // [Script]
     void OnEnable() override;
     void OnDisable() override;
+    void OnUpdate() override;
 
     // [INetworkSerializable]
     void Serialize(NetworkStream* stream) override;
     void Deserialize(NetworkStream* stream) override;
+
+private:
+    void Set(const Transform& transform);
 };
 
-DECLARE_ENUM_OPERATORS(NetworkTransform::SyncModes);
+DECLARE_ENUM_OPERATORS(NetworkTransform::ReplicationComponents);

Source/Engine/Networking/NetworkRpc.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include "Engine/Core/Log.h"
 #include "Engine/Core/Types/StringView.h"
 #include "Engine/Core/Types/Pair.h"
 #include "Engine/Core/Collections/Array.h"

Source/Engine/Networking/Networking.Build.cs

@@ -13,6 +13,7 @@ public class Networking : EngineModule
     {
         base.Setup(options);
 
+        Tags["Network"] = string.Empty;
         options.PublicDefinitions.Add("COMPILE_WITH_NETWORKING");
     }
 }