Add **Network Replication Hierarchy** for robust control over replication in multiplayer games

Source/Engine/Networking/NetworkReplicationHierarchy.cpp

@@ -0,0 +1,199 @@
+// Copyright (c) 2012-2023 Wojciech Figat. All rights reserved.
+
+#include "NetworkReplicationHierarchy.h"
+#include "NetworkManager.h"
+#include "Engine/Level/Actor.h"
+#include "Engine/Level/SceneObject.h"
+
+uint16 NetworkReplicationNodeObjectCounter = 0;
+NetworkClientsMask NetworkClientsMask::All = { MAX_uint64, MAX_uint64 };
+
+Actor* NetworkReplicationHierarchyObject::GetActor() const
+{
+    auto* actor = ScriptingObject::Cast<Actor>(Object);
+    if (!actor)
+    {
+        if (const auto* sceneObject = ScriptingObject::Cast<SceneObject>(Object))
+            actor = sceneObject->GetParent();
+    }
+    return actor;
+}
+
+void NetworkReplicationHierarchyUpdateResult::Init()
+{
+    _clientsHaveLocation = false;
+    _clients.Resize(NetworkManager::Clients.Count());
+    _clientsMask = NetworkClientsMask();
+    for (int32 i = 0; i < _clients.Count(); i++)
+        _clientsMask.SetBit(i);
+    _entries.Clear();
+    ReplicationScale = 1.0f;
+}
+
+void NetworkReplicationHierarchyUpdateResult::SetClientLocation(int32 clientIndex, const Vector3& location)
+{
+    CHECK(clientIndex >= 0 && clientIndex < _clients.Count());
+    _clientsHaveLocation = true;
+    Client& client = _clients[clientIndex];
+    client.HasLocation = true;
+    client.Location = location;
+}
+
+bool NetworkReplicationHierarchyUpdateResult::GetClientLocation(int32 clientIndex, Vector3& location) const
+{
+    CHECK_RETURN(clientIndex >= 0 && clientIndex < _clients.Count(), false);
+    const Client& client = _clients[clientIndex];
+    location = client.Location;
+    return client.HasLocation;
+}
+
+void NetworkReplicationNode::AddObject(NetworkReplicationHierarchyObject obj)
+{
+    ASSERT(obj.Object && obj.ReplicationFPS > 0.0f);
+
+    // Randomize initial replication update to spread rep rates more evenly for large scenes that register all objects within the same frame
+    obj.ReplicationUpdatesLeft = NetworkReplicationNodeObjectCounter++ % Math::Clamp(Math::RoundToInt(NetworkManager::NetworkFPS / obj.ReplicationFPS), 1, 60);
+
+    Objects.Add(obj);
+}
+
+bool NetworkReplicationNode::RemoveObject(ScriptingObject* obj)
+{
+    return !Objects.Remove(obj);
+}
+
+bool NetworkReplicationNode::DirtyObject(ScriptingObject* obj)
+{
+    const int32 index = Objects.Find(obj);
+    if (index != -1)
+    {
+        NetworkReplicationHierarchyObject& e = Objects[index];
+        e.ReplicationUpdatesLeft = 0;
+    }
+    return index != -1;
+}
+
+void NetworkReplicationNode::Update(NetworkReplicationHierarchyUpdateResult* result)
+{
+    CHECK(result);
+    const float networkFPS = NetworkManager::NetworkFPS / result->ReplicationScale;
+    for (NetworkReplicationHierarchyObject& obj : Objects)
+    {
+        if (obj.ReplicationUpdatesLeft > 0)
+        {
+            // Move to the next frame
+            obj.ReplicationUpdatesLeft--;
+        }
+        else
+        {
+            NetworkClientsMask targetClients = result->GetClientsMask();
+            if (result->_clientsHaveLocation)
+            {
+                // Cull object against viewers locations
+                if (const Actor* actor = obj.GetActor())
+                {
+                    const Vector3 objPosition = actor->GetPosition();
+                    const Real cullDistanceSq = Math::Square(obj.CullDistance);
+                    for (int32 clientIndex = 0; clientIndex < result->_clients.Count(); clientIndex++)
+                    {
+                        const auto& client = result->_clients[clientIndex];
+                        if (client.HasLocation)
+                        {
+                            const Real distanceSq = Vector3::DistanceSquared(objPosition, client.Location);
+                            // TODO: scale down replication FPS when object is far away from all clients (eg. by 10-50%)
+                            if (distanceSq >= cullDistanceSq)
+                            {
+                                // Object is too far from this viewer so don't send data to him
+                                targetClients.UnsetBit(clientIndex);
+                            }
+                        }
+                    }
+                }
+            }
+            if (targetClients)
+            {
+                // Replicate this frame
+                result->AddObject(obj.Object, targetClients);
+            }
+
+            // Calculate frames until next replication
+            obj.ReplicationUpdatesLeft = (uint16)Math::Clamp<int32>(Math::RoundToInt(networkFPS / obj.ReplicationFPS) - 1, 0, MAX_uint16);
+        }
+    }
+}
+
+NetworkReplicationGridNode::~NetworkReplicationGridNode()
+{
+    for (const auto& e : _children)
+        Delete(e.Value.Node);
+}
+
+void NetworkReplicationGridNode::AddObject(NetworkReplicationHierarchyObject obj)
+{
+    // Chunk actors locations into a grid coordinates
+    Int3 coord = Int3::Zero;
+    if (const Actor* actor = obj.GetActor())
+    {
+        coord = actor->GetPosition() / CellSize;
+    }
+
+    Cell* cell = _children.TryGet(coord);
+    if (!cell)
+    {
+        // Allocate new cell
+        cell = &_children[coord];
+        cell->Node = New<NetworkReplicationNode>();
+        cell->MinCullDistance = obj.CullDistance;
+    }
+    cell->Node->AddObject(obj);
+
+    // Cache minimum culling distance for a whole cell to skip it at once
+    cell->MinCullDistance = Math::Min(cell->MinCullDistance, obj.CullDistance);
+}
+
+bool NetworkReplicationGridNode::RemoveObject(ScriptingObject* obj)
+{
+    for (const auto& e : _children)
+    {
+        if (e.Value.Node->RemoveObject(obj))
+        {
+            // TODO: remove empty cells?
+            // TODO: update MinCullDistance for cell?
+            return true;
+        }
+    }
+    return false;
+}
+
+void NetworkReplicationGridNode::Update(NetworkReplicationHierarchyUpdateResult* result)
+{
+    CHECK(result);
+    if (result->_clientsHaveLocation)
+    {
+        // Update only cells within a range
+        const Real cellRadiusSq = Math::Square(CellSize * 1.414f);
+        for (const auto& e : _children)
+        {
+            const Vector3 cellPosition = (e.Key * CellSize) + (CellSize * 0.5f);
+            Real distanceSq = MAX_Real;
+            for (auto& client : result->_clients)
+            {
+                if (client.HasLocation)
+                    distanceSq = Math::Min(distanceSq, Vector3::DistanceSquared(cellPosition, client.Location));
+            }
+            const Real minCullDistanceSq = Math::Square(e.Value.MinCullDistance);
+            if (distanceSq < minCullDistanceSq + cellRadiusSq)
+            {
+                e.Value.Node->Update(result);
+            }
+        }
+    }
+    else
+    {
+        // Brute-force over all cells
+        for (const auto& e : _children)
+        {
+            e.Value.Node->Update(result);
+        }
+    }
+}

Source/Engine/Networking/NetworkReplicationHierarchy.cs

@@ -0,0 +1,25 @@
+// Copyright (c) 2012-2023 Wojciech Figat. All rights reserved.
+
+namespace FlaxEngine.Networking
+{
+    partial struct NetworkReplicationHierarchyObject
+    {
+        /// <summary>
+        /// Gets the actors context (object itself or parent actor).
+        /// </summary>
+        public Actor Actor
+        {
+            get
+            {
+                var actor = Object as Actor;
+                if (actor == null)
+                {
+                    var sceneObject = Object as SceneObject;
+                    if (sceneObject != null)
+                        actor = sceneObject.Parent;
+                }
+                return actor;
+            }
+        }
+    }
+}

Source/Engine/Networking/NetworkReplicationHierarchy.h

@@ -0,0 +1,258 @@
+// Copyright (c) 2012-2023 Wojciech Figat. All rights reserved.
+
+#pragma once
+
+#include "Types.h"
+#include "Engine/Core/Math/Vector3.h"
+#include "Engine/Core/Collections/Array.h"
+#include "Engine/Core/Collections/Dictionary.h"
+#include "Engine/Scripting/ScriptingObject.h"
+#include "Engine/Scripting/ScriptingObjectReference.h"
+
+class Actor;
+
+/// <summary>
+/// Network replication hierarchy object data.
+/// </summary>
+API_STRUCT(NoDefault, Namespace = "FlaxEngine.Networking") struct FLAXENGINE_API NetworkReplicationHierarchyObject
+{
+    DECLARE_SCRIPTING_TYPE_MINIMAL(NetworkReplicationObjectInfo);
+    // The object to replicate.
+    API_FIELD() ScriptingObject* Object;
+    // The target amount of the replication updates per second (frequency of the replication). Constrained by NetworkManager::NetworkFPS.
+    API_FIELD() float ReplicationFPS = 60;
+    // The minimum distance from the player to the object at which it can process replication. For example, players further away won't receive object data.
+    API_FIELD() float CullDistance = 15000;
+    // Runtime value for update frames left for the next replication of this object. Matches NetworkManager::NetworkFPS calculated from ReplicationFPS.
+    API_FIELD(Attributes="HideInEditor") uint16 ReplicationUpdatesLeft = 0;
+
+    FORCE_INLINE NetworkReplicationHierarchyObject(const ScriptingObjectReference<ScriptingObject>& obj)
+        : Object(obj.Get())
+    {
+    }
+
+    FORCE_INLINE NetworkReplicationHierarchyObject(ScriptingObject* obj = nullptr)
+        : Object(obj)
+    {
+    }
+
+    // Gets the actors context (object itself or parent actor).
+    Actor* GetActor() const;
+
+    bool operator==(const NetworkReplicationHierarchyObject& other) const
+    {
+        return Object == other.Object;
+    }
+
+    bool operator==(const ScriptingObject* other) const
+    {
+        return Object == other;
+    }
+};
+
+inline uint32 GetHash(const NetworkReplicationHierarchyObject& key)
+{
+    return GetHash(key.Object);
+}
+
+/// <summary>
+/// Bit mask for NetworkClient list (eg. to selectively send object replication).
+/// </summary>
+API_STRUCT(NoDefault, Namespace = "FlaxEngine.Networking") struct FLAXENGINE_API NetworkClientsMask
+{
+    DECLARE_SCRIPTING_TYPE_MINIMAL(NetworkClientsMask);
+    // The first 64 bits (each for one client).
+    API_FIELD() uint64 Word0 = 0;
+    // The second 64 bits (each for one client).
+    API_FIELD() uint64 Word1 = 0;
+
+    // All bits set for all clients.
+    API_FIELD() static NetworkClientsMask All;
+
+    FORCE_INLINE bool HasBit(int32 bitIndex) const
+    {
+        const int32 wordIndex = bitIndex / 64;
+        const uint64 wordMask = 1ull << (uint64)(bitIndex - wordIndex * 64);
+        const uint64 word = *(&Word0 + wordIndex);
+        return (word & wordMask) == wordMask;
+    }
+
+    FORCE_INLINE void SetBit(int32 bitIndex)
+    {
+        const int32 wordIndex = bitIndex / 64;
+        const uint64 wordMask = 1ull << (uint64)(bitIndex - wordIndex * 64);
+        uint64& word = *(&Word0 + wordIndex);
+        word |= wordMask;
+    }
+
+    FORCE_INLINE void UnsetBit(int32 bitIndex)
+    {
+        const int32 wordIndex = bitIndex / 64;
+        const uint64 wordMask = 1ull << (uint64)(bitIndex - wordIndex * 64);
+        uint64& word = *(&Word0 + wordIndex);
+        word &= ~wordMask;
+    }
+
+    FORCE_INLINE operator bool() const
+    {
+        return Word0 + Word1 != 0;
+    }
+
+    bool operator==(const NetworkClientsMask& other) const
+    {
+        return Word0 == other.Word0 && Word1 == other.Word1;
+    }
+};
+
+/// <summary>
+/// Network replication hierarchy output data to send.
+/// </summary>
+API_CLASS(Namespace = "FlaxEngine.Networking") class FLAXENGINE_API NetworkReplicationHierarchyUpdateResult : public ScriptingObject
+{
+    DECLARE_SCRIPTING_TYPE_WITH_CONSTRUCTOR_IMPL(NetworkReplicationHierarchyUpdateResult, ScriptingObject);
+    friend class NetworkInternal;
+    friend class NetworkReplicationNode;
+    friend class NetworkReplicationGridNode;
+
+private:
+    struct Client
+    {
+        bool HasLocation;
+        Vector3 Location;
+    };
+
+    struct Entry
+    {
+        ScriptingObject* Object;
+        NetworkClientsMask TargetClients;
+    };
+
+    bool _clientsHaveLocation;
+    NetworkClientsMask _clientsMask;
+    Array<Client> _clients;
+    Array<Entry> _entries;
+
+    void Init();
+
+public:
+    // Scales the ReplicationFPS property of objects in hierarchy. Can be used to slow down or speed up replication rate.
+    API_FIELD() float ReplicationScale = 1.0f;
+
+    // Adds object to the update results.
+    API_FUNCTION() void AddObject(ScriptingObject* obj)
+    {
+        Entry& e = _entries.AddOne();
+        e.Object = obj;
+        e.TargetClients = NetworkClientsMask::All;
+    }
+
+    // Adds object to the update results. Defines specific clients to receive the update (server-only, unused on client). Mask matches NetworkManager::Clients.
+    API_FUNCTION() void AddObject(ScriptingObject* obj, NetworkClientsMask targetClients)
+    {
+        Entry& e = _entries.AddOne();
+        e.Object = obj;
+        e.TargetClients = targetClients;
+    }
+
+    // Gets amount of the clients to use. Matches NetworkManager::Clients.
+    API_PROPERTY() int32 GetClientsCount() const
+    {
+        return _clients.Count();
+    }
+
+    // Gets mask with all client bits set. Matches NetworkManager::Clients.
+    API_PROPERTY() NetworkClientsMask GetClientsMask() const
+    {
+        return _clientsMask;
+    }
+
+    // Sets the viewer location for a certain client. Client index must match NetworkManager::Clients.
+    API_FUNCTION() void SetClientLocation(int32 clientIndex, const Vector3& location);
+
+    // Gets the viewer location for a certain client. Client index must match NetworkManager::Clients. Returns true if got a location set, otherwise false.
+    API_FUNCTION() bool GetClientLocation(int32 clientIndex, API_PARAM(out) Vector3& location) const;
+};
+
+/// <summary>
+/// Base class for the network objects replication hierarchy nodes. Contains a list of objects.
+/// </summary>
+API_CLASS(Abstract, Namespace = "FlaxEngine.Networking") class FLAXENGINE_API NetworkReplicationNode : public ScriptingObject
+{
+    DECLARE_SCRIPTING_TYPE_WITH_CONSTRUCTOR_IMPL(NetworkReplicationNode, ScriptingObject);
+
+    /// <summary>
+    /// List with objects stored in this node.
+    /// </summary>
+    API_FIELD() Array<NetworkReplicationHierarchyObject> Objects;
+
+    /// <summary>
+    /// Adds an object into the hierarchy.
+    /// </summary>
+    /// <param name="obj">The object to add.</param>
+    API_FUNCTION() virtual void AddObject(NetworkReplicationHierarchyObject obj);
+
+    /// <summary>
+    /// Removes object from the hierarchy.
+    /// </summary>
+    /// <param name="obj">The object to remove.</param>
+    /// <returns>True on successful removal, otherwise false.</returns>
+    API_FUNCTION() virtual bool RemoveObject(ScriptingObject* obj);
+
+    /// <summary>
+    /// Force replicates the object during the next update. Resets any internal tracking state to force the synchronization.
+    /// </summary>
+    /// <param name="obj">The object to update.</param>
+    /// <returns>True on successful update, otherwise false.</returns>
+    API_FUNCTION() virtual bool DirtyObject(ScriptingObject* obj);
+
+    /// <summary>
+    /// Iterates over all objects and adds them to the replication work.
+    /// </summary>
+    /// <param name="result">The update results container.</param>
+    API_FUNCTION() virtual void Update(NetworkReplicationHierarchyUpdateResult* result);
+};
+
+inline uint32 GetHash(const Int3& key)
+{
+    uint32 hash = GetHash(key.X);
+    CombineHash(hash, GetHash(key.Y));
+    CombineHash(hash, GetHash(key.Z));
+    return hash;
+}
+
+/// <summary>
+/// Network replication hierarchy node with 3D grid spatialization. Organizes static objects into chunks to improve performance in large worlds.
+/// </summary>
+API_CLASS(Namespace = "FlaxEngine.Networking") class FLAXENGINE_API NetworkReplicationGridNode : public NetworkReplicationNode
+{
+    DECLARE_SCRIPTING_TYPE_WITH_CONSTRUCTOR_IMPL(NetworkReplicationGridNode, NetworkReplicationNode);
+    ~NetworkReplicationGridNode();
+
+private:
+    struct Cell
+    {
+        NetworkReplicationNode* Node;
+        float MinCullDistance;
+    };
+
+    Dictionary<Int3, Cell> _children;
+
+public:
+    /// <summary>
+    /// Size of the grid cell (in world units). Used to chunk the space for separate nodes.
+    /// </summary>
+    API_FIELD() float CellSize = 10000.0f;
+
+    void AddObject(NetworkReplicationHierarchyObject obj) override;
+    bool RemoveObject(ScriptingObject* obj) override;
+    void Update(NetworkReplicationHierarchyUpdateResult* result) override;
+};
+
+/// <summary>
+/// Defines the network objects replication hierarchy (tree structure) that controls chunking and configuration of the game objects replication.
+/// Contains only 'owned' objects. It's used by the networking system only on a main thread.
+/// </summary>
+API_CLASS(Namespace = "FlaxEngine.Networking") class FLAXENGINE_API NetworkReplicationHierarchy : public NetworkReplicationNode
+{
+    DECLARE_SCRIPTING_TYPE_WITH_CONSTRUCTOR_IMPL(NetworkReplicationHierarchy, NetworkReplicationNode);
+};

Source/Engine/Networking/NetworkReplicator.cpp

@@ -12,6 +12,7 @@
 #include "NetworkRpc.h"
 #include "INetworkSerializable.h"
 #include "INetworkObject.h"
+#include "NetworkReplicationHierarchy.h"
 #include "Engine/Core/Collections/HashSet.h"
 #include "Engine/Core/Collections/Dictionary.h"
 #include "Engine/Core/Collections/ChunkedArray.h"
@@ -199,6 +200,8 @@ namespace
     Dictionary<Guid, Guid> IdsRemappingTable;
     NetworkStream* CachedWriteStream = nullptr;
     NetworkStream* CachedReadStream = nullptr;
+    NetworkReplicationHierarchyUpdateResult* CachedReplicationResult = nullptr;
+    NetworkReplicationHierarchy* Hierarchy = nullptr;
     Array<NetworkClient*> NewClients;
     Array<NetworkConnection> CachedTargets;
     Dictionary<ScriptingTypeHandle, Serializer> SerializersTable;
@@ -307,14 +310,15 @@ void BuildCachedTargets(const Array<NetworkClient*>& clients, const NetworkClien
     }
 }
 
-void BuildCachedTargets(const Array<NetworkClient*>& clients, const DataContainer<uint32>& clientIds, const uint32 excludedClientId = NetworkManager::ServerClientId)
+void BuildCachedTargets(const Array<NetworkClient*>& clients, const DataContainer<uint32>& clientIds, const uint32 excludedClientId = NetworkManager::ServerClientId, const NetworkClientsMask clientsMask = NetworkClientsMask::All)
 {
     CachedTargets.Clear();
     if (clientIds.IsValid())
     {
-        for (const NetworkClient* client : clients)
+        for (int32 clientIndex = 0; clientIndex < clients.Count(); clientIndex++)
         {
-            if (client->State == NetworkConnectionState::Connected && client->ClientId != excludedClientId)
+            const NetworkClient* client = clients.Get()[clientIndex];
+            if (client->State == NetworkConnectionState::Connected && client->ClientId != excludedClientId && clientsMask.HasBit(clientIndex))
             {
                 for (int32 i = 0; i < clientIds.Length(); i++)
                 {
@@ -329,9 +333,10 @@ void BuildCachedTargets(const Array<NetworkClient*>& clients, const DataContaine
     }
     else
     {
-        for (const NetworkClient* client : clients)
+        for (int32 clientIndex = 0; clientIndex < clients.Count(); clientIndex++)
         {
-            if (client->State == NetworkConnectionState::Connected && client->ClientId != excludedClientId)
+            const NetworkClient* client = clients.Get()[clientIndex];
+            if (client->State == NetworkConnectionState::Connected && client->ClientId != excludedClientId && clientsMask.HasBit(clientIndex))
                 CachedTargets.Add(client->Connection);
         }
     }
@@ -377,10 +382,10 @@ void BuildCachedTargets(const Array<NetworkClient*>& clients, const DataContaine
     }
 }
 
-FORCE_INLINE void BuildCachedTargets(const NetworkReplicatedObject& item)
+FORCE_INLINE void BuildCachedTargets(const NetworkReplicatedObject& item, const NetworkClientsMask clientsMask = NetworkClientsMask::All)
 {
     // By default send object to all connected clients excluding the owner but with optional TargetClientIds list
-    BuildCachedTargets(NetworkManager::Clients, item.TargetClientIds, item.OwnerClientId);
+    BuildCachedTargets(NetworkManager::Clients, item.TargetClientIds, item.OwnerClientId, clientsMask);
 }
 
 FORCE_INLINE void GetNetworkName(char buffer[128], const StringAnsiView& name)
@@ -561,9 +566,10 @@ void FindObjectsForSpawn(SpawnGroup& group, ChunkedArray<SpawnItem, 256>& spawnI
     }
 }
 
-void DirtyObjectImpl(NetworkReplicatedObject& item, ScriptingObject* obj)
+FORCE_INLINE void DirtyObjectImpl(NetworkReplicatedObject& item, ScriptingObject* obj)
 {
-    // TODO: implement objects state replication frequency and dirtying
+    if (Hierarchy)
+        Hierarchy->DirtyObject(obj);
 }
 
 template<typename MessageType>
@@ -703,6 +709,34 @@ StringAnsiView NetworkReplicator::GetCSharpCachedName(const StringAnsiView& name
 
 #endif
 
+NetworkReplicationHierarchy* NetworkReplicator::GetHierarchy()
+{
+    return Hierarchy;
+}
+
+void NetworkReplicator::SetHierarchy(NetworkReplicationHierarchy* value)
+{
+    ScopeLock lock(ObjectsLock);
+    if (Hierarchy == value)
+        return;
+    NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Set hierarchy to '{}'", value ? value->ToString() : String::Empty);
+    if (Hierarchy)
+    {
+        // Clear old hierarchy
+        Delete(Hierarchy);
+    }
+    Hierarchy = value;
+    if (value)
+    {
+        // Add all owned objects to the hierarchy
+        for (auto& e : Objects)
+        {
+            if (e.Item.Object && e.Item.Role == NetworkObjectRole::OwnedAuthoritative)
+                value->AddObject(e.Item.Object);
+        }
+    }
+}
+
 void NetworkReplicator::AddSerializer(const ScriptingTypeHandle& typeHandle, SerializeFunc serialize, SerializeFunc deserialize, void* serializeTag, void* deserializeTag)
 {
     if (!typeHandle)
@@ -788,6 +822,8 @@ void NetworkReplicator::AddObject(ScriptingObject* obj, const ScriptingObject* p
         }
     }
     Objects.Add(MoveTemp(item));
+    if (Hierarchy && item.Role == NetworkObjectRole::OwnedAuthoritative)
+        Hierarchy->AddObject(obj);
 }
 
 void NetworkReplicator::RemoveObject(ScriptingObject* obj)
@@ -801,6 +837,8 @@ void NetworkReplicator::RemoveObject(ScriptingObject* obj)
 
     // Remove object from the list
     NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Remove object {}, owned by {}", obj->GetID().ToString(), it->Item.ParentId.ToString());
+    if (Hierarchy && it->Item.Role == NetworkObjectRole::OwnedAuthoritative)
+        Hierarchy->RemoveObject(obj);
     Objects.Remove(it);
 }
 
@@ -870,6 +908,8 @@ void NetworkReplicator::DespawnObject(ScriptingObject* obj)
     DespawnedObjects.Add(item.ObjectId);
     if (item.AsNetworkObject)
         item.AsNetworkObject->OnNetworkDespawn();
+    if (Hierarchy && item.Role == NetworkObjectRole::OwnedAuthoritative)
+        Hierarchy->RemoveObject(obj);
     Objects.Remove(it);
     DeleteNetworkObject(obj);
 }
@@ -1004,6 +1044,8 @@ void NetworkReplicator::SetObjectOwnership(ScriptingObject* obj, uint32 ownerCli
             {
                 // Change role locally
                 CHECK(localRole != NetworkObjectRole::OwnedAuthoritative);
+                if (Hierarchy && item.Role == NetworkObjectRole::OwnedAuthoritative)
+                    Hierarchy->RemoveObject(obj);
                 item.OwnerClientId = ownerClientId;
                 item.LastOwnerFrame = 1;
                 item.Role = localRole;
@@ -1014,6 +1056,8 @@ void NetworkReplicator::SetObjectOwnership(ScriptingObject* obj, uint32 ownerCli
         {
             // Allow to change local role of the object (except ownership)
             CHECK(localRole != NetworkObjectRole::OwnedAuthoritative);
+            if (Hierarchy && it->Item.Role == NetworkObjectRole::OwnedAuthoritative)
+                Hierarchy->RemoveObject(obj);
             item.Role = localRole;
         }
     }
@@ -1107,6 +1151,8 @@ void NetworkInternal::NetworkReplicatorClientDisconnected(NetworkClient* client)
 
             // Delete object locally
             NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Despawn object {}", item.ObjectId);
+            if (Hierarchy && item.Role == NetworkObjectRole::OwnedAuthoritative)
+                Hierarchy->RemoveObject(obj);
             if (item.AsNetworkObject)
                 item.AsNetworkObject->OnNetworkDespawn();
             DeleteNetworkObject(obj);
@@ -1121,6 +1167,7 @@ void NetworkInternal::NetworkReplicatorClear()
 
     // Cleanup
     NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Shutdown");
+    NetworkReplicator::SetHierarchy(nullptr);
     for (auto it = Objects.Begin(); it.IsNotEnd(); ++it)
     {
         auto& item = it->Item;
@@ -1140,6 +1187,7 @@ void NetworkInternal::NetworkReplicatorClear()
     IdsRemappingTable.Clear();
     SAFE_DELETE(CachedWriteStream);
     SAFE_DELETE(CachedReadStream);
+    SAFE_DELETE(CachedReplicationResult);
     NewClients.Clear();
     CachedTargets.Clear();
     DespawnedObjects.Clear();
@@ -1268,7 +1316,14 @@ void NetworkInternal::NetworkReplicatorUpdate()
 
             if (e.HasOwnership)
             {
-                item.Role = e.Role;
+                if (item.Role != e.Role)
+                {
+                    if (Hierarchy && item.Role == NetworkObjectRole::OwnedAuthoritative)
+                        Hierarchy->RemoveObject(obj);
+                    item.Role = e.Role;
+                    if (Hierarchy && item.Role == NetworkObjectRole::OwnedAuthoritative)
+                        Hierarchy->AddObject(obj);
+                }
                 item.OwnerClientId = e.OwnerClientId;
                 if (e.HierarchicalOwnership)
                     NetworkReplicator::SetObjectOwnership(obj, e.OwnerClientId, e.Role, true);
@@ -1329,114 +1384,141 @@ void NetworkInternal::NetworkReplicatorUpdate()
         }
     }
 
-    // Brute force synchronize all networked objects with clients
+    // Replicate all owned networked objects with other clients or server
-    if (CachedWriteStream == nullptr)
+    if (!CachedReplicationResult)
-        CachedWriteStream = New<NetworkStream>();
+        CachedReplicationResult = New<NetworkReplicationHierarchyUpdateResult>();
-    NetworkStream* stream = CachedWriteStream;
+    CachedReplicationResult->Init();
-    stream->SenderId = NetworkManager::LocalClientId;
+    if (!isClient && NetworkManager::Clients.IsEmpty())
-    // TODO: introduce NetworkReplicationHierarchy to optimize objects replication in large worlds (eg. batched culling networked scene objects that are too far from certain client to be relevant)
-    // TODO: per-object sync interval (in frames) - could be scaled by hierarchy (eg. game could slow down sync rate for objects far from player)
-    for (auto it = Objects.Begin(); it.IsNotEnd(); ++it)
     {
-        auto& item = it->Item;
+        // No need to update replication when nobody's around
-        ScriptingObject* obj = item.Object.Get();
+    }
-        if (!obj)
+    else if (Hierarchy)
+    {
+        // Tick using hierarchy
+        PROFILE_CPU_NAMED("ReplicationHierarchyUpdate");
+        Hierarchy->Update(CachedReplicationResult);
+    }
+    else
+    {
+        // Tick all owned objects
+        PROFILE_CPU_NAMED("ReplicationUpdate");
+        for (auto it = Objects.Begin(); it.IsNotEnd(); ++it)
         {
-            // Object got deleted
+            auto& item = it->Item;
-            NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Remove object {}, owned by {}", item.ToString(), item.ParentId.ToString());
+            ScriptingObject* obj = item.Object.Get();
-            Objects.Remove(it);
+            if (!obj)
-            continue;
+            {
-        }
+                // Object got deleted
-        if (item.Role != NetworkObjectRole::OwnedAuthoritative && (!isClient && item.OwnerClientId != NetworkManager::LocalClientId))
+                NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Remove object {}, owned by {}", item.ToString(), item.ParentId.ToString());
-            continue; // Send replication messages of only owned objects or from other client objects
+                Objects.Remove(it);
-
-        // Skip serialization of objects that none will receive
-        if (!isClient)
-        {
-            // TODO: per-object relevancy for connected clients (eg. skip replicating actor to far players)
-            BuildCachedTargets(item);
-            if (CachedTargets.Count() == 0)
                 continue;
+            }
+            if (item.Role != NetworkObjectRole::OwnedAuthoritative)
+                continue; // Send replication messages of only owned objects or from other client objects
+            CachedReplicationResult->AddObject(obj);
         }
-
+    }
-        if (item.AsNetworkObject)
+    if (CachedReplicationResult->_entries.HasItems())
-            item.AsNetworkObject->OnNetworkSerialize();
+    {
-
+        PROFILE_CPU_NAMED("Replication");
-        // Serialize object
+        if (CachedWriteStream == nullptr)
-        stream->Initialize();
+            CachedWriteStream = New<NetworkStream>();
-        const bool failed = NetworkReplicator::InvokeSerializer(obj->GetTypeHandle(), obj, stream, true);
+        NetworkStream* stream = CachedWriteStream;
-        if (failed)
+        stream->SenderId = NetworkManager::LocalClientId;
+        // TODO: use Job System when replicated objects count is large
+        for (auto& e : CachedReplicationResult->_entries)
         {
-            //NETWORK_REPLICATOR_LOG(Error, "[NetworkReplicator] Cannot serialize object {} of type {} (missing serialization logic)", item.ToString(), obj->GetType().ToString());
+            ScriptingObject* obj = e.Object;
-            continue;
+            auto it = Objects.Find(obj->GetID());
-        }
+            ASSERT(it.IsNotEnd());
+            auto& item = it->Item;
 
-        // Send object to clients
+            // Skip serialization of objects that none will receive
-        {
+            if (!isClient)
-            const uint32 size = stream->GetPosition();
-            ASSERT(size <= MAX_uint16)
-            NetworkMessageObjectReplicate msgData;
-            msgData.OwnerFrame = NetworkManager::Frame;
-            msgData.ObjectId = item.ObjectId;
-            msgData.ParentId = item.ParentId;
-            if (isClient)
             {
-                // Remap local client object ids into server ids
+                // TODO: per-object relevancy for connected clients (eg. skip replicating actor to far players)
-                IdsRemappingTable.KeyOf(msgData.ObjectId, &msgData.ObjectId);
+                BuildCachedTargets(item, e.TargetClients);
-                IdsRemappingTable.KeyOf(msgData.ParentId, &msgData.ParentId);
+                if (CachedTargets.Count() == 0)
-            }
+                    return;
-            GetNetworkName(msgData.ObjectTypeName, obj->GetType().Fullname);
-            msgData.DataSize = size;
-            const uint32 msgMaxData = peer->Config.MessageSize - sizeof(NetworkMessageObjectReplicate);
-            const uint32 partMaxData = peer->Config.MessageSize - sizeof(NetworkMessageObjectReplicatePart);
-            uint32 partsCount = 1;
-            uint32 dataStart = 0;
-            uint32 msgDataSize = size;
-            if (size > msgMaxData)
-            {
-                // Send msgMaxData within first message
-                msgDataSize = msgMaxData;
-                dataStart += msgMaxData;
-
-                // Send rest of the data in separate parts
-                partsCount += Math::DivideAndRoundUp(size - dataStart, partMaxData);
-            }
-            else
-                dataStart += size;
-            ASSERT(partsCount <= MAX_uint8)
-            msgData.PartsCount = partsCount;
-            NetworkMessage msg = peer->BeginSendMessage();
-            msg.WriteStructure(msgData);
-            msg.WriteBytes(stream->GetBuffer(), msgDataSize);
-            if (isClient)
-                peer->EndSendMessage(NetworkChannelType::Unreliable, msg);
-            else
-            {
-                peer->EndSendMessage(NetworkChannelType::Unreliable, msg, CachedTargets);
             }
 
-            // Send all other parts
+            if (item.AsNetworkObject)
-            for (uint32 partIndex = 1; partIndex < partsCount; partIndex++)
+                item.AsNetworkObject->OnNetworkSerialize();
+
+            // Serialize object
+            stream->Initialize();
+            const bool failed = NetworkReplicator::InvokeSerializer(obj->GetTypeHandle(), obj, stream, true);
+            if (failed)
             {
-                NetworkMessageObjectReplicatePart msgDataPart;
+                //NETWORK_REPLICATOR_LOG(Error, "[NetworkReplicator] Cannot serialize object {} of type {} (missing serialization logic)", item.ToString(), obj->GetType().ToString());
-                msgDataPart.OwnerFrame = msgData.OwnerFrame;
+                return;
-                msgDataPart.ObjectId = msgData.ObjectId;
+            }
-                msgDataPart.DataSize = msgData.DataSize;
+
-                msgDataPart.PartsCount = msgData.PartsCount;
+            // Send object to clients
-                msgDataPart.PartStart = dataStart;
+            {
-                msgDataPart.PartSize = Math::Min(size - dataStart, partMaxData);
+                const uint32 size = stream->GetPosition();
-                msg = peer->BeginSendMessage();
+                ASSERT(size <= MAX_uint16)
-                msg.WriteStructure(msgDataPart);
+                NetworkMessageObjectReplicate msgData;
-                msg.WriteBytes(stream->GetBuffer() + msgDataPart.PartStart, msgDataPart.PartSize);
+                msgData.OwnerFrame = NetworkManager::Frame;
-                dataStart += msgDataPart.PartSize;
+                msgData.ObjectId = item.ObjectId;
+                msgData.ParentId = item.ParentId;
+                if (isClient)
+                {
+                    // Remap local client object ids into server ids
+                    IdsRemappingTable.KeyOf(msgData.ObjectId, &msgData.ObjectId);
+                    IdsRemappingTable.KeyOf(msgData.ParentId, &msgData.ParentId);
+                }
+                GetNetworkName(msgData.ObjectTypeName, obj->GetType().Fullname);
+                msgData.DataSize = size;
+                const uint32 msgMaxData = peer->Config.MessageSize - sizeof(NetworkMessageObjectReplicate);
+                const uint32 partMaxData = peer->Config.MessageSize - sizeof(NetworkMessageObjectReplicatePart);
+                uint32 partsCount = 1;
+                uint32 dataStart = 0;
+                uint32 msgDataSize = size;
+                if (size > msgMaxData)
+                {
+                    // Send msgMaxData within first message
+                    msgDataSize = msgMaxData;
+                    dataStart += msgMaxData;
+
+                    // Send rest of the data in separate parts
+                    partsCount += Math::DivideAndRoundUp(size - dataStart, partMaxData);
+                }
+                else
+                    dataStart += size;
+                ASSERT(partsCount <= MAX_uint8)
+                msgData.PartsCount = partsCount;
+                NetworkMessage msg = peer->BeginSendMessage();
+                msg.WriteStructure(msgData);
+                msg.WriteBytes(stream->GetBuffer(), msgDataSize);
                 if (isClient)
                     peer->EndSendMessage(NetworkChannelType::Unreliable, msg);
                 else
                     peer->EndSendMessage(NetworkChannelType::Unreliable, msg, CachedTargets);
-            }
-            ASSERT_LOW_LAYER(dataStart == size);
 
-            // TODO: stats for bytes send per object type
+                // Send all other parts
+                for (uint32 partIndex = 1; partIndex < partsCount; partIndex++)
+                {
+                    NetworkMessageObjectReplicatePart msgDataPart;
+                    msgDataPart.OwnerFrame = msgData.OwnerFrame;
+                    msgDataPart.ObjectId = msgData.ObjectId;
+                    msgDataPart.DataSize = msgData.DataSize;
+                    msgDataPart.PartsCount = msgData.PartsCount;
+                    msgDataPart.PartStart = dataStart;
+                    msgDataPart.PartSize = Math::Min(size - dataStart, partMaxData);
+                    msg = peer->BeginSendMessage();
+                    msg.WriteStructure(msgDataPart);
+                    msg.WriteBytes(stream->GetBuffer() + msgDataPart.PartStart, msgDataPart.PartSize);
+                    dataStart += msgDataPart.PartSize;
+                    if (isClient)
+                        peer->EndSendMessage(NetworkChannelType::Unreliable, msg);
+                    else
+                        peer->EndSendMessage(NetworkChannelType::Unreliable, msg, CachedTargets);
+                }
+                ASSERT_LOW_LAYER(dataStart == size);
+
+                // TODO: stats for bytes send per object type
+            }
         }
     }
 
@@ -1564,7 +1646,11 @@ void NetworkInternal::OnNetworkMessageObjectSpawn(NetworkEvent& event, NetworkCl
                 // Server always knows the best so update ownership of the existing object
                 item.OwnerClientId = msgData.OwnerClientId;
                 if (item.Role == NetworkObjectRole::OwnedAuthoritative)
+                {
+                    if (Hierarchy)
+                        Hierarchy->AddObject(item.Object);
                     item.Role = NetworkObjectRole::Replicated;
+                }
             }
             else if (item.OwnerClientId != msgData.OwnerClientId)
             {
@@ -1719,6 +1805,8 @@ void NetworkInternal::OnNetworkMessageObjectSpawn(NetworkEvent& event, NetworkCl
         item.Spawned = true;
         NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Add new object {}:{}, parent {}:{}", item.ToString(), obj->GetType().ToString(), item.ParentId.ToString(), parent ? parent->Object->GetType().ToString() : String::Empty);
         Objects.Add(MoveTemp(item));
+        if (Hierarchy && item.Role == NetworkObjectRole::OwnedAuthoritative)
+            Hierarchy->AddObject(obj);
 
         // Boost future lookups by using indirection
         NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Remap object ID={} into object {}:{}", msgDataItem.ObjectId, item.ToString(), obj->GetType().ToString());
@@ -1786,6 +1874,8 @@ void NetworkInternal::OnNetworkMessageObjectDespawn(NetworkEvent& event, Network
 
         // Remove object
         NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Despawn object {}", msgData.ObjectId);
+        if (Hierarchy && item.Role == NetworkObjectRole::OwnedAuthoritative)
+            Hierarchy->RemoveObject(obj);
         DespawnedObjects.Add(msgData.ObjectId);
         if (item.AsNetworkObject)
             item.AsNetworkObject->OnNetworkDespawn();
@@ -1822,12 +1912,16 @@ void NetworkInternal::OnNetworkMessageObjectRole(NetworkEvent& event, NetworkCli
         if (item.OwnerClientId == NetworkManager::LocalClientId)
         {
             // Upgrade ownership automatically
+            if (Hierarchy && item.Role != NetworkObjectRole::OwnedAuthoritative)
+                Hierarchy->AddObject(obj);
             item.Role = NetworkObjectRole::OwnedAuthoritative;
             item.LastOwnerFrame = 0;
         }
         else if (item.Role == NetworkObjectRole::OwnedAuthoritative)
         {
             // Downgrade ownership automatically
+            if (Hierarchy)
+                Hierarchy->RemoveObject(obj);
             item.Role = NetworkObjectRole::Replicated;
         }
         if (!NetworkManager::IsClient())

Source/Engine/Networking/NetworkReplicator.h

@@ -42,6 +42,17 @@ public:
     API_FIELD() static bool EnableLog;
 #endif
 
+    /// <summary>
+    /// Gets the network replication hierarchy.
+    /// </summary>
+    API_PROPERTY() static NetworkReplicationHierarchy* GetHierarchy();
+
+    /// <summary>
+    /// Sets the network replication hierarchy.
+    /// </summary>
+    API_PROPERTY() static void SetHierarchy(NetworkReplicationHierarchy* value);
+
+public:
     /// <summary>
     /// Adds the network replication serializer for a given type.
     /// </summary>

Source/Engine/Networking/Types.h

@@ -11,6 +11,7 @@ class INetworkSerializable;
 class NetworkPeer;
 class NetworkClient;
 class NetworkStream;
+class NetworkReplicationHierarchy;
 
 struct NetworkEvent;
 struct NetworkConnection;