Add object replication data cache and send via Reliable channel to reduce data transfer

Source/Engine/Networking/NetworkInternal.h

@@ -7,6 +7,15 @@
 #include "Engine/Core/Collections/Dictionary.h"
 #endif
 
+// Internal version number of networking implementation. Updated once engine changes serialization or connection rules.
+#define NETWORK_PROTOCOL_VERSION 4
+
+// Enables encoding object ids and typenames via uint32 keys rather than full data send.
+#define USE_NETWORK_KEYS 1
+
+// Cached replication messages if contents didn't change
+#define USE_NETWORK_REPLICATOR_CACHE 1
+
 enum class NetworkMessageIDs : uint8
 {
     None = 0,

Source/Engine/Networking/NetworkManager.cpp

@@ -16,8 +16,6 @@
 #include "Engine/Profiler/ProfilerCPU.h"
 #include "Engine/Scripting/Scripting.h"
 
-#define NETWORK_PROTOCOL_VERSION 4
-
 float NetworkManager::NetworkFPS = 60.0f;
 NetworkPeer* NetworkManager::Peer = nullptr;
 NetworkManagerMode NetworkManager::Mode = NetworkManagerMode::Offline;
@@ -140,6 +138,7 @@ FORCE_INLINE bool IsNetworkKeyValid(uint32 index)
 
 void NetworkMessage::WriteNetworkId(const Guid& id)
 {
+#if USE_NETWORK_KEYS
     ScopeLock lock(Keys.Lock);
     uint32 index = MAX_uint32;
     bool hasIndex = Keys.LookupId.TryGet(id, index);
@@ -158,10 +157,14 @@ void NetworkMessage::WriteNetworkId(const Guid& id)
             Keys.PendingIds.Add(id, NetworkKey(id));
         }
     }
+#else
+    WriteBytes((const uint8*)&id, sizeof(Guid));
+#endif
 }
 
 void NetworkMessage::ReadNetworkId(Guid& id)
 {
+#if USE_NETWORK_KEYS
     ScopeLock lock(Keys.Lock);
     uint32 index = ReadUInt32();
     if (index != MAX_uint32)
@@ -193,10 +196,14 @@ void NetworkMessage::ReadNetworkId(Guid& id)
             Keys.PendingIds.Add(id, NetworkKey(id));
         }
     }
+#else
+    ReadBytes((uint8*)&id, sizeof(Guid));
+#endif
 }
 
 void NetworkMessage::WriteNetworkName(const StringAnsiView& name)
 {
+#if USE_NETWORK_KEYS
     ScopeLock lock(Keys.Lock);
     uint32 index = MAX_uint32;
     bool hasIndex = Keys.LookupName.TryGet(name, index);
@@ -216,10 +223,14 @@ void NetworkMessage::WriteNetworkName(const StringAnsiView& name)
             Keys.PendingNames.Add(newName, NetworkKey(newName));
         }
     }
+#else
+    WriteStringAnsi(name);
+#endif
 }
 
 void NetworkMessage::ReadNetworkName(StringAnsiView& name)
 {
+#if USE_NETWORK_KEYS
     ScopeLock lock(Keys.Lock);
     uint32 index = ReadUInt32();
     if (index != MAX_uint32)
@@ -252,6 +263,9 @@ void NetworkMessage::ReadNetworkName(StringAnsiView& name)
             Keys.PendingNames.Add(newName, NetworkKey(newName));
         }
     }
+#else
+    name = ReadStringAnsi();
+#endif
 }
 
 void OnNetworkMessageHandshake(NetworkEvent& event, NetworkClient* client, NetworkPeer* peer)

Source/Engine/Networking/NetworkReplicator.cpp

@@ -126,12 +126,29 @@ struct NetworkReplicatedObject
     DataContainer<uint32> TargetClientIds;
     INetworkObject* AsNetworkObject;
 
+    struct
+    {
+        NetworkClientsMask Mask;
+        BytesContainer Data;
+
+        void Clear()
+        {
+            Mask = NetworkClientsMask();
+            Data.Release();
+        }
+    } RepCache;
+
     NetworkReplicatedObject()
     {
         Spawned = 0;
         Synced = 0;
     }
 
+    void Dirty()
+    {
+        RepCache.Mask = NetworkClientsMask();
+    }
+
     bool operator==(const NetworkReplicatedObject& other) const
     {
         return Object == other.Object;
@@ -684,6 +701,7 @@ void FindObjectsForSpawn(SpawnGroup& group, ChunkedArray<SpawnItem, 256>& spawnI
 
 FORCE_INLINE void DirtyObjectImpl(NetworkReplicatedObject& item, ScriptingObject* obj)
 {
+    item.Dirty();
     if (Hierarchy)
         Hierarchy->DirtyObject(obj);
 }
@@ -787,6 +805,7 @@ void InvokeObjectSpawn(const NetworkMessageObjectSpawn& msgData, const Guid& pre
             {
                 // Server always knows the best so update ownership of the existing object
                 item.OwnerClientId = msgData.OwnerClientId;
+                item.RepCache.Clear();
                 if (item.Role == NetworkObjectRole::OwnedAuthoritative)
                 {
                     if (Hierarchy)
@@ -1442,6 +1461,7 @@ void NetworkReplicator::SetObjectOwnership(ScriptingObject* obj, uint32 ownerCli
                 item.OwnerClientId = ownerClientId;
                 item.LastOwnerFrame = 1;
                 item.Role = localRole;
+                item.RepCache.Clear();
                 SendObjectRoleMessage(item);
             }
         }
@@ -1449,11 +1469,11 @@ void NetworkReplicator::SetObjectOwnership(ScriptingObject* obj, uint32 ownerCli
         {
             // Allow to change local role of the object (except ownership)
 #if !BUILD_RELEASE
-                if (localRole == NetworkObjectRole::OwnedAuthoritative)
+            if (localRole == NetworkObjectRole::OwnedAuthoritative)
-                {
+            {
-                    LOG(Error, "Cannot change ownership of object (Id={}) to the remote client (Id={}) if the local role is set to OwnedAuthoritative.", obj->GetID(), ownerClientId);
+                LOG(Error, "Cannot change ownership of object (Id={}) to the remote client (Id={}) if the local role is set to OwnedAuthoritative.", obj->GetID(), ownerClientId);
-                    return;
+                return;
-                }
+            }
 #endif
             if (Hierarchy && it->Item.Role == NetworkObjectRole::OwnedAuthoritative)
                 Hierarchy->RemoveObject(obj);
@@ -1864,10 +1884,28 @@ void NetworkInternal::NetworkReplicatorUpdate()
                 //NETWORK_REPLICATOR_LOG(Error, "[NetworkReplicator] Cannot serialize object {} of type {} (missing serialization logic)", item.ToString(), obj->GetType().ToString());
                 continue;
             }
+            const uint32 size = stream->GetPosition();
+            if (size > MAX_uint16)
+            {
+                LOG(Error, "Too much data for object {} replication ({} bytes provided while limit is {}).", item.ToString(), size, MAX_uint16);
+                continue;
+            }
+
+#if USE_NETWORK_REPLICATOR_CACHE
+            // Process replication cache to skip sending object data if it didn't change
+            if (item.RepCache.Data.Length() == size &&
+                item.RepCache.Mask == e.TargetClients &&
+                Platform::MemoryCompare(item.RepCache.Data.Get(), stream->GetBuffer(), size) == 0)
+            {
+                continue;
+            }
+            item.RepCache.Mask = e.TargetClients;
+            item.RepCache.Data.Copy(stream->GetBuffer(), size);
+#endif
+            // TODO: use Unreliable for dynamic objects that are replicated every frame? (eg. player state)
+            constexpr NetworkChannelType repChannel = NetworkChannelType::Reliable;
 
             // Send object to clients
-            const uint32 size = stream->GetPosition();
-            ASSERT(size <= MAX_uint16);
             NetworkMessageObjectReplicate msgData;
             msgData.OwnerFrame = NetworkManager::Frame;
             Guid objectId = item.ObjectId, parentId = item.ParentId;
@@ -1908,9 +1946,9 @@ void NetworkInternal::NetworkReplicatorUpdate()
             msg.WriteBytes(stream->GetBuffer(), msgDataSize);
             uint32 dataSize = msgDataSize, messageSize = msg.Length;
             if (isClient)
-                peer->EndSendMessage(NetworkChannelType::Unreliable, msg);
+                peer->EndSendMessage(repChannel, msg);
             else
-                peer->EndSendMessage(NetworkChannelType::Unreliable, msg, CachedTargets);
+                peer->EndSendMessage(repChannel, msg, CachedTargets);
 
             // Send all other parts
             for (uint32 partIndex = 1; partIndex < partsCount; partIndex++)
@@ -1929,9 +1967,9 @@ void NetworkInternal::NetworkReplicatorUpdate()
                 dataSize += msgDataPart.PartSize;
                 dataStart += msgDataPart.PartSize;
                 if (isClient)
-                    peer->EndSendMessage(NetworkChannelType::Unreliable, msg);
+                    peer->EndSendMessage(repChannel, msg);
                 else
-                    peer->EndSendMessage(NetworkChannelType::Unreliable, msg, CachedTargets);
+                    peer->EndSendMessage(repChannel, msg, CachedTargets);
             }
             ASSERT_LOW_LAYER(dataStart == size);
 

Source/Engine/Tests/TestNetworking.cpp

@@ -37,7 +37,6 @@ TEST_CASE("Networking")
                     }
                 }
             }
-            int dataSizeInkB = writeStream->GetPosition() / 1024; // 4291 -> 1869
             readStream->Initialize(writeStream->GetBuffer(), writeStream->GetPosition());
             for (int32 x = 0; x <= QuatRes; x++)
             {