// Copyright (c) Wojciech Figat. All rights reserved. #include "NetworkReplicator.h" #include "NetworkClient.h" #include "NetworkManager.h" #include "NetworkInternal.h" #include "NetworkStream.h" #include "NetworkMessage.h" #include "NetworkPeer.h" #include "NetworkChannelType.h" #include "NetworkEvent.h" #include "NetworkRpc.h" #include "INetworkSerializable.h" #include "INetworkObject.h" #include "NetworkReplicationHierarchy.h" #include "Engine/Core/Collections/HashSet.h" #include "Engine/Core/Collections/Dictionary.h" #include "Engine/Core/Collections/ChunkedArray.h" #include "Engine/Core/Types/DataContainer.h" #include "Engine/Platform/CriticalSection.h" #include "Engine/Engine/EngineService.h" #include "Engine/Level/Actor.h" #include "Engine/Level/SceneObject.h" #include "Engine/Level/Prefabs/Prefab.h" #include "Engine/Level/Prefabs/PrefabManager.h" #include "Engine/Profiler/ProfilerCPU.h" #include "Engine/Profiler/ProfilerMemory.h" #include "Engine/Scripting/Script.h" #include "Engine/Scripting/Scripting.h" #include "Engine/Scripting/ScriptingObjectReference.h" #include "Engine/Threading/Threading.h" #include "Engine/Threading/ThreadLocal.h" #if USE_EDITOR #include "FlaxEngine.Gen.h" #endif #if !BUILD_RELEASE bool NetworkReplicator::EnableLog = false; #include "Engine/Core/Log.h" #include "Engine/Content/Content.h" #define NETWORK_REPLICATOR_LOG(messageType, format, ...) if (NetworkReplicator::EnableLog) { LOG(messageType, format, ##__VA_ARGS__); } #define USE_NETWORK_REPLICATOR_LOG 1 #else #define NETWORK_REPLICATOR_LOG(messageType, format, ...) #endif #if COMPILE_WITH_PROFILER bool NetworkInternal::EnableProfiling = false; Dictionary, NetworkInternal::ProfilerEvent> NetworkInternal::ProfilerEvents; #endif PACK_STRUCT(struct NetworkMessageObjectReplicate { NetworkMessageIDs ID = NetworkMessageIDs::ObjectReplicate; uint32 OwnerFrame; }); PACK_STRUCT(struct NetworkMessageObjectReplicatePayload { uint16 DataSize; uint16 PartsCount; uint16 PartSize; }); PACK_STRUCT(struct NetworkMessageObjectReplicatePart { NetworkMessageIDs ID = NetworkMessageIDs::ObjectReplicatePart; uint32 OwnerFrame; uint16 DataSize; uint16 PartsCount; uint16 PartStart; uint16 PartSize; }); PACK_STRUCT(struct NetworkMessageObjectSpawn { NetworkMessageIDs ID = NetworkMessageIDs::ObjectSpawn; uint32 OwnerClientId; uint32 OwnerSpawnId; // Unique for peer who spawned it and matches OwnerSpawnId inside following part messages uint16 ItemsCount; // Total items count uint8 UseParts : 1; // True if spawn message is header-only and all items come in the separate parts }); PACK_STRUCT(struct NetworkMessageObjectSpawnPart { NetworkMessageIDs ID = NetworkMessageIDs::ObjectSpawnPart; uint32 OwnerClientId; uint32 OwnerSpawnId; }); // TODO: optimize spawn item to use Network Keys rather than fixed-size data PACK_STRUCT(struct NetworkMessageObjectSpawnItem { Guid ObjectId; Guid ParentId; Guid PrefabObjectID; char ObjectTypeName[128]; }); PACK_STRUCT(struct NetworkMessageObjectDespawn { NetworkMessageIDs ID = NetworkMessageIDs::ObjectDespawn; }); PACK_STRUCT(struct NetworkMessageObjectRole { NetworkMessageIDs ID = NetworkMessageIDs::ObjectRole; uint32 OwnerClientId; }); PACK_STRUCT(struct NetworkMessageObjectRpc { NetworkMessageIDs ID = NetworkMessageIDs::ObjectRpc; uint16 ArgsSize; }); struct NetworkReplicatedObject { ScriptingObjectReference Object; Guid ObjectId; Guid ParentId; uint32 OwnerClientId; uint32 LastOwnerFrame = 0; NetworkObjectRole Role; uint8 Spawned : 1; uint8 Synced : 1; DataContainer TargetClientIds; INetworkObject* AsNetworkObject; struct { NetworkClientsMask Mask; BytesContainer Data; void Clear() { Mask = NetworkClientsMask(); Data.Release(); } } RepCache; NetworkReplicatedObject() { Spawned = 0; Synced = 0; } void Dirty() { RepCache.Mask = NetworkClientsMask(); } bool operator==(const NetworkReplicatedObject& other) const { return Object == other.Object; } bool operator==(const ScriptingObject* other) const { return Object == other; } bool operator==(const Guid& other) const { return ObjectId == other; } String ToString() const { return ObjectId.ToString(); } }; inline uint32 GetHash(const NetworkReplicatedObject& key) { return GetHash(key.ObjectId); } struct Serializer { NetworkReplicator::SerializeFunc Methods[2]; void* Tags[2]; }; struct ReplicateItem { ScriptingObjectReference Object; Guid ObjectId; uint16 PartsLeft; uint32 OwnerFrame; uint32 OwnerClientId; Array Data; }; struct SpawnItem { ScriptingObjectReference Object; DataContainer Targets; bool HasOwnership = false; bool HierarchicalOwnership = false; uint32 OwnerClientId; NetworkObjectRole Role; }; struct SpawnItemParts { NetworkMessageObjectSpawn MsgData; Guid PrefabId; Array Items; }; struct SpawnGroup { Array> Items; }; struct DespawnItem { Guid Id; DataContainer Targets; }; struct RpcItem { ScriptingObjectReference Object; NetworkRpcName Name; NetworkRpcInfo Info; BytesContainer ArgsData; DataContainer Targets; }; namespace { CriticalSection ObjectsLock; HashSet Objects; Array ReplicationParts; Array SpawnParts; Array SpawnQueue; Array DespawnQueue; Array RpcQueue; Dictionary IdsRemappingTable; NetworkStream* CachedWriteStream = nullptr; NetworkStream* CachedReadStream = nullptr; NetworkReplicationHierarchyUpdateResult* CachedReplicationResult = nullptr; NetworkReplicationHierarchy* Hierarchy = nullptr; Array NewClients; Array CachedTargets; Dictionary SerializersTable; #if !COMPILE_WITHOUT_CSHARP Dictionary CSharpCachedNames; #endif Array DespawnedObjects; uint32 SpawnId = 0; #if USE_EDITOR void OnScriptsReloading() { ScopeLock lock(ObjectsLock); if (Objects.HasItems()) LOG(Warning, "Hot-reloading scripts with network objects active."); if (Hierarchy) { Delete(Hierarchy); Hierarchy = nullptr; } // Clear any references to non-engine scripts before code hot-reload BinaryModule* flaxModule = GetBinaryModuleFlaxEngine(); for (auto i = SerializersTable.Begin(); i.IsNotEnd(); ++i) { if (i->Key.Module != flaxModule) SerializersTable.Remove(i); } for (auto i = NetworkRpcInfo::RPCsTable.Begin(); i.IsNotEnd(); ++i) { if (i->Key.First.Module != flaxModule) NetworkRpcInfo::RPCsTable.Remove(i); } } #endif } class NetworkReplicationService : public EngineService { public: NetworkReplicationService() : EngineService(TEXT("Network Replication"), 1100) { } bool Init() override; void Dispose() override; }; bool NetworkReplicationService::Init() { #if USE_EDITOR Scripting::ScriptsReloading.Bind(OnScriptsReloading); #endif return false; } void NetworkReplicationService::Dispose() { NetworkInternal::NetworkReplicatorClear(); #if !COMPILE_WITHOUT_CSHARP CSharpCachedNames.ClearDelete(); #endif } NetworkReplicationService NetworkReplicationServiceInstance; void INetworkSerializable_Native_Serialize(void* instance, NetworkStream* stream, void* tag) { const int16 vtableOffset = (int16)(intptr)tag; ((INetworkSerializable*)((byte*)instance + vtableOffset))->Serialize(stream); } void INetworkSerializable_Native_Deserialize(void* instance, NetworkStream* stream, void* tag) { const int16 vtableOffset = (int16)(intptr)tag; ((INetworkSerializable*)((byte*)instance + vtableOffset))->Deserialize(stream); } void INetworkSerializable_Script_Serialize(void* instance, NetworkStream* stream, void* tag) { auto obj = (ScriptingObject*)instance; auto interface = ScriptingObject::ToInterface(obj); interface->Serialize(stream); } void INetworkSerializable_Script_Deserialize(void* instance, NetworkStream* stream, void* tag) { auto obj = (ScriptingObject*)instance; auto interface = ScriptingObject::ToInterface(obj); interface->Deserialize(stream); } NetworkReplicatedObject* ResolveObject(Guid objectId) { auto it = Objects.Find(objectId); if (it != Objects.End()) return &it->Item; IdsRemappingTable.TryGet(objectId, objectId); it = Objects.Find(objectId); return it != Objects.End() ? &it->Item : nullptr; } NetworkReplicatedObject* ResolveObject(Guid objectId, Guid parentId, const StringAnsiView& objectTypeName) { // Lookup object NetworkReplicatedObject* obj = ResolveObject(objectId); if (obj) return obj; // Try to find the object within the same parent (eg. spawned locally on both client and server) IdsRemappingTable.TryGet(parentId, parentId); const ScriptingTypeHandle objectType = Scripting::FindScriptingType(objectTypeName); if (!objectType) return nullptr; for (auto& e : Objects) { auto& item = e.Item; const ScriptingObject* obj = item.Object.Get(); if (item.LastOwnerFrame == 0 && item.ParentId == parentId && obj && obj->GetTypeHandle() == objectType && !IdsRemappingTable.ContainsValue(item.ObjectId)) { if (NetworkManager::IsClient()) { // Boost future lookups by using indirection NETWORK_REPLICATOR_LOG(Info, "[NetworkReplicator] Remap object ID={} into object {}:{}", objectId, item.ToString(), obj->GetType().ToString()); IdsRemappingTable.Add(objectId, item.ObjectId); } return &item; } } return nullptr; } void BuildCachedTargets(const Array& clients) { CachedTargets.Clear(); for (const NetworkClient* client : clients) { if (client->State == NetworkConnectionState::Connected) CachedTargets.Add(client->Connection); } } void BuildCachedTargets(const Array& clients, const NetworkClient* excludedClient) { CachedTargets.Clear(); for (const NetworkClient* client : clients) { if (client->State == NetworkConnectionState::Connected && client != excludedClient) CachedTargets.Add(client->Connection); } } void BuildCachedTargets(const Array& clients, const DataContainer& clientIds, const uint32 excludedClientId = NetworkManager::ServerClientId, const NetworkClientsMask clientsMask = NetworkClientsMask::All) { CachedTargets.Clear(); if (clientIds.IsValid()) { for (int32 clientIndex = 0; clientIndex < clients.Count(); clientIndex++) { const NetworkClient* client = clients.Get()[clientIndex]; if (client->State == NetworkConnectionState::Connected && client->ClientId != excludedClientId && clientsMask.HasBit(clientIndex)) { for (int32 i = 0; i < clientIds.Length(); i++) { if (clientIds[i] == client->ClientId) { CachedTargets.Add(client->Connection); break; } } } } } else { for (int32 clientIndex = 0; clientIndex < clients.Count(); clientIndex++) { const NetworkClient* client = clients.Get()[clientIndex]; if (client->State == NetworkConnectionState::Connected && client->ClientId != excludedClientId && clientsMask.HasBit(clientIndex)) CachedTargets.Add(client->Connection); } } } void BuildCachedTargets(const Array& clients, const DataContainer& clientIds1, const Span& clientIds2, const uint32 excludedClientId = NetworkManager::ServerClientId) { CachedTargets.Clear(); if (clientIds1.IsValid()) { if (clientIds2.IsValid()) { for (const NetworkClient* client : clients) { if (client->State == NetworkConnectionState::Connected && client->ClientId != excludedClientId) { for (int32 i = 0; i < clientIds1.Length(); i++) { if (clientIds1[i] == client->ClientId) { for (int32 j = 0; j < clientIds2.Length(); j++) { if (clientIds2[j] == client->ClientId) { CachedTargets.Add(client->Connection); break; } } break; } } } } } else { BuildCachedTargets(clients, clientIds1, excludedClientId); } } else { BuildCachedTargets(clients, clientIds2, excludedClientId); } } FORCE_INLINE void BuildCachedTargets(const NetworkReplicatedObject& item, const NetworkClientsMask clientsMask = NetworkClientsMask::All) { // By default send object to all connected clients excluding the owner but with optional TargetClientIds list BuildCachedTargets(NetworkManager::Clients, item.TargetClientIds, item.OwnerClientId, clientsMask); } void SetupObjectSpawnMessageItem(SpawnItem* e, NetworkMessage& msg) { ScriptingObject* obj = e->Object.Get(); auto it = Objects.Find(obj->GetID()); const auto& item = it->Item; // Add object into spawn message NetworkMessageObjectSpawnItem msgDataItem; msgDataItem.ObjectId = item.ObjectId; msgDataItem.ParentId = item.ParentId; { // Remap local client object ids into server ids IdsRemappingTable.KeyOf(msgDataItem.ObjectId, &msgDataItem.ObjectId); IdsRemappingTable.KeyOf(msgDataItem.ParentId, &msgDataItem.ParentId); } msgDataItem.PrefabObjectID = Guid::Empty; auto* objScene = ScriptingObject::Cast(obj); if (objScene && objScene->HasPrefabLink()) msgDataItem.PrefabObjectID = objScene->GetPrefabObjectID(); const StringAnsiView objectTypeName = obj->GetType().Fullname; Platform::MemoryCopy(msgDataItem.ObjectTypeName, objectTypeName.Get(), objectTypeName.Length()); msgDataItem.ObjectTypeName[objectTypeName.Length()] = 0; msg.WriteStructure(msgDataItem); } void SendObjectSpawnMessage(const SpawnGroup& group, const Array& clients) { PROFILE_CPU(); const bool isClient = NetworkManager::IsClient(); auto* peer = NetworkManager::Peer; NetworkMessage msg = peer->BeginSendMessage(); NetworkMessageObjectSpawn msgData; msgData.ItemsCount = group.Items.Count(); Guid prefabId; { // The first object is a root of the group (eg. prefab instance root actor) SpawnItem* e = group.Items[0]; ScriptingObject* obj = e->Object.Get(); msgData.OwnerClientId = e->OwnerClientId; auto* objScene = ScriptingObject::Cast(obj); prefabId = objScene && objScene->HasPrefabLink() ? objScene->GetPrefabID() : Guid::Empty; // Setup clients that should receive this spawn message auto it = Objects.Find(obj->GetID()); const auto& item = it->Item; BuildCachedTargets(clients, item.TargetClientIds); } // Network Peer has fixed size of messages so split spawn message into parts if there are too many objects to fit at once msgData.OwnerSpawnId = ++SpawnId; msgData.UseParts = msg.BufferSize - msg.Position < group.Items.Count() * sizeof(NetworkMessageObjectSpawnItem); msg.WriteStructure(msgData); msg.WriteNetworkId(prefabId); if (msgData.UseParts) { if (isClient) peer->EndSendMessage(NetworkChannelType::Reliable, msg); else peer->EndSendMessage(NetworkChannelType::Reliable, msg, CachedTargets); // Send spawn items in separate parts NetworkMessageObjectSpawnPart msgDataPart; msgDataPart.OwnerClientId = msgData.OwnerClientId; msgDataPart.OwnerSpawnId = msgData.OwnerSpawnId; uint16 itemIndex = 0; constexpr uint32 spawnItemMaxSize = sizeof(uint16) + sizeof(NetworkMessageObjectSpawnItem); // Index + Data while (itemIndex < msgData.ItemsCount) { msg = peer->BeginSendMessage(); msg.WriteStructure(msgDataPart); // Write as many items as possible into this message while (msg.Position + spawnItemMaxSize <= msg.BufferSize && itemIndex < msgData.ItemsCount) { msg.WriteUInt16(itemIndex); SetupObjectSpawnMessageItem(group.Items[itemIndex], msg); itemIndex++; } if (isClient) peer->EndSendMessage(NetworkChannelType::Reliable, msg); else peer->EndSendMessage(NetworkChannelType::Reliable, msg, CachedTargets); } } else { // Send all spawn items within the spawn message for (SpawnItem* e : group.Items) SetupObjectSpawnMessageItem(e, msg); if (isClient) peer->EndSendMessage(NetworkChannelType::Reliable, msg); else peer->EndSendMessage(NetworkChannelType::Reliable, msg, CachedTargets); } } void SendObjectRoleMessage(const NetworkReplicatedObject& item, const NetworkClient* excludedClient = nullptr) { NetworkMessageObjectRole msgData; Guid objectId = item.ObjectId; IdsRemappingTable.KeyOf(objectId, &objectId); msgData.OwnerClientId = item.OwnerClientId; auto peer = NetworkManager::Peer; NetworkMessage msg = peer->BeginSendMessage(); msg.WriteStructure(msgData); msg.WriteNetworkId(objectId); if (NetworkManager::IsClient()) { NetworkManager::Peer->EndSendMessage(NetworkChannelType::ReliableOrdered, msg); } else { BuildCachedTargets(NetworkManager::Clients, excludedClient); peer->EndSendMessage(NetworkChannelType::ReliableOrdered, msg, CachedTargets); } } void DeleteNetworkObject(ScriptingObject* obj) { // Remove from the mapping table const Guid id = obj->GetID(); IdsRemappingTable.Remove(id); IdsRemappingTable.RemoveValue(id); if (obj->Is