diff --git a/Source/Engine/Level/SceneObjectsFactory.cpp b/Source/Engine/Level/SceneObjectsFactory.cpp
index 574a179d6..a46edf070 100644
--- a/Source/Engine/Level/SceneObjectsFactory.cpp
+++ b/Source/Engine/Level/SceneObjectsFactory.cpp
@@ -48,8 +48,9 @@ void MissingScript::SetReferenceScript(const ScriptingObjectReference<Script>& v
 
 SceneObjectsFactory::Context::Context(ISerializeModifier* modifier)
     : Modifier(modifier)
-    , MainThreadId(Platform::GetCurrentThreadID())
 {
+    // Override the main thread value to not create it again in GetModifier() if called from the same thread
+    Modifiers.Set(modifier);
 }
 
 SceneObjectsFactory::Context::~Context()
@@ -59,7 +60,11 @@ SceneObjectsFactory::Context::~Context()
         Array<ISerializeModifier*, FixedAllocation<PLATFORM_THREADS_LIMIT>> modifiers;
         Modifiers.GetValues(modifiers);
         for (ISerializeModifier* e : modifiers)
+        {
+            if (e == Modifier)
+                continue;
             Cache::ISerializeModifier.Put(e);
+        }
     }
 }
 
@@ -68,12 +73,10 @@ ISerializeModifier* SceneObjectsFactory::Context::GetModifier()
     ISerializeModifier* modifier = Modifier;
     if (Async)
     {
-        // When using context in async then use one ISerializeModifier for each jpb thread
+        // When using context in async then use one ISerializeModifier per-thread
         ISerializeModifier*& modifierThread = Modifiers.Get();
         if (!modifierThread)
         {
-            if (Platform::GetCurrentThreadID() == MainThreadId)
-                return modifier;
             modifierThread = Cache::ISerializeModifier.GetUnscoped();
             Modifiers.Set(modifierThread);
             Locker.Lock();
diff --git a/Source/Engine/Level/SceneObjectsFactory.h b/Source/Engine/Level/SceneObjectsFactory.h
index f670feba5..b1a9b8a9d 100644
--- a/Source/Engine/Level/SceneObjectsFactory.h
+++ b/Source/Engine/Level/SceneObjectsFactory.h
@@ -28,7 +28,6 @@ public:
         Dictionary<Guid, int32> ObjectToInstance;
         CriticalSection Locker;
         ThreadLocal<ISerializeModifier*> Modifiers;
-        uint64 MainThreadId;
 
         Context(ISerializeModifier* modifier);
         ~Context();