Fixed issue involving stale scripting assemblies in FlaxEngine.Json dynamic type resolution

Added new ExtendedSerializationBinder
Added callback to clear serializer cache on scripting assembly reload
Added low-cost mechanism to invalidate the SerializerCache after domain reload

Source/Engine/Engine/NativeInterop.Unmanaged.cs

@@ -1022,6 +1022,8 @@ namespace FlaxEngine.Interop
                 pair.Value.Free();
             classAttributesCacheCollectible.Clear();
 
+            FlaxEngine.Json.JsonSerializer.ResetCache();
+
             // Unload the ALC
             bool unloading = true;
             scriptingAssemblyLoadContext.Unloading += (alc) => { unloading = false; };

Source/Engine/Engine/NativeInterop.cs

@@ -278,44 +278,65 @@ namespace FlaxEngine.Interop
             if (typeCache.TryGetValue(typeName, out Type type))
                 return type;
 
-            type = Type.GetType(typeName, ResolveAssemblyByName, null);
+            type = Type.GetType(typeName, ResolveAssembly, null);
             if (type == null)
-            {
-                foreach (var assembly in scriptingAssemblyLoadContext.Assemblies)
-                {
-                    type = assembly.GetType(typeName);
-                    if (type != null)
-                        break;
-                }
-            }
+                type = ResolveSlow(typeName);
 
             if (type == null)
             {
-                string oldTypeName = typeName;
+                string fullTypeName = typeName;
                 typeName = typeName.Substring(0, typeName.IndexOf(','));
-                type = Type.GetType(typeName, ResolveAssemblyByName, null);
+                type = Type.GetType(typeName, ResolveAssembly, null);
                 if (type == null)
-                {
-                    foreach (var assembly in scriptingAssemblyLoadContext.Assemblies)
-                    {
-                        type = assembly.GetType(typeName);
-                        if (type != null)
-                            break;
-                    }
-                }
-                typeName = oldTypeName;
+                    type = ResolveSlow(typeName);
+
+                typeName = fullTypeName;
             }
 
             typeCache.Add(typeName, type);
 
             return type;
+
+            /// <summary>Resolve the type by manually checking every scripting assembly</summary>
+            static Type ResolveSlow(string typeName) {
+                foreach (var assembly in scriptingAssemblyLoadContext.Assemblies) {
+                    var type = assembly.GetType(typeName);
+                    if (type != null)
+                        return type;
+                }
+                return null;
+            }
+
+            /// <summary>Resolve the assembly by name</summary>
+            static Assembly ResolveAssembly(AssemblyName name) => ResolveScriptingAssemblyByName(name, allowPartial: false);
         }
 
-        private static Assembly ResolveAssemblyByName(AssemblyName assemblyName)
+        /// <summary>Find <paramref name="assemblyName"/> among the scripting assemblies.</summary>
+        /// <param name="assemblyName">The name to find</param>
+        /// <param name="allowPartial">If true, partial names should be allowed to be resolved.</param>
+        /// <returns>The resolved assembly, or null if none could be found.</returns>
+        internal static Assembly ResolveScriptingAssemblyByName(AssemblyName assemblyName, bool allowPartial = false)
         {
             foreach (Assembly assembly in scriptingAssemblyLoadContext.Assemblies)
-                if (assembly.GetName() == assemblyName)
+            {
+                var curName = assembly.GetName();
+
+                if (curName == assemblyName || (allowPartial && curName.Name == assemblyName.Name))
                     return assembly;
+            }
+
+            if (allowPartial) // Check partial names if full name isn't found
+            {
+                string partialName = assemblyName.Name;
+
+                foreach (Assembly assembly in scriptingAssemblyLoadContext.Assemblies)
+                {
+                    var curName = assembly.GetName();
+
+                    if (curName.Name == partialName)
+                        return assembly;
+                }
+            }
             return null;
         }
 

Source/Engine/Serialization/JsonCustomSerializers/ExtendedSerializationBinder.cs

@@ -0,0 +1,238 @@
+// Copyright (c) 2012-2023 Wojciech Figat. All rights reserved.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Reflection;
+using System.Runtime.Serialization;
+using FlaxEngine.Interop;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Serialization;
+
+#nullable enable
+
+namespace FlaxEngine.Json.JsonCustomSerializers
+{
+    internal class ExtendedSerializationBinder : SerializationBinder, ISerializationBinder
+    {
+        private record struct TypeKey(string? assemblyName, string typeName);
+
+        private ConcurrentDictionary<TypeKey, Type> _cache;
+        private Func<TypeKey, Type> _resolve;
+
+        /// <summary>Clear the cache</summary>
+        /// <remarks>Should be cleared on scripting domain reload to avoid out of date types participating in dynamic type resolution</remarks>
+        public void ResetCache()
+        {
+            _cache.Clear();
+        }
+
+        public override Type BindToType(string? assemblyName, string typeName)
+        {
+            return FindCachedType(new(assemblyName, typeName));
+        }
+
+        public override void BindToName(Type serializedType, out string? assemblyName, out string? typeName)
+        {
+            assemblyName = serializedType.Assembly.FullName;
+            typeName = serializedType.FullName;
+        }
+
+
+        public ExtendedSerializationBinder()
+        {
+            _resolve = ResolveBind;
+            _cache = new();
+        }
+
+        Type ResolveBind(TypeKey key)
+        {
+            Type? type = null;
+            if (key.assemblyName is null) { // No assembly name, attempt to find globally
+                type = FindTypeGlobal(key.typeName);
+            }
+
+            if (type is null && key.assemblyName is not null) { // Type not found yet, but we have assembly name
+                Assembly? assembly = null;
+
+                assembly = FindScriptingAssembly(key.assemblyName); // Attempt to load from scripting assembly
+
+                if (assembly is null)
+                    assembly = FindLoadedAssembly(key.assemblyName); // Attempt to load from loaded assemblies
+
+                if (assembly is null)
+                    assembly = FindUnloadedAssembly(key.assemblyName); // Attempt to load from unloaded assemblies
+
+                if (assembly is null)
+                    throw MakeAsmResolutionException(key.assemblyName); // Assembly failed to resolve
+
+                type = FindTypeInAssembly(key.typeName, assembly); // We have assembly, attempt to load from assembly
+            }
+
+            //if (type is null)
+            //    type = _fallBack.BindToType(key.assemblyName, key.typeName); // Use fallback
+
+            if (type is null)
+                throw MakeTypeResolutionException(key.assemblyName, key.typeName);
+
+            return type;
+        }
+
+        /// <summary>Attempt to find the assembly among loaded scripting assemblies</summary>
+        Assembly? FindScriptingAssembly(string assemblyName)
+        {
+            return NativeInterop.ResolveScriptingAssemblyByName(new AssemblyName(assemblyName), allowPartial: true);
+        }
+
+        /// <summary>Attempt to find the assembly by name</summary>
+        Assembly? FindLoadedAssembly(string assemblyName) // TODO
+        {
+            return null;
+        }
+
+        /// <summary>Attempt to find the assembly by name</summary>
+        Assembly? FindUnloadedAssembly(string assemblyName)
+        {
+            Assembly? assembly = null;
+
+            assembly = Assembly.Load(new AssemblyName(assemblyName));
+
+            if (assembly is null)
+                assembly = Assembly.LoadWithPartialName(assemblyName); // Copying behavior of DefaultSerializationBinder
+
+
+            return assembly;
+        }
+
+
+
+        Type? FindTypeInAssembly(string typeName, Assembly assembly)
+        {
+            var type = assembly.GetType(typeName); // Attempt to load directly
+
+            if (type is null && typeName.IndexOf('`') >= 0) // Attempt failed, but name has generic variant tick, try resolving generic manually
+                type = FindTypeGeneric(typeName, assembly);
+
+            return type;
+        }
+
+        /// <summary>Attempt to find unqualified type by only name</summary>
+        Type? FindTypeGlobal(string typeName)
+        {
+            return Type.GetType(typeName);
+        }
+
+        /// <summary>Get type from the cache</summary>
+        private Type FindCachedType(TypeKey key)
+        {
+            return _cache.GetOrAdd(key, _resolve);
+        }
+
+
+
+        /*********************************************
+         ** Below code is adapted from Newtonsoft.Json
+         *********************************************/
+
+        /// <summary>Attempt to recursively resolve a generic type</summary>
+        private Type? FindTypeGeneric(string typeName, Assembly assembly)
+        {
+            Type? type = null;
+            int openBracketIndex = typeName.IndexOf('[', StringComparison.Ordinal);
+            if (openBracketIndex >= 0) {
+                string genericTypeDefName = typeName.Substring(0, openBracketIndex); // Find the unspecialized type
+                Type? genericTypeDef = assembly.GetType(genericTypeDefName);
+                if (genericTypeDef != null) {
+                    List<Type> genericTypeArguments = new List<Type>(); // Recursively resolve the arguments
+                    int scope = 0;
+                    int typeArgStartIndex = 0;
+                    int endIndex = typeName.Length - 1;
+                    for (int i = openBracketIndex + 1; i < endIndex; ++i) {
+                        char current = typeName[i];
+                        switch (current) {
+                        case '[':
+                            if (scope == 0) {
+                                typeArgStartIndex = i + 1;
+                            }
+                            ++scope;
+                            break;
+                        case ']':
+                            --scope;
+                            if (scope == 0) { // All arguments resolved, compose our type
+                                string typeArgAssemblyQualifiedName = typeName.Substring(typeArgStartIndex, i - typeArgStartIndex);
+
+                                TypeKey typeNameKey = SplitFullyQualifiedTypeName(typeArgAssemblyQualifiedName);
+                                genericTypeArguments.Add(FindCachedType(typeNameKey));
+                            }
+                            break;
+                        }
+                    }
+
+                    type = genericTypeDef.MakeGenericType(genericTypeArguments.ToArray());
+                }
+            }
+
+            return type;
+        }
+
+        /// <summary>Split a fully qualified type name into assembly name, and type name</summary>
+        private static TypeKey SplitFullyQualifiedTypeName(string fullyQualifiedTypeName)
+        {
+            int? assemblyDelimiterIndex = GetAssemblyDelimiterIndex(fullyQualifiedTypeName);
+
+            ReadOnlySpan<char> typeName;
+            ReadOnlySpan<char> assemblyName;
+
+            if (assemblyDelimiterIndex != null) {
+                typeName = fullyQualifiedTypeName.AsSpan().Slice(0, assemblyDelimiterIndex ?? 0);
+                assemblyName = fullyQualifiedTypeName.AsSpan().Slice((assemblyDelimiterIndex ?? 0) + 1);
+            } else {
+                typeName = fullyQualifiedTypeName;
+                assemblyName = null;
+            }
+            
+            return new(new(assemblyName), new(typeName));
+        }
+
+        /// <summary>Find the assembly name inside a fully qualified type name</summary>
+        private static int? GetAssemblyDelimiterIndex(string fullyQualifiedTypeName)
+        {
+            // we need to get the first comma following all surrounded in brackets because of generic types
+            // e.g. System.Collections.Generic.Dictionary`2[[System.String, mscorlib,Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089],[System.String, mscorlib, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089]], mscorlib, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089
+            int scope = 0;
+            for (int i = 0; i < fullyQualifiedTypeName.Length; i++) {
+                char current = fullyQualifiedTypeName[i];
+                switch (current) {
+                case '[':
+                    scope++;
+                    break;
+                case ']':
+                    scope--;
+                    break;
+                case ',':
+                    if (scope == 0) {
+                        return i;
+                    }
+                    break;
+                }
+            }
+
+            return null;
+        }
+
+
+        private static JsonSerializationException MakeAsmResolutionException(string asmName)
+        {
+            return new($"Could not load assembly '{asmName}'.");
+        }
+        
+        private static JsonSerializationException MakeTypeResolutionException(string? asmName, string typeName)
+        {
+            if (asmName is null)
+                return new($"Could not find '{typeName}'");
+            else
+                return new($"Could not find '{typeName}' in assembly '{asmName}'.");
+        }
+    }
+}

Source/Engine/Serialization/JsonSerializer.cs

@@ -19,6 +19,7 @@ namespace FlaxEngine.Json
     {
         internal class SerializerCache
         {
+            public readonly JsonSerializerSettings settings;
             public Newtonsoft.Json.JsonSerializer JsonSerializer;
             public StringBuilder StringBuilder;
             public StringWriter StringWriter;
@@ -28,37 +29,35 @@ namespace FlaxEngine.Json
             public StreamReader Reader;
             public bool IsWriting;
             public bool IsReading;
+            public uint cacheVersion;
 
             public unsafe SerializerCache(JsonSerializerSettings settings)
             {
-                JsonSerializer = Newtonsoft.Json.JsonSerializer.CreateDefault(settings);
-                JsonSerializer.Formatting = Formatting.Indented;
-                JsonSerializer.ReferenceLoopHandling = ReferenceLoopHandling.Serialize;
+                this.settings = settings;
+
                 StringBuilder = new StringBuilder(256);
                 StringWriter = new StringWriter(StringBuilder, CultureInfo.InvariantCulture);
-                SerializerWriter = new JsonSerializerInternalWriter(JsonSerializer);
                 MemoryStream = new UnmanagedMemoryStream((byte*)0, 0);
-                Reader = new StreamReader(MemoryStream, Encoding.UTF8, false);
-                JsonWriter = new JsonTextWriter(StringWriter)
+
+                lock (cacheSyncRoot)
                 {
-                    IndentChar = '\t',
-                    Indentation = 1,
-                    Formatting = JsonSerializer.Formatting,
-                    DateFormatHandling = JsonSerializer.DateFormatHandling,
-                    DateTimeZoneHandling = JsonSerializer.DateTimeZoneHandling,
-                    FloatFormatHandling = JsonSerializer.FloatFormatHandling,
-                    StringEscapeHandling = JsonSerializer.StringEscapeHandling,
-                    Culture = JsonSerializer.Culture,
-                    DateFormatString = JsonSerializer.DateFormatString,
-                };
+                    BuildSerializer();
+                    BuildRead();
+                    BuildWrite();
+
+                    cacheVersion = currentCacheVersion;
+                }
             }
 
             public void ReadBegin()
             {
+                CheckCacheVersionRebuild();
+
+                // TODO: Reset reading state (eg if previous deserialization got exception)
                 if (IsReading)
-                {
-                    // TODO: Reset reading state (eg if previous deserialization got exception)
-                }
+                    BuildRead();
+
+
                 IsWriting = false;
                 IsReading = true;
             }
@@ -70,23 +69,12 @@ namespace FlaxEngine.Json
 
             public void WriteBegin()
             {
+                CheckCacheVersionRebuild();
+
+                // Reset writing state (eg if previous serialization got exception)
                 if (IsWriting)
-                {
-                    // Reset writing state (eg if previous serialization got exception)
-                    SerializerWriter = new JsonSerializerInternalWriter(JsonSerializer);
-                    JsonWriter = new JsonTextWriter(StringWriter)
-                    {
-                        IndentChar = '\t',
-                        Indentation = 1,
-                        Formatting = JsonSerializer.Formatting,
-                        DateFormatHandling = JsonSerializer.DateFormatHandling,
-                        DateTimeZoneHandling = JsonSerializer.DateTimeZoneHandling,
-                        FloatFormatHandling = JsonSerializer.FloatFormatHandling,
-                        StringEscapeHandling = JsonSerializer.StringEscapeHandling,
-                        Culture = JsonSerializer.Culture,
-                        DateFormatString = JsonSerializer.DateFormatString,
-                    };
-                }
+                    BuildWrite();
+
                 StringBuilder.Clear();
                 IsWriting = true;
                 IsReading = false;
@@ -96,21 +84,83 @@ namespace FlaxEngine.Json
             {
                 IsWriting = false;
             }
+
+            /// <summary>Check that the cache is up to date, rebuild it if it isn't</summary>
+            private void CheckCacheVersionRebuild()
+            {
+                var cCV = currentCacheVersion;
+                if (cacheVersion == cCV)
+                    return;
+
+                lock (cacheSyncRoot)
+                {
+                    cCV = currentCacheVersion;
+                    if (cacheVersion == cCV)
+                        return;
+
+                    BuildSerializer();
+
+                    BuildRead();
+                    BuildWrite();
+
+                    cacheVersion = cCV;
+                }
+            }
+
+            /// <summary>Builds the serializer</summary>
+            private void BuildSerializer()
+            {
+                JsonSerializer = Newtonsoft.Json.JsonSerializer.CreateDefault(settings);
+                JsonSerializer.Formatting = Formatting.Indented;
+                JsonSerializer.ReferenceLoopHandling = ReferenceLoopHandling.Serialize;
+            }
+
+            /// <summary>Builds the reader state</summary>
+            private void BuildRead()
+            {
+                Reader = new StreamReader(MemoryStream, Encoding.UTF8, false);
+            }
+
+            /// <summary>Builds the writer state</summary>
+            private void BuildWrite()
+            {
+                SerializerWriter = new JsonSerializerInternalWriter(JsonSerializer);
+                JsonWriter = new JsonTextWriter(StringWriter) {
+                    IndentChar = '\t',
+                    Indentation = 1,
+                    Formatting = JsonSerializer.Formatting,
+                    DateFormatHandling = JsonSerializer.DateFormatHandling,
+                    DateTimeZoneHandling = JsonSerializer.DateTimeZoneHandling,
+                    FloatFormatHandling = JsonSerializer.FloatFormatHandling,
+                    StringEscapeHandling = JsonSerializer.StringEscapeHandling,
+                    Culture = JsonSerializer.Culture,
+                    DateFormatString = JsonSerializer.DateFormatString,
+                };
+            }
         }
 
         internal static JsonSerializerSettings Settings = CreateDefaultSettings(false);
         internal static JsonSerializerSettings SettingsManagedOnly = CreateDefaultSettings(true);
+        internal static ExtendedSerializationBinder SerializationBinder;
         internal static FlaxObjectConverter ObjectConverter;
+
         internal static ThreadLocal<SerializerCache> Current = new ThreadLocal<SerializerCache>();
         internal static ThreadLocal<SerializerCache> Cache = new ThreadLocal<SerializerCache>(() => new SerializerCache(Settings));
         internal static ThreadLocal<SerializerCache> CacheManagedOnly = new ThreadLocal<SerializerCache>(() => new SerializerCache(SettingsManagedOnly));
         internal static ThreadLocal<IntPtr> CachedGuidBuffer = new ThreadLocal<IntPtr>(() => Marshal.AllocHGlobal(32 * sizeof(char)), true);
         internal static string CachedGuidDigits = "0123456789abcdef";
+        /// <summary>The version of the cache, used to check that a cache is not out of date</summary>
+        internal static uint currentCacheVersion = 0;
+        /// <summary>Used to synchronize cache operations such as <see cref="SerializerCache"/> rebuild, and <see cref="ResetCache"/></summary>
+        internal static readonly object cacheSyncRoot = new();
 
         internal static JsonSerializerSettings CreateDefaultSettings(bool isManagedOnly)
         {
             //Newtonsoft.Json.Utilities.MiscellaneousUtils.ValueEquals = ValueEquals;
 
+            if (SerializationBinder is null)
+                SerializationBinder = new();
+
             var settings = new JsonSerializerSettings
             {
                 ContractResolver = new ExtendedDefaultContractResolver(isManagedOnly),
@@ -118,6 +168,7 @@ namespace FlaxEngine.Json
                 TypeNameHandling = TypeNameHandling.Auto,
                 NullValueHandling = NullValueHandling.Include,
                 ObjectCreationHandling = ObjectCreationHandling.Auto,
+                SerializationBinder = SerializationBinder,
             };
             if (ObjectConverter == null)
                 ObjectConverter = new FlaxObjectConverter();
@@ -134,6 +185,18 @@ namespace FlaxEngine.Json
             return settings;
         }
 
+        /// <summary>Called to reset the serialization cache</summary>
+        internal static void ResetCache()
+        {
+            lock (cacheSyncRoot)
+            {
+                unchecked { currentCacheVersion++; }
+
+                Newtonsoft.Json.JsonSerializer.ClearCache();
+                SerializationBinder.ResetCache();
+            }
+        }
+
         internal static void Dispose()
         {
             CachedGuidBuffer.Values.ForEach(Marshal.FreeHGlobal);