From d6e93a7fab1544c50b02242e42d37dabfcc09d66 Mon Sep 17 00:00:00 2001 From: SS Date: Mon, 6 Nov 2023 18:49:30 -0700 Subject: [PATCH] Fixed issue involving stale scripting assemblies in FlaxEngine.Json dynamic type resolution Added new ExtendedSerializationBinder Added callback to clear serializer cache on scripting assembly reload Added low-cost mechanism to invalidate the SerializerCache after domain reload --- .../Engine/Engine/NativeInterop.Unmanaged.cs | 2 + Source/Engine/Engine/NativeInterop.cs | 65 +++-- .../ExtendedSerializationBinder.cs | 238 ++++++++++++++++++ Source/Engine/Serialization/JsonSerializer.cs | 133 +++++++--- 4 files changed, 381 insertions(+), 57 deletions(-) create mode 100644 Source/Engine/Serialization/JsonCustomSerializers/ExtendedSerializationBinder.cs diff --git a/Source/Engine/Engine/NativeInterop.Unmanaged.cs b/Source/Engine/Engine/NativeInterop.Unmanaged.cs index 0391974b6..574b74580 100644 --- a/Source/Engine/Engine/NativeInterop.Unmanaged.cs +++ b/Source/Engine/Engine/NativeInterop.Unmanaged.cs @@ -1022,6 +1022,8 @@ namespace FlaxEngine.Interop pair.Value.Free(); classAttributesCacheCollectible.Clear(); + FlaxEngine.Json.JsonSerializer.ResetCache(); + // Unload the ALC bool unloading = true; scriptingAssemblyLoadContext.Unloading += (alc) => { unloading = false; }; diff --git a/Source/Engine/Engine/NativeInterop.cs b/Source/Engine/Engine/NativeInterop.cs index fea1fadda..28ac5024d 100644 --- a/Source/Engine/Engine/NativeInterop.cs +++ b/Source/Engine/Engine/NativeInterop.cs @@ -278,44 +278,65 @@ namespace FlaxEngine.Interop if (typeCache.TryGetValue(typeName, out Type type)) return type; - type = Type.GetType(typeName, ResolveAssemblyByName, null); + type = Type.GetType(typeName, ResolveAssembly, null); if (type == null) - { - foreach (var assembly in scriptingAssemblyLoadContext.Assemblies) - { - type = assembly.GetType(typeName); - if (type != null) - break; - } - } + type = ResolveSlow(typeName); if (type == null) { - string oldTypeName = typeName; + string fullTypeName = typeName; typeName = typeName.Substring(0, typeName.IndexOf(',')); - type = Type.GetType(typeName, ResolveAssemblyByName, null); + type = Type.GetType(typeName, ResolveAssembly, null); if (type == null) - { - foreach (var assembly in scriptingAssemblyLoadContext.Assemblies) - { - type = assembly.GetType(typeName); - if (type != null) - break; - } - } - typeName = oldTypeName; + type = ResolveSlow(typeName); + + typeName = fullTypeName; } typeCache.Add(typeName, type); return type; + + /// Resolve the type by manually checking every scripting assembly + static Type ResolveSlow(string typeName) { + foreach (var assembly in scriptingAssemblyLoadContext.Assemblies) { + var type = assembly.GetType(typeName); + if (type != null) + return type; + } + return null; + } + + /// Resolve the assembly by name + static Assembly ResolveAssembly(AssemblyName name) => ResolveScriptingAssemblyByName(name, allowPartial: false); } - private static Assembly ResolveAssemblyByName(AssemblyName assemblyName) + /// Find among the scripting assemblies. + /// The name to find + /// If true, partial names should be allowed to be resolved. + /// The resolved assembly, or null if none could be found. + internal static Assembly ResolveScriptingAssemblyByName(AssemblyName assemblyName, bool allowPartial = false) { foreach (Assembly assembly in scriptingAssemblyLoadContext.Assemblies) - if (assembly.GetName() == assemblyName) + { + var curName = assembly.GetName(); + + if (curName == assemblyName || (allowPartial && curName.Name == assemblyName.Name)) return assembly; + } + + if (allowPartial) // Check partial names if full name isn't found + { + string partialName = assemblyName.Name; + + foreach (Assembly assembly in scriptingAssemblyLoadContext.Assemblies) + { + var curName = assembly.GetName(); + + if (curName.Name == partialName) + return assembly; + } + } return null; } diff --git a/Source/Engine/Serialization/JsonCustomSerializers/ExtendedSerializationBinder.cs b/Source/Engine/Serialization/JsonCustomSerializers/ExtendedSerializationBinder.cs new file mode 100644 index 000000000..5f976f24f --- /dev/null +++ b/Source/Engine/Serialization/JsonCustomSerializers/ExtendedSerializationBinder.cs @@ -0,0 +1,238 @@ +// Copyright (c) 2012-2023 Wojciech Figat. All rights reserved. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Reflection; +using System.Runtime.Serialization; +using FlaxEngine.Interop; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +#nullable enable + +namespace FlaxEngine.Json.JsonCustomSerializers +{ + internal class ExtendedSerializationBinder : SerializationBinder, ISerializationBinder + { + private record struct TypeKey(string? assemblyName, string typeName); + + private ConcurrentDictionary _cache; + private Func _resolve; + + /// Clear the cache + /// Should be cleared on scripting domain reload to avoid out of date types participating in dynamic type resolution + public void ResetCache() + { + _cache.Clear(); + } + + public override Type BindToType(string? assemblyName, string typeName) + { + return FindCachedType(new(assemblyName, typeName)); + } + + public override void BindToName(Type serializedType, out string? assemblyName, out string? typeName) + { + assemblyName = serializedType.Assembly.FullName; + typeName = serializedType.FullName; + } + + + public ExtendedSerializationBinder() + { + _resolve = ResolveBind; + _cache = new(); + } + + Type ResolveBind(TypeKey key) + { + Type? type = null; + if (key.assemblyName is null) { // No assembly name, attempt to find globally + type = FindTypeGlobal(key.typeName); + } + + if (type is null && key.assemblyName is not null) { // Type not found yet, but we have assembly name + Assembly? assembly = null; + + assembly = FindScriptingAssembly(key.assemblyName); // Attempt to load from scripting assembly + + if (assembly is null) + assembly = FindLoadedAssembly(key.assemblyName); // Attempt to load from loaded assemblies + + if (assembly is null) + assembly = FindUnloadedAssembly(key.assemblyName); // Attempt to load from unloaded assemblies + + if (assembly is null) + throw MakeAsmResolutionException(key.assemblyName); // Assembly failed to resolve + + type = FindTypeInAssembly(key.typeName, assembly); // We have assembly, attempt to load from assembly + } + + //if (type is null) + // type = _fallBack.BindToType(key.assemblyName, key.typeName); // Use fallback + + if (type is null) + throw MakeTypeResolutionException(key.assemblyName, key.typeName); + + return type; + } + + /// Attempt to find the assembly among loaded scripting assemblies + Assembly? FindScriptingAssembly(string assemblyName) + { + return NativeInterop.ResolveScriptingAssemblyByName(new AssemblyName(assemblyName), allowPartial: true); + } + + /// Attempt to find the assembly by name + Assembly? FindLoadedAssembly(string assemblyName) // TODO + { + return null; + } + + /// Attempt to find the assembly by name + Assembly? FindUnloadedAssembly(string assemblyName) + { + Assembly? assembly = null; + + assembly = Assembly.Load(new AssemblyName(assemblyName)); + + if (assembly is null) + assembly = Assembly.LoadWithPartialName(assemblyName); // Copying behavior of DefaultSerializationBinder + + + return assembly; + } + + + + Type? FindTypeInAssembly(string typeName, Assembly assembly) + { + var type = assembly.GetType(typeName); // Attempt to load directly + + if (type is null && typeName.IndexOf('`') >= 0) // Attempt failed, but name has generic variant tick, try resolving generic manually + type = FindTypeGeneric(typeName, assembly); + + return type; + } + + /// Attempt to find unqualified type by only name + Type? FindTypeGlobal(string typeName) + { + return Type.GetType(typeName); + } + + /// Get type from the cache + private Type FindCachedType(TypeKey key) + { + return _cache.GetOrAdd(key, _resolve); + } + + + + /********************************************* + ** Below code is adapted from Newtonsoft.Json + *********************************************/ + + /// Attempt to recursively resolve a generic type + private Type? FindTypeGeneric(string typeName, Assembly assembly) + { + Type? type = null; + int openBracketIndex = typeName.IndexOf('[', StringComparison.Ordinal); + if (openBracketIndex >= 0) { + string genericTypeDefName = typeName.Substring(0, openBracketIndex); // Find the unspecialized type + Type? genericTypeDef = assembly.GetType(genericTypeDefName); + if (genericTypeDef != null) { + List genericTypeArguments = new List(); // Recursively resolve the arguments + int scope = 0; + int typeArgStartIndex = 0; + int endIndex = typeName.Length - 1; + for (int i = openBracketIndex + 1; i < endIndex; ++i) { + char current = typeName[i]; + switch (current) { + case '[': + if (scope == 0) { + typeArgStartIndex = i + 1; + } + ++scope; + break; + case ']': + --scope; + if (scope == 0) { // All arguments resolved, compose our type + string typeArgAssemblyQualifiedName = typeName.Substring(typeArgStartIndex, i - typeArgStartIndex); + + TypeKey typeNameKey = SplitFullyQualifiedTypeName(typeArgAssemblyQualifiedName); + genericTypeArguments.Add(FindCachedType(typeNameKey)); + } + break; + } + } + + type = genericTypeDef.MakeGenericType(genericTypeArguments.ToArray()); + } + } + + return type; + } + + /// Split a fully qualified type name into assembly name, and type name + private static TypeKey SplitFullyQualifiedTypeName(string fullyQualifiedTypeName) + { + int? assemblyDelimiterIndex = GetAssemblyDelimiterIndex(fullyQualifiedTypeName); + + ReadOnlySpan typeName; + ReadOnlySpan assemblyName; + + if (assemblyDelimiterIndex != null) { + typeName = fullyQualifiedTypeName.AsSpan().Slice(0, assemblyDelimiterIndex ?? 0); + assemblyName = fullyQualifiedTypeName.AsSpan().Slice((assemblyDelimiterIndex ?? 0) + 1); + } else { + typeName = fullyQualifiedTypeName; + assemblyName = null; + } + + return new(new(assemblyName), new(typeName)); + } + + /// Find the assembly name inside a fully qualified type name + private static int? GetAssemblyDelimiterIndex(string fullyQualifiedTypeName) + { + // we need to get the first comma following all surrounded in brackets because of generic types + // e.g. System.Collections.Generic.Dictionary`2[[System.String, mscorlib,Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089],[System.String, mscorlib, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089]], mscorlib, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + int scope = 0; + for (int i = 0; i < fullyQualifiedTypeName.Length; i++) { + char current = fullyQualifiedTypeName[i]; + switch (current) { + case '[': + scope++; + break; + case ']': + scope--; + break; + case ',': + if (scope == 0) { + return i; + } + break; + } + } + + return null; + } + + + private static JsonSerializationException MakeAsmResolutionException(string asmName) + { + return new($"Could not load assembly '{asmName}'."); + } + + private static JsonSerializationException MakeTypeResolutionException(string? asmName, string typeName) + { + if (asmName is null) + return new($"Could not find '{typeName}'"); + else + return new($"Could not find '{typeName}' in assembly '{asmName}'."); + } + } +} diff --git a/Source/Engine/Serialization/JsonSerializer.cs b/Source/Engine/Serialization/JsonSerializer.cs index c36e00b6a..2eb9eec99 100644 --- a/Source/Engine/Serialization/JsonSerializer.cs +++ b/Source/Engine/Serialization/JsonSerializer.cs @@ -19,6 +19,7 @@ namespace FlaxEngine.Json { internal class SerializerCache { + public readonly JsonSerializerSettings settings; public Newtonsoft.Json.JsonSerializer JsonSerializer; public StringBuilder StringBuilder; public StringWriter StringWriter; @@ -28,37 +29,35 @@ namespace FlaxEngine.Json public StreamReader Reader; public bool IsWriting; public bool IsReading; + public uint cacheVersion; public unsafe SerializerCache(JsonSerializerSettings settings) { - JsonSerializer = Newtonsoft.Json.JsonSerializer.CreateDefault(settings); - JsonSerializer.Formatting = Formatting.Indented; - JsonSerializer.ReferenceLoopHandling = ReferenceLoopHandling.Serialize; + this.settings = settings; + StringBuilder = new StringBuilder(256); StringWriter = new StringWriter(StringBuilder, CultureInfo.InvariantCulture); - SerializerWriter = new JsonSerializerInternalWriter(JsonSerializer); MemoryStream = new UnmanagedMemoryStream((byte*)0, 0); - Reader = new StreamReader(MemoryStream, Encoding.UTF8, false); - JsonWriter = new JsonTextWriter(StringWriter) + + lock (cacheSyncRoot) { - IndentChar = '\t', - Indentation = 1, - Formatting = JsonSerializer.Formatting, - DateFormatHandling = JsonSerializer.DateFormatHandling, - DateTimeZoneHandling = JsonSerializer.DateTimeZoneHandling, - FloatFormatHandling = JsonSerializer.FloatFormatHandling, - StringEscapeHandling = JsonSerializer.StringEscapeHandling, - Culture = JsonSerializer.Culture, - DateFormatString = JsonSerializer.DateFormatString, - }; + BuildSerializer(); + BuildRead(); + BuildWrite(); + + cacheVersion = currentCacheVersion; + } } public void ReadBegin() { + CheckCacheVersionRebuild(); + + // TODO: Reset reading state (eg if previous deserialization got exception) if (IsReading) - { - // TODO: Reset reading state (eg if previous deserialization got exception) - } + BuildRead(); + + IsWriting = false; IsReading = true; } @@ -70,23 +69,12 @@ namespace FlaxEngine.Json public void WriteBegin() { + CheckCacheVersionRebuild(); + + // Reset writing state (eg if previous serialization got exception) if (IsWriting) - { - // Reset writing state (eg if previous serialization got exception) - SerializerWriter = new JsonSerializerInternalWriter(JsonSerializer); - JsonWriter = new JsonTextWriter(StringWriter) - { - IndentChar = '\t', - Indentation = 1, - Formatting = JsonSerializer.Formatting, - DateFormatHandling = JsonSerializer.DateFormatHandling, - DateTimeZoneHandling = JsonSerializer.DateTimeZoneHandling, - FloatFormatHandling = JsonSerializer.FloatFormatHandling, - StringEscapeHandling = JsonSerializer.StringEscapeHandling, - Culture = JsonSerializer.Culture, - DateFormatString = JsonSerializer.DateFormatString, - }; - } + BuildWrite(); + StringBuilder.Clear(); IsWriting = true; IsReading = false; @@ -96,21 +84,83 @@ namespace FlaxEngine.Json { IsWriting = false; } + + /// Check that the cache is up to date, rebuild it if it isn't + private void CheckCacheVersionRebuild() + { + var cCV = currentCacheVersion; + if (cacheVersion == cCV) + return; + + lock (cacheSyncRoot) + { + cCV = currentCacheVersion; + if (cacheVersion == cCV) + return; + + BuildSerializer(); + + BuildRead(); + BuildWrite(); + + cacheVersion = cCV; + } + } + + /// Builds the serializer + private void BuildSerializer() + { + JsonSerializer = Newtonsoft.Json.JsonSerializer.CreateDefault(settings); + JsonSerializer.Formatting = Formatting.Indented; + JsonSerializer.ReferenceLoopHandling = ReferenceLoopHandling.Serialize; + } + + /// Builds the reader state + private void BuildRead() + { + Reader = new StreamReader(MemoryStream, Encoding.UTF8, false); + } + + /// Builds the writer state + private void BuildWrite() + { + SerializerWriter = new JsonSerializerInternalWriter(JsonSerializer); + JsonWriter = new JsonTextWriter(StringWriter) { + IndentChar = '\t', + Indentation = 1, + Formatting = JsonSerializer.Formatting, + DateFormatHandling = JsonSerializer.DateFormatHandling, + DateTimeZoneHandling = JsonSerializer.DateTimeZoneHandling, + FloatFormatHandling = JsonSerializer.FloatFormatHandling, + StringEscapeHandling = JsonSerializer.StringEscapeHandling, + Culture = JsonSerializer.Culture, + DateFormatString = JsonSerializer.DateFormatString, + }; + } } internal static JsonSerializerSettings Settings = CreateDefaultSettings(false); internal static JsonSerializerSettings SettingsManagedOnly = CreateDefaultSettings(true); + internal static ExtendedSerializationBinder SerializationBinder; internal static FlaxObjectConverter ObjectConverter; + internal static ThreadLocal Current = new ThreadLocal(); internal static ThreadLocal Cache = new ThreadLocal(() => new SerializerCache(Settings)); internal static ThreadLocal CacheManagedOnly = new ThreadLocal(() => new SerializerCache(SettingsManagedOnly)); internal static ThreadLocal CachedGuidBuffer = new ThreadLocal(() => Marshal.AllocHGlobal(32 * sizeof(char)), true); internal static string CachedGuidDigits = "0123456789abcdef"; + /// The version of the cache, used to check that a cache is not out of date + internal static uint currentCacheVersion = 0; + /// Used to synchronize cache operations such as rebuild, and + internal static readonly object cacheSyncRoot = new(); internal static JsonSerializerSettings CreateDefaultSettings(bool isManagedOnly) { //Newtonsoft.Json.Utilities.MiscellaneousUtils.ValueEquals = ValueEquals; + if (SerializationBinder is null) + SerializationBinder = new(); + var settings = new JsonSerializerSettings { ContractResolver = new ExtendedDefaultContractResolver(isManagedOnly), @@ -118,6 +168,7 @@ namespace FlaxEngine.Json TypeNameHandling = TypeNameHandling.Auto, NullValueHandling = NullValueHandling.Include, ObjectCreationHandling = ObjectCreationHandling.Auto, + SerializationBinder = SerializationBinder, }; if (ObjectConverter == null) ObjectConverter = new FlaxObjectConverter(); @@ -134,6 +185,18 @@ namespace FlaxEngine.Json return settings; } + /// Called to reset the serialization cache + internal static void ResetCache() + { + lock (cacheSyncRoot) + { + unchecked { currentCacheVersion++; } + + Newtonsoft.Json.JsonSerializer.ClearCache(); + SerializationBinder.ResetCache(); + } + } + internal static void Dispose() { CachedGuidBuffer.Values.ForEach(Marshal.FreeHGlobal);