Fix native library resolver not working after hot-reload

Source/Engine/Engine/NativeInterop.cs

@@ -847,27 +847,27 @@ namespace FlaxEngine
         private static Dictionary<object, GCHandle> classAttributesCacheCollectible = new();
         private static Dictionary<Assembly, GCHandle> assemblyHandles = new Dictionary<Assembly, GCHandle>();
 
-        private static string hostExecutable;
-        private static IntPtr hostExecutableHandle = IntPtr.Zero;
+        private static Dictionary<string, IntPtr> loadedNativeLibraries = new Dictionary<string, IntPtr>();
+        private static Dictionary<string, string> nativeLibraryPaths = new Dictionary<string, string>();
+        private static Dictionary<Assembly, string> assemblyOwnedNativeLibraries = new Dictionary<Assembly, string>();
         private static AssemblyLoadContext scriptingAssemblyLoadContext;
 
         [System.Diagnostics.DebuggerStepThrough]
-        private static IntPtr InternalDllResolver(string libraryName, Assembly assembly, DllImportSearchPath? dllImportSearchPath)
+        private static IntPtr NativeLibraryImportResolver(string libraryName, Assembly assembly, DllImportSearchPath? dllImportSearchPath)
         {
-            if (libraryName == "FlaxEngine")
+            if (!loadedNativeLibraries.TryGetValue(libraryName, out IntPtr nativeLibrary))
             {
-                if (hostExecutableHandle == IntPtr.Zero)
-                    hostExecutableHandle = NativeLibrary.Load(hostExecutable, assembly, dllImportSearchPath);
-                return hostExecutableHandle;
+                nativeLibrary = NativeLibrary.Load(nativeLibraryPaths[libraryName], assembly, dllImportSearchPath);
+                loadedNativeLibraries.Add(libraryName, nativeLibrary);
+                assemblyOwnedNativeLibraries.Add(assembly, libraryName);
             }
-            return IntPtr.Zero;
+            return nativeLibrary;
         }
 
         [UnmanagedCallersOnly]
-        internal static unsafe void Init(IntPtr hostExecutableName)
+        internal static unsafe void Init()
         {
-            hostExecutable = Marshal.PtrToStringUni(hostExecutableName);
-            NativeLibrary.SetDllImportResolver(Assembly.GetExecutingAssembly(), InternalDllResolver);
+            NativeLibrary.SetDllImportResolver(Assembly.GetExecutingAssembly(), NativeLibraryImportResolver);
 
             // Change default culture to match with Mono runtime default culture
             CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
@@ -886,6 +886,15 @@ namespace FlaxEngine
         {
         }
 
+        [UnmanagedCallersOnly]
+        internal static void RegisterNativeLibrary(IntPtr moduleName_, IntPtr modulePath_)
+        {
+            string moduleName = Marshal.PtrToStringAnsi(moduleName_);
+            string modulePath = Marshal.PtrToStringAnsi(modulePath_);
+
+            nativeLibraryPaths[moduleName] = modulePath;
+        }
+
         internal static T[] GCHandleArrayToManagedArray<T>(ManagedArray ptrArray)
         {
             Span<IntPtr> span = ptrArray.GetSpan<IntPtr>();
@@ -2145,6 +2154,7 @@ namespace FlaxEngine
             string assemblyPath = Marshal.PtrToStringAnsi(assemblyPath_);
 
             Assembly assembly = scriptingAssemblyLoadContext.LoadFromAssemblyPath(assemblyPath);
+            NativeLibrary.SetDllImportResolver(assembly, NativeLibraryImportResolver);
 
             *assemblyName = Marshal.StringToCoTaskMemAnsi(assembly.GetName().Name);
             *assemblyFullName = Marshal.StringToCoTaskMemAnsi(assembly.FullName);
@@ -2172,6 +2182,7 @@ namespace FlaxEngine
 
             using MemoryStream stream = new MemoryStream(raw);
             Assembly assembly = scriptingAssemblyLoadContext.LoadFromStream(stream);
+            NativeLibrary.SetDllImportResolver(assembly, NativeLibraryImportResolver);
 
             // Assemblies loaded via streams have no Location: https://github.com/dotnet/runtime/issues/12822
             AssemblyLocations.Add(assembly.FullName, assemblyPath);
@@ -2231,6 +2242,16 @@ namespace FlaxEngine
                 pair.Value.Free();
             classAttributesCacheCollectible.Clear();
 
+            // Unload native library handles associated for this assembly
+            string nativeLibraryName = assemblyOwnedNativeLibraries.GetValueOrDefault(assembly);
+            if (nativeLibraryName != null && loadedNativeLibraries.TryGetValue(nativeLibraryName, out IntPtr nativeLibrary))
+            {
+                NativeLibrary.Free(nativeLibrary);
+                loadedNativeLibraries.Remove(nativeLibraryName);
+            }
+            if (nativeLibraryName != null)
+                nativeLibraryPaths.Remove(nativeLibraryName);
+
             // Unload the ALC
             bool unloading = true;
             scriptingAssemblyLoadContext.Unloading += (alc) => { unloading = false; };

Source/Engine/Scripting/DotNet/CoreCLR.cpp

@@ -121,6 +121,12 @@ void* CoreCLR::GetStaticMethodPointer(const String& methodName)
     return fun;
 }
 
+void CoreCLR::RegisterNativeLibrary(const char* moduleName, const char* modulePath)
+{
+    static void* RegisterNativeLibraryPtr = CoreCLR::GetStaticMethodPointer(TEXT("RegisterNativeLibrary"));
+    CoreCLR::CallStaticMethod<void, const char*, const char*>(RegisterNativeLibraryPtr, moduleName, modulePath);
+}
+
 void* CoreCLR::Allocate(int size)
 {
 #if PLATFORM_WINDOWS

Source/Engine/Scripting/DotNet/CoreCLR.h

@@ -47,6 +47,8 @@ public:
         return ((fun)methodPtr)(args...);
     }
 
+    static void RegisterNativeLibrary(const char* moduleName, const char* modulePath);
+
     static const char* GetClassFullname(void* klass);
     static void* Allocate(int size);
     static void Free(void* ptr);

Source/Engine/Scripting/ManagedCLR/MCore.cpp

@@ -135,8 +135,9 @@ bool MCore::LoadEngine()
         return false;
 
     // Prepare managed side
-    const String hostExecutable = Platform::GetExecutableFilePath();
-    CoreCLR::CallStaticMethodByName<void, const Char*>(TEXT("Init"), hostExecutable.Get());
+    const StringAnsi hostExecutable(Platform::GetExecutableFilePath());
+    CoreCLR::CallStaticMethodByName<void>(TEXT("Init"));
+    CoreCLR::RegisterNativeLibrary("FlaxEngine", hostExecutable.Get());
 
     MRootDomain = New<MDomain>("Root");
     MDomains.Add(MRootDomain);

Source/Engine/Scripting/Scripting.cpp

@@ -34,6 +34,9 @@
 #include <ThirdParty/mono-2.0/mono/metadata/mono-debug.h>
 #include <ThirdParty/mono-2.0/mono/metadata/object.h>
 #endif
+#if USE_NETCORE
+#include "DotNet/CoreCLR.h"
+#endif
 
 extern void registerFlaxEngineInternalCalls();
 
@@ -408,6 +411,13 @@ bool Scripting::LoadBinaryModules(const String& path, const String& projectFolde
                     LOG(Error, "Failed to load C# assembly '{0}' for binary module {1}.", managedPath, name);
                     return true;
                 }
+#if USE_NETCORE
+                // Provide new path of hot-reloaded native library path for managed DllImport
+                if (nativePath.HasChars())
+                {
+                    CoreCLR::RegisterNativeLibrary(nameAnsi.Get(), StringAnsi(nativePath).Get());
+                }
+#endif
             }
 #endif