Fix crash on Clang-platforms when calling base method from overriden scripting method (C# or Visual Script)

Source/Engine/Scripting/InternalCalls.h

@@ -7,6 +7,30 @@
 #include "ScriptingType.h"
 #include "Types.h"
 
+#if defined(__clang__)
+// Helper utility to override vtable entry with automatic restore
+// See BindingsGenerator.Cpp.cs that generates virtuall method wrappers for scripting to properly call overriden base method
+struct FLAXENGINE_API VTableFunctionInjector
+{
+    void** VTableAddr;
+    void* OriginalValue;
+
+    VTableFunctionInjector(void* object, void* originalFunc, void* func)
+    {
+        void** vtable = *(void***)object;
+        const int32 vtableIndex = GetVTableIndex(vtable, 200, originalFunc);
+        VTableAddr = vtable + vtableIndex;
+        OriginalValue = *VTableAddr;
+        *VTableAddr = func;
+    }
+
+    ~VTableFunctionInjector()
+    {
+        *VTableAddr = OriginalValue;
+    }
+};
+#endif
+
 #if USE_MONO
 
 extern "C" FLAXENGINE_API void mono_add_internal_call(const char* name, const void* method);

Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cpp.cs

@@ -237,6 +237,41 @@ namespace Flax.Build.Bindings
             return $"({typeInfo}){value}";
         }
 
+        public static void GenerateCppVirtualWrapperCallBaseMethod(BuildData buildData, StringBuilder contents, VirtualClassInfo classInfo, FunctionInfo functionInfo, string scriptVTableBase, string scriptVTableOffset)
+        {
+            contents.AppendLine("            // Prevent stack overflow by calling native base method");
+            if (buildData.Toolchain is Platforms.UnixToolchain)
+            {
+                // Clang compiler
+                // TODO: secure VTableFunctionInjector with mutex (even at cost of performance)
+                contents.AppendLine($"            {functionInfo.UniqueName}_Signature funcPtr = &{classInfo.NativeName}::{functionInfo.Name};");
+                contents.AppendLine($"            VTableFunctionInjector vtableInjector(object, *(void**)&funcPtr, {scriptVTableBase}[{scriptVTableOffset} + 2]); // TODO: this is not thread-safe");
+                if (classInfo is InterfaceInfo)
+                {
+                    contents.Append($"            return (({classInfo.NativeName}*)(void*)object)->{functionInfo.Name}(");
+                }
+                else
+                {
+                    contents.Append("            return (object->*funcPtr)(");
+                }
+            }
+            else
+            {
+                // MSVC or other compiler
+                contents.Append($"            return (this->**({functionInfo.UniqueName}_Internal_Signature*)&{scriptVTableBase}[{scriptVTableOffset} + 2])(");
+            }
+            bool separator = false;
+            for (var i = 0; i < functionInfo.Parameters.Count; i++)
+            {
+                var parameterInfo = functionInfo.Parameters[i];
+                if (separator)
+                    contents.Append(", ");
+                separator = true;
+                contents.Append(parameterInfo.Name);
+            }
+            contents.AppendLine(");");
+        }
+
         private static string GenerateCppGetNativeClass(BuildData buildData, TypeInfo typeInfo, ApiTypeInfo caller, FunctionInfo functionInfo)
         {
             // Optimal path for in-build types
@@ -1198,19 +1233,7 @@ namespace Flax.Build.Bindings
 
             contents.AppendLine("        if (WrapperCallInstance == object)");
             contents.AppendLine("        {");
-            contents.AppendLine("            // Prevent stack overflow by calling native base method");
+            GenerateCppVirtualWrapperCallBaseMethod(buildData, contents, classInfo, functionInfo, "managedTypePtr->Script.ScriptVTableBase", scriptVTableOffset);
-            contents.AppendLine("            const auto scriptVTableBase = managedTypePtr->Script.ScriptVTableBase;");
-            contents.Append($"            return (this->**({functionInfo.UniqueName}_Internal_Signature*)&scriptVTableBase[{scriptVTableOffset} + 2])(");
-            separator = false;
-            for (var i = 0; i < functionInfo.Parameters.Count; i++)
-            {
-                var parameterInfo = functionInfo.Parameters[i];
-                if (separator)
-                    contents.Append(", ");
-                separator = true;
-                contents.Append(parameterInfo.Name);
-            }
-            contents.AppendLine(");");
             contents.AppendLine("        }");
             contents.AppendLine("        auto scriptVTable = (MMethod**)managedTypePtr->Script.ScriptVTable;");
             contents.AppendLine($"        ASSERT(scriptVTable && scriptVTable[{scriptVTableOffset}]);");
@@ -1327,8 +1350,12 @@ namespace Flax.Build.Bindings
                 }
                 var t = functionInfo.IsConst ? " const" : string.Empty;
                 contents.AppendLine($"    typedef {functionInfo.ReturnType} ({classInfo.NativeName}::*{functionInfo.UniqueName}_Signature)({thunkParams}){t};");
+                if (!(buildData.Toolchain is Platforms.UnixToolchain))
+                {
+                    // MSVC or other compiler
                     contents.AppendLine($"    typedef {functionInfo.ReturnType} ({classInfo.NativeName}Internal::*{functionInfo.UniqueName}_Internal_Signature)({thunkParams}){t};");
                 }
+            }
             contents.AppendLine("");
 
             contents.AppendLine("    static void SetupScriptVTable(MClass* mclass, void**& scriptVTable, void**& scriptVTableBase)");

Source/Tools/Flax.Build/Build/Plugins/VisualScriptingPlugin.cs

@@ -67,19 +67,7 @@ namespace Flax.Build.Plugins
             contents.AppendLine("        static THREADLOCAL void* WrapperCallInstance = nullptr;");
             contents.AppendLine("        if (WrapperCallInstance == object)");
             contents.AppendLine("        {");
-            contents.AppendLine("            // Prevent stack overflow by calling base method");
+            BindingsGenerator.GenerateCppVirtualWrapperCallBaseMethod(buildData, contents, classInfo, functionInfo, "object->GetType().Script.ScriptVTableBase", scriptVTableOffset);
-            contents.AppendLine("            const auto scriptVTableBase = object->GetType().Script.ScriptVTableBase;");
-            contents.Append($"            return (this->**({functionInfo.UniqueName}_Internal_Signature*)&scriptVTableBase[{scriptVTableOffset} + 2])(");
-            separator = false;
-            for (var i = 0; i < functionInfo.Parameters.Count; i++)
-            {
-                var parameterInfo = functionInfo.Parameters[i];
-                if (separator)
-                    contents.Append(", ");
-                separator = true;
-                contents.Append(parameterInfo.Name);
-            }
-            contents.AppendLine(");");
             contents.AppendLine("        }");
             contents.AppendLine("        auto scriptVTable = (VisualScript::Method**)object->GetType().Script.ScriptVTable;");
             contents.AppendLine($"        ASSERT(scriptVTable && scriptVTable[{scriptVTableOffset}]);");