From db47531a6d2a669ee148ef40c899b88b9840d833 Mon Sep 17 00:00:00 2001 From: Ari Vuollet Date: Sun, 27 Apr 2025 17:20:06 +0300 Subject: [PATCH] Support marshalling Arrays as dynamic arrays in scripting API --- .../Engine/NativeInterop.Marshallers.cs | 211 ++++++++++++++++++ .../Bindings/BindingsGenerator.CSharp.cs | 25 ++- .../Bindings/BindingsGenerator.Parsing.cs | 3 + .../Tools/Flax.Build/Bindings/FunctionInfo.cs | 3 + 4 files changed, 240 insertions(+), 2 deletions(-) diff --git a/Source/Engine/Engine/NativeInterop.Marshallers.cs b/Source/Engine/Engine/NativeInterop.Marshallers.cs index 0f3c9beee..009855a47 100644 --- a/Source/Engine/Engine/NativeInterop.Marshallers.cs +++ b/Source/Engine/Engine/NativeInterop.Marshallers.cs @@ -658,6 +658,217 @@ namespace FlaxEngine.Interop } } +#if FLAX_EDITOR + [HideInEditor] +#endif + [CustomMarshaller(typeof(List<>), MarshalMode.ManagedToUnmanagedIn, typeof(ListMarshaller<,>.ManagedToNative))] + [CustomMarshaller(typeof(List<>), MarshalMode.UnmanagedToManagedOut, typeof(ListMarshaller<,>.ManagedToNative))] + [CustomMarshaller(typeof(List<>), MarshalMode.ManagedToUnmanagedOut, typeof(ListMarshaller<,>.NativeToManaged))] + [CustomMarshaller(typeof(List<>), MarshalMode.UnmanagedToManagedIn, typeof(ListMarshaller<,>.NativeToManaged))] + [CustomMarshaller(typeof(List<>), MarshalMode.ElementOut, typeof(ListMarshaller<,>.NativeToManaged))] + [CustomMarshaller(typeof(List<>), MarshalMode.ManagedToUnmanagedRef, typeof(ListMarshaller<,>.Bidirectional))] + [CustomMarshaller(typeof(List<>), MarshalMode.UnmanagedToManagedRef, typeof(ListMarshaller<,>.Bidirectional))] + [CustomMarshaller(typeof(List<>), MarshalMode.ElementRef, typeof(ListMarshaller<,>))] + [ContiguousCollectionMarshaller] + public static unsafe class ListMarshaller where TUnmanagedElement : unmanaged + { +#if FLAX_EDITOR + [HideInEditor] +#endif + public static class NativeToManaged + { + public static List AllocateContainerForManagedElements(TUnmanagedElement* unmanaged, int numElements) + { + if (unmanaged is null) + return null; + return new List(numElements); + } + + public static TUnmanagedElement* AllocateContainerForUnmanagedElements(List managed, out int numElements) + { + if (managed is null) + { + numElements = 0; + return null; + } + numElements = managed.Count; + (ManagedHandle managedArrayHandle, _) = ManagedArray.AllocatePooledArray(managed.Count); + return (TUnmanagedElement*)ManagedHandle.ToIntPtr(managedArrayHandle); + } + + public static ReadOnlySpan GetManagedValuesSource(List managed) => CollectionsMarshal.AsSpan(managed); + + public static Span GetUnmanagedValuesDestination(TUnmanagedElement* unmanaged) + { + if (unmanaged == null) + return Span.Empty; + ManagedArray managedArray = Unsafe.As(ManagedHandle.FromIntPtr(new IntPtr(unmanaged)).Target); + return managedArray.ToSpan(); + } + + public static Span GetManagedValuesDestination(List managed) => CollectionsMarshal.AsSpan(managed); + + public static ReadOnlySpan GetUnmanagedValuesSource(TUnmanagedElement* unmanaged, int numElements) + { + if (unmanaged == null) + return ReadOnlySpan.Empty; + ManagedArray managedArray = Unsafe.As(ManagedHandle.FromIntPtr(new IntPtr(unmanaged)).Target); + return managedArray.ToSpan(); + } + + public static void Free(TUnmanagedElement* unmanaged) + { + if (unmanaged == null) + return; + ManagedHandle handle = ManagedHandle.FromIntPtr(new IntPtr(unmanaged)); + (Unsafe.As(handle.Target)).Free(); + handle.Free(); + } + + public static Span GetUnmanagedValuesDestination(TUnmanagedElement* unmanaged, int numElements) + { + if (unmanaged == null) + return Span.Empty; + ManagedArray managedArray = Unsafe.As(ManagedHandle.FromIntPtr(new IntPtr(unmanaged)).Target); + return managedArray.ToSpan(); + } + } + +#if FLAX_EDITOR + [HideInEditor] +#endif + public ref struct ManagedToNative + { + List sourceArray; + ManagedArray managedArray; + ManagedHandle managedHandle; + + public void FromManaged(List managed) + { + if (managed is null) + return; + sourceArray = managed; + (managedHandle, managedArray) = ManagedArray.AllocatePooledArray(managed.Count); + } + + public ReadOnlySpan GetManagedValuesSource() => CollectionsMarshal.AsSpan(sourceArray); + + public Span GetUnmanagedValuesDestination() => managedArray != null ? managedArray.ToSpan() : Span.Empty; + + public TUnmanagedElement* ToUnmanaged() => (TUnmanagedElement*)ManagedHandle.ToIntPtr(managedHandle); + + public void Free() => managedArray?.FreePooled(); + } + +#if FLAX_EDITOR + [HideInEditor] +#endif + public struct Bidirectional + { + List sourceArray; + ManagedArray managedArray; + ManagedHandle handle; // Valid only for pooled array + + public void FromManaged(List managed) + { + if (managed == null) + return; + sourceArray = managed; + (handle, managedArray) = ManagedArray.AllocatePooledArray(managed.Count); + } + + public ReadOnlySpan GetManagedValuesSource() => CollectionsMarshal.AsSpan(sourceArray); + + public Span GetUnmanagedValuesDestination() + { + if (managedArray == null) + return Span.Empty; + return managedArray.ToSpan(); + } + + public TUnmanagedElement* ToUnmanaged() => (TUnmanagedElement*)ManagedHandle.ToIntPtr(handle); + + public void FromUnmanaged(TUnmanagedElement* unmanaged) + { + ManagedArray arr = Unsafe.As(ManagedHandle.FromIntPtr(new IntPtr(unmanaged)).Target); + if (sourceArray == null || sourceArray.Count != arr.Length) + { + // Array was resized when returned from native code (as ref parameter) + managedArray.FreePooled(); + if (sourceArray.Capacity < arr.Length) + sourceArray.Capacity = arr.Length; + for (int i = sourceArray.Count - 1; i > arr.Length; i--) + sourceArray.RemoveAt(i); + for (int i = sourceArray.Count; i < arr.Length; i++) + sourceArray.Add(default); + if (sourceArray.Count != arr.Length) + throw new Exception(); + managedArray = arr; + handle = new ManagedHandle(); // Invalidate as it's not pooled array anymore + } + } + + public ReadOnlySpan GetUnmanagedValuesSource(int numElements) + { + if (managedArray == null) + return ReadOnlySpan.Empty; + return managedArray.ToSpan(); + } + + public Span GetManagedValuesDestination(int numElements) => CollectionsMarshal.AsSpan(sourceArray); + + public List ToManaged() => sourceArray; + + public void Free() + { + if (handle.IsAllocated) + managedArray.FreePooled(); + } + } + + public static TUnmanagedElement* AllocateContainerForUnmanagedElements(List managed, out int numElements) + { + if (managed is null) + { + numElements = 0; + return null; + } + numElements = managed.Count; + (ManagedHandle managedArrayHandle, _) = ManagedArray.AllocatePooledArray(managed.Count); + return (TUnmanagedElement*)ManagedHandle.ToIntPtr(managedArrayHandle); + } + + public static ReadOnlySpan GetManagedValuesSource(List managed) => CollectionsMarshal.AsSpan(managed); + + public static Span GetUnmanagedValuesDestination(TUnmanagedElement* unmanaged, int numElements) + { + if (unmanaged == null) + return Span.Empty; + ManagedArray unmanagedArray = Unsafe.As(ManagedHandle.FromIntPtr(new IntPtr(unmanaged)).Target); + return unmanagedArray.ToSpan(); + } + + public static List AllocateContainerForManagedElements(TUnmanagedElement* unmanaged, int numElements) => unmanaged is null ? null : new List(numElements); + + public static Span GetManagedValuesDestination(List managed) => CollectionsMarshal.AsSpan(managed); + + public static ReadOnlySpan GetUnmanagedValuesSource(TUnmanagedElement* unmanaged, int numElements) + { + if (unmanaged == null) + return ReadOnlySpan.Empty; + ManagedArray array = Unsafe.As(ManagedHandle.FromIntPtr(new IntPtr(unmanaged)).Target); + return array.ToSpan(); + } + + public static void Free(TUnmanagedElement* unmanaged) + { + if (unmanaged == null) + return; + ManagedHandle handle = ManagedHandle.FromIntPtr(new IntPtr(unmanaged)); + Unsafe.As(handle.Target).FreePooled(); + } + } + #if FLAX_EDITOR [HideInEditor] #endif diff --git a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.CSharp.cs b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.CSharp.cs index 54329634f..7413730df 100644 --- a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.CSharp.cs +++ b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.CSharp.cs @@ -686,6 +686,8 @@ namespace Flax.Build.Bindings parameterMarshalType = "MarshalUsing(typeof(FlaxEngine.Interop.SystemArrayMarshaller))"; else if (nativeType == "object[]") parameterMarshalType = "MarshalUsing(typeof(FlaxEngine.Interop.SystemObjectArrayMarshaller))"; + else if (parameterInfo.Type.Type == "Array" && parameterInfo.MarshalAsDynamicArray) + parameterMarshalType = $"MarshalUsing(typeof(FlaxEngine.Interop.ListMarshaller<,>), CountElementName = \"__{parameterInfo.Name}Count\")"; else if (parameterInfo.Type.Type == "Array" && parameterInfo.Type.GenericArgs.Count > 0 && parameterInfo.Type.GenericArgs[0].Type == "bool") parameterMarshalType = $"MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.U1, SizeParamIndex = {(!functionInfo.IsStatic ? 1 : 0) + functionInfo.Parameters.Count + (functionInfo.Glue.CustomParameters.FindIndex(x => x.Name == $"__{parameterInfo.Name}Count"))})"; else if (parameterInfo.Type.Type == "Array" || parameterInfo.Type.Type == "Span" || parameterInfo.Type.Type == "DataContainer" || parameterInfo.Type.Type == "BytesContainer" || nativeType == "Array") @@ -716,6 +718,11 @@ namespace Flax.Build.Bindings // Out parameters that need additional converting will be converted at the native side (eg. object reference) if (parameterInfo.IsOut && !string.IsNullOrEmpty(GenerateCSharpManagedToNativeConverter(buildData, parameterInfo.Type, caller))) nativeType = parameterInfo.Type.Type; + if (parameterInfo.Type.Type == "Array" && parameterInfo.MarshalAsDynamicArray) + { + var dynamicArrayType = TypeInfo.FromString($"List<{parameterInfo.Type.GenericArgs[0].Type}>"); + nativeType = "System.Collections.Generic." + GenerateCSharpManagedToNativeType(buildData, dynamicArrayType, caller, true); + } contents.Append(nativeType); contents.Append(' '); @@ -736,7 +743,9 @@ namespace Flax.Build.Bindings if (parameterInfo.IsOut && parameterInfo.DefaultValue == "var __resultAsRef") { // TODO: make this code shared with MarshalUsing selection from the above - if (parameterInfo.Type.Type == "Array" || parameterInfo.Type.Type == "Span" || parameterInfo.Type.Type == "DataContainer" || parameterInfo.Type.Type == "BytesContainer") + if (parameterInfo.Type.Type == "Array" && parameterInfo.MarshalAsDynamicArray) + parameterMarshalType = $"MarshalUsing(typeof(FlaxEngine.Interop.ListMarshaller<,>), CountElementName = \"{parameterInfo.Name}Count\")"; + else if (parameterInfo.Type.Type == "Array" || parameterInfo.Type.Type == "Span" || parameterInfo.Type.Type == "DataContainer" || parameterInfo.Type.Type == "BytesContainer") parameterMarshalType = $"MarshalUsing(typeof(FlaxEngine.Interop.ArrayMarshaller<,>), CountElementName = \"{parameterInfo.Name}Count\")"; else if (parameterInfo.Type.Type == "Dictionary") parameterMarshalType = "MarshalUsing(typeof(FlaxEngine.Interop.DictionaryMarshaller<,>), ConstantElementCount = 0)"; @@ -772,7 +781,12 @@ namespace Flax.Build.Bindings if (parameterInfo.Type.IsArray || parameterInfo.Type.Type == "Array" || parameterInfo.Type.Type == "Span" || parameterInfo.Type.Type == "BytesContainer" || parameterInfo.Type.Type == "DataContainer" || parameterInfo.Type.Type == "BitArray") { if (!parameterInfo.IsOut) - contents.Append($"var __{parameterInfo.Name}Count = {(isSetter ? "value" : parameterInfo.Name)}?.Length ?? 0; "); + { + if (parameterInfo.Type.Type == "Array" && parameterInfo.MarshalAsDynamicArray) + contents.Append($"var __{parameterInfo.Name}Count = {(isSetter ? "value" : parameterInfo.Name)}?.Count ?? 0; "); + else + contents.Append($"var __{parameterInfo.Name}Count = {(isSetter ? "value" : parameterInfo.Name)}?.Length ?? 0; "); + } } } #endif @@ -1366,6 +1380,13 @@ namespace Flax.Build.Bindings contents.Append('[').Append(parameterInfo.Attributes).Append(']').Append(' '); var managedType = GenerateCSharpNativeToManaged(buildData, parameterInfo.Type, classInfo); + if (parameterInfo.Type.Type == "Array" && parameterInfo.MarshalAsDynamicArray) + { + var dynamicArrayType = TypeInfo.FromString($"List<{parameterInfo.Type.GenericArgs[0].Type}>"); + managedType = GenerateCSharpNativeToManaged(buildData, dynamicArrayType, classInfo); + managedType = $"System.Collections.Generic.{managedType}"; + } + if (parameterInfo.IsOut) contents.Append("out "); else if (parameterInfo.IsRef) diff --git a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Parsing.cs b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Parsing.cs index 7841b98c3..c21e45a57 100644 --- a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Parsing.cs +++ b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Parsing.cs @@ -378,6 +378,9 @@ namespace Flax.Build.Bindings case "defaultvalue": currentParam.DefaultValue = tag.Value; break; + case "dynamicarray": + currentParam.MarshalAsDynamicArray = true; + break; default: bool valid = false; ParseFunctionParameterTag?.Invoke(ref valid, tag, ref currentParam); diff --git a/Source/Tools/Flax.Build/Bindings/FunctionInfo.cs b/Source/Tools/Flax.Build/Bindings/FunctionInfo.cs index d8b440ccd..f39a93961 100644 --- a/Source/Tools/Flax.Build/Bindings/FunctionInfo.cs +++ b/Source/Tools/Flax.Build/Bindings/FunctionInfo.cs @@ -16,6 +16,7 @@ namespace Flax.Build.Bindings public TypeInfo Type; public string DefaultValue; public string Attributes; + public bool MarshalAsDynamicArray; public bool IsRef; public bool IsOut; public bool IsThis; @@ -35,6 +36,7 @@ namespace Flax.Build.Bindings BindingsGenerator.Write(writer, DefaultValue); BindingsGenerator.Write(writer, Attributes); // TODO: convert into flags + writer.Write(MarshalAsDynamicArray); writer.Write(IsRef); writer.Write(IsOut); writer.Write(IsThis); @@ -48,6 +50,7 @@ namespace Flax.Build.Bindings DefaultValue = BindingsGenerator.Read(reader, DefaultValue); Attributes = BindingsGenerator.Read(reader, Attributes); // TODO: convert into flags + MarshalAsDynamicArray = reader.ReadBoolean(); IsRef = reader.ReadBoolean(); IsOut = reader.ReadBoolean(); IsThis = reader.ReadBoolean();