Merge branch 'FlaxEngine:master' into hash_set_crash_fix

This commit is contained in:
VitaminCpp
2025-11-28 15:55:15 +01:00
committed by GitHub
18 changed files with 186 additions and 104 deletions

View File

@@ -10,9 +10,10 @@
#include "Engine/Serialization/JsonTools.h"
#include "Engine/Serialization/JsonWriters.h"
#include "Editor/Cooker/PlatformTools.h"
#include "Engine/Engine/Globals.h"
#include "Editor/Editor.h"
#include "Editor/ProjectInfo.h"
#include "Engine/Engine/Globals.h"
#include "Editor/Utilities/EditorUtilities.h"
#if PLATFORM_MAC
#include <sys/stat.h>
#endif
@@ -127,7 +128,7 @@ bool CompileScriptsStep::DeployBinaries(CookingData& data, const String& path, c
const String dst = dstPath / StringUtils::GetFileName(file);
if (dst == file)
continue;
if (FileSystem::CopyFile(dst, file))
if (EditorUtilities::CopyFileIfNewer(dst, file))
{
data.Error(String::Format(TEXT("Failed to copy file from {0} to {1}."), file, dst));
return true;

View File

@@ -59,6 +59,7 @@ bool PrecompileAssembliesStep::Perform(CookingData& data)
data.StepProgress(infoMsg, 0);
// Override Newtonsoft.Json with AOT-version (one that doesn't use System.Reflection.Emit)
// TODO: remove it since EngineModule does properly reference AOT lib now
EditorUtilities::CopyFileIfNewer(data.ManagedCodeOutputPath / TEXT("Newtonsoft.Json.dll"), Globals::StartupFolder / TEXT("Source/Platforms/DotNet/AOT/Newtonsoft.Json.dll"));
FileSystem::DeleteFile(data.ManagedCodeOutputPath / TEXT("Newtonsoft.Json.xml"));
FileSystem::DeleteFile(data.ManagedCodeOutputPath / TEXT("Newtonsoft.Json.pdb"));

View File

@@ -30,6 +30,7 @@
#include "Engine/Profiler/ProfilerCPU.h"
#include "Engine/Profiler/ProfilerMemory.h"
#include "Engine/Scripting/ManagedCLR/MClass.h"
#include "Engine/Scripting/Internal/InternalCalls.h"
#include "Engine/Scripting/Scripting.h"
#if USE_EDITOR
#include "Editor/Editor.h"
@@ -346,17 +347,21 @@ int32 LoadingThread::Run()
ContentLoadTask* task;
ThisLoadThread = this;
MONO_THREAD_INFO_TYPE* monoThreadInfo = nullptr;
while (Platform::AtomicRead(&_exitFlag) == 0)
{
if (LoadTasks.try_dequeue(task))
{
Run(task);
MONO_THREAD_INFO_GET(monoThreadInfo);
}
else
{
MONO_ENTER_GC_SAFE_WITH_INFO(monoThreadInfo);
LoadTasksMutex.Lock();
LoadTasksSignal.Wait(LoadTasksMutex);
LoadTasksMutex.Unlock();
MONO_EXIT_GC_SAFE_WITH_INFO;
}
}

View File

@@ -79,10 +79,11 @@ namespace FlaxEngine.Interop
NativeLibrary.SetDllImportResolver(Assembly.GetExecutingAssembly(), NativeLibraryImportResolver);
// Change default culture to match with Mono runtime default culture
CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
System.Threading.Thread.CurrentThread.CurrentUICulture = CultureInfo.InvariantCulture;
var culture = CultureInfo.InvariantCulture;
CultureInfo.DefaultThreadCurrentCulture = culture;
CultureInfo.DefaultThreadCurrentUICulture = culture;
System.Threading.Thread.CurrentThread.CurrentCulture = culture;
System.Threading.Thread.CurrentThread.CurrentUICulture = culture;
}

View File

@@ -422,24 +422,8 @@ bool AndroidFileSystem::getFilesFromDirectoryTop(Array<String>& results, const c
if (S_ISREG(statEntry.st_mode) != 0)
{
// Validate with filter
const int32 fullPathLength = StringUtils::Length(fullPath);
const int32 searchPatternLength = StringUtils::Length(searchPattern);
if (searchPatternLength == 0 || StringUtils::Compare(searchPattern, "*") == 0)
{
// All files
}
else if (searchPattern[0] == '*' && searchPatternLength < fullPathLength && StringUtils::Compare(fullPath + fullPathLength - searchPatternLength + 1, searchPattern + 1) == 0)
{
// Path ending
}
else
{
// TODO: implement all cases in a generic way
continue;
}
// Add file
results.Add(String(fullPath));
if (FileSystem::PathFilterHelper(fullPath, searchPattern))
results.Add(String(fullPath));
}
}

View File

@@ -7,6 +7,7 @@
#include "Engine/Core/Types/StringView.h"
#include "Engine/Core/Collections/Array.h"
#include "Engine/Core/Math/Math.h"
#include "Engine/Core/Log.h"
#include "Engine/Engine/Globals.h"
bool FileSystemBase::ShowOpenFileDialog(Window* parentWindow, const StringView& initialDirectory, const StringView& filter, bool multiSelect, const StringView& title, Array<String, HeapAllocation>& filenames)
@@ -313,3 +314,39 @@ bool FileSystemBase::DirectoryCopyHelper(const String& dst, const String& src, b
return false;
}
bool FileSystemBase::PathFilterHelper(const char* path, const char* searchPattern)
{
// Validate with filter
const int32 pathLength = StringUtils::Length(path);
const int32 searchPatternLength = StringUtils::Length(searchPattern);
if (searchPatternLength == 0 ||
StringUtils::Compare(searchPattern, "*") == 0 ||
StringUtils::Compare(searchPattern, "*.*") == 0)
{
// All files
return true;
}
else if (searchPattern[0] == '*' && StringUtils::Find(searchPattern + 1, "*") == nullptr)
{
// Path ending
return searchPatternLength < pathLength && StringUtils::Compare(path + pathLength - searchPatternLength + 1, searchPattern + 1, searchPatternLength - 1) == 0;
}
else if (searchPattern[0] == '*' && searchPatternLength > 2 && searchPattern[searchPatternLength - 1] == '*')
{
// Contains pattern
bool match = false;
for (int32 i = 0; i < pathLength - searchPatternLength - 1; i++)
{
int32 len = Math::Min(searchPatternLength - 2, pathLength - i);
if (StringUtils::Compare(&path[i], &searchPattern[1], len) == 0)
return true;
}
}
else
{
// TODO: implement all cases in a generic way
LOG(Warning, "DirectoryGetFiles: Wildcard filter is not implemented ({})", String(searchPattern));
}
return false;
}

View File

@@ -284,6 +284,6 @@ public:
/// <returns>Relative path</returns>
static String ConvertAbsolutePathToRelative(const String& basePath, const String& path);
private:
static bool DirectoryCopyHelper(const String& dst, const String& src, bool withSubDirectories);
static bool PathFilterHelper(const char* path, const char* searchPattern);
};

View File

@@ -367,43 +367,8 @@ bool UnixFileSystem::getFilesFromDirectoryTop(Array<String>& results, const char
if (S_ISREG(statEntry.st_mode) != 0)
{
// Validate with filter
const int32 fullPathLength = StringUtils::Length(fullPath);
const int32 searchPatternLength = StringUtils::Length(searchPattern);
if (searchPatternLength == 0 ||
StringUtils::Compare(searchPattern, "*") == 0 ||
StringUtils::Compare(searchPattern, "*.*") == 0)
{
// All files
}
else if (searchPattern[0] == '*' && searchPatternLength < fullPathLength && StringUtils::Compare(fullPath + fullPathLength - searchPatternLength + 1, searchPattern + 1, searchPatternLength - 1) == 0)
{
// Path ending
}
else if (searchPattern[0] == '*' && searchPatternLength > 2 && searchPattern[searchPatternLength-1] == '*')
{
// Contains pattern
bool match = false;
for (int32 i = 0; i < pathLength - searchPatternLength - 1; i++)
{
int32 len = Math::Min(searchPatternLength - 2, pathLength - i);
if (StringUtils::Compare(&entry->d_name[i], &searchPattern[1], len) == 0)
{
match = true;
break;
}
}
if (!match)
continue;
}
else
{
// TODO: implement all cases in a generic way
LOG(Warning, "DirectoryGetFiles: Wildcard filter is not implemented");
continue;
}
// Add file
results.Add(String(fullPath));
if (FileSystemBase::PathFilterHelper(fullPath, searchPattern))
results.Add(String(fullPath));
}
}

View File

@@ -9,7 +9,7 @@
#if defined(__clang__)
// Helper utility to override vtable entry with automatic restore
// See BindingsGenerator.Cpp.cs that generates virtuall method wrappers for scripting to properly call overriden base method
// See BindingsGenerator.Cpp.cs that generates virtual method wrappers for scripting to properly call overriden base method
struct FLAXENGINE_API VTableFunctionInjector
{
void** VTableAddr;
@@ -100,3 +100,50 @@ T& InternalGetReference(T* obj)
DebugLog::ThrowNullReference();
return *obj;
}
#ifdef USE_MONO_AOT_COOP
// Cooperative Suspend - where threads suspend themselves when the runtime requests it.
// https://www.mono-project.com/docs/advanced/runtime/docs/coop-suspend/
typedef struct _MonoStackData {
void* stackpointer;
const char* function_name;
} MonoStackData;
#if BUILD_DEBUG
#define MONO_STACKDATA(x) MonoStackData x = { &x, __func__ }
#else
#define MONO_STACKDATA(x) MonoStackData x = { &x, NULL }
#endif
#define MONO_THREAD_INFO_TYPE struct MonoThreadInfo
DLLIMPORT extern "C" MONO_THREAD_INFO_TYPE* mono_thread_info_attach(void);
DLLIMPORT extern "C" void* mono_threads_enter_gc_safe_region_with_info(MONO_THREAD_INFO_TYPE* info, MonoStackData* stackdata);
DLLIMPORT extern "C" void mono_threads_exit_gc_safe_region_internal(void* cookie, MonoStackData* stackdata);
#ifndef _MONO_UTILS_FORWARD_
typedef struct _MonoDomain MonoDomain;
DLLIMPORT extern "C" MonoDomain* mono_domain_get(void);
#endif
#define MONO_ENTER_GC_SAFE \
do { \
MONO_STACKDATA(__gc_safe_dummy); \
void* __gc_safe_cookie = mono_threads_enter_gc_safe_region_internal(&__gc_safe_dummy)
#define MONO_EXIT_GC_SAFE \
mono_threads_exit_gc_safe_region_internal(__gc_safe_cookie, &__gc_safe_dummy); \
} while (0)
#define MONO_ENTER_GC_SAFE_WITH_INFO(info) \
do { \
MONO_STACKDATA(__gc_safe_dummy); \
void* __gc_safe_cookie = mono_threads_enter_gc_safe_region_with_info((info), &__gc_safe_dummy)
#define MONO_EXIT_GC_SAFE_WITH_INFO MONO_EXIT_GC_SAFE
#define MONO_THREAD_INFO_GET(info) if (!info && mono_domain_get()) info = mono_thread_info_attach()
#else
#define MONO_THREAD_INFO_TYPE void
#define MONO_ENTER_GC_SAFE
#define MONO_EXIT_GC_SAFE
#define MONO_ENTER_GC_SAFE_WITH_INFO(info)
#define MONO_EXIT_GC_SAFE_WITH_INFO
#define MONO_THREAD_INFO_GET(info)
#define mono_thread_info_attach() nullptr
#endif

View File

@@ -2147,7 +2147,13 @@ bool InitHostfxr()
#endif
// Adjust GC threads suspending mode to not block attached native threads (eg. Job System)
// https://www.mono-project.com/docs/advanced/runtime/docs/coop-suspend/
#if USE_MONO_AOT_COOP
Platform::SetEnvironmentVariable(TEXT("MONO_THREADS_SUSPEND"), TEXT("coop"));
Platform::SetEnvironmentVariable(TEXT("MONO_SLEEP_ABORT_LIMIT"), TEXT("5000")); // in ms
#else
Platform::SetEnvironmentVariable(TEXT("MONO_THREADS_SUSPEND"), TEXT("preemptive"));
#endif
#if defined(USE_MONO_AOT_MODE)
// Enable AOT mode (per-platform)
@@ -2155,9 +2161,9 @@ bool InitHostfxr()
#endif
// Platform-specific setup
#if PLATFORM_IOS || PLATFORM_SWITCH
setenv("MONO_AOT_MODE", "aot", 1);
setenv("DOTNET_SYSTEM_GLOBALIZATION_INVARIANT", "1", 1);
#if PLATFORM_IOS || PLATFORM_SWITCH || PLATFORM_PS4 || PLATFORM_PS5
Platform::SetEnvironmentVariable(TEXT("MONO_AOT_MODE"), TEXT("aot"));
Platform::SetEnvironmentVariable(TEXT("DOTNET_SYSTEM_GLOBALIZATION_INVARIANT"), TEXT("1"));
#endif
#ifdef USE_MONO_AOT_MODULE

View File

@@ -198,7 +198,7 @@ namespace FlaxEngine
private static void OnLocalizationChanged()
{
// Invariant-globalization only (see InitHostfxr with Mono)
#if !(PLATFORM_IOS || PLATFORM_SWITCH)
#if !(PLATFORM_IOS || PLATFORM_SWITCH || PLATFORM_PS4 || PLATFORM_PS5)
var currentThread = Thread.CurrentThread;
var language = Localization.CurrentLanguage;
if (language != null)
@@ -261,7 +261,7 @@ namespace FlaxEngine
internal static ManagedHandle CultureInfoToManaged(int lcid)
{
#if PLATFORM_IOS || PLATFORM_SWITCH
#if PLATFORM_IOS || PLATFORM_SWITCH || PLATFORM_PS4 || PLATFORM_PS5
// Invariant-globalization only (see InitHostfxr with Mono)
lcid = 0;
#endif

View File

@@ -15,6 +15,7 @@
#include "Engine/Profiler/ProfilerMemory.h"
#if USE_CSHARP
#include "Engine/Scripting/ManagedCLR/MCore.h"
#include "Engine/Scripting/Internal/InternalCalls.h"
#endif
#define JOB_SYSTEM_ENABLED 1
@@ -184,6 +185,7 @@ int32 JobSystemThread::Run()
JobData data;
Function<void(int32)> job;
bool attachCSharpThread = true;
MONO_THREAD_INFO_TYPE* monoThreadInfo = nullptr;
while (Platform::AtomicRead(&ExitFlag) == 0)
{
// Try to get a job
@@ -205,6 +207,7 @@ int32 JobSystemThread::Run()
{
MCore::Thread::Attach();
attachCSharpThread = false;
monoThreadInfo = mono_thread_info_attach();
}
#endif
@@ -244,9 +247,11 @@ int32 JobSystemThread::Run()
else
{
// Wait for signal
MONO_ENTER_GC_SAFE_WITH_INFO(monoThreadInfo);
JobsMutex.Lock();
JobsSignal.Wait(JobsMutex);
JobsMutex.Unlock();
MONO_EXIT_GC_SAFE_WITH_INFO;
}
}
return 0;

View File

@@ -15,6 +15,7 @@
#include "Engine/Platform/CPUInfo.h"
#include "Engine/Platform/Thread.h"
#include "Engine/Profiler/ProfilerMemory.h"
#include "Engine/Scripting/Internal/InternalCalls.h"
FLAXENGINE_API bool IsInMainThread()
{
@@ -117,6 +118,7 @@ int32 ThreadPool::ThreadProc()
Platform::SetThreadAffinityMask(THREAD_POOL_AFFINITY_MASK((int32)index));
#endif
ThreadPoolTask* task;
MONO_THREAD_INFO_TYPE* monoThreadInfo = nullptr;
// Work until end
while (Platform::AtomicRead(&ThreadPoolImpl::ExitFlag) == 0)
@@ -125,12 +127,15 @@ int32 ThreadPool::ThreadProc()
if (ThreadPoolImpl::Jobs.try_dequeue(task))
{
task->Execute();
MONO_THREAD_INFO_GET(monoThreadInfo);
}
else
{
MONO_ENTER_GC_SAFE_WITH_INFO(monoThreadInfo);
ThreadPoolImpl::JobsMutex.Lock();
ThreadPoolImpl::JobsSignal.Wait(ThreadPoolImpl::JobsMutex);
ThreadPoolImpl::JobsMutex.Unlock();
MONO_EXIT_GC_SAFE_WITH_INFO;
}
}

View File

@@ -462,7 +462,7 @@ namespace Flax.Build
else
{
// Copy to the destination folder
Utilities.FileCopy(assemblyPath, Path.Combine(dotnetOutputPath, assemblyFileName));
Utilities.FileCopy(assemblyPath, Path.Combine(dotnetOutputPath, assemblyFileName), Utilities.CopyMode.OverrideIfNewer);
}
};
if (Configuration.MaxConcurrency > 1 && Configuration.ConcurrencyProcessorScale > 0.0f && !dotnetAotDebug)

View File

@@ -42,7 +42,8 @@ namespace Flax.Build
BinaryModuleName = "FlaxEngine";
options.ScriptingAPI.Defines.Add("FLAX");
options.ScriptingAPI.Defines.Add("FLAX_ASSERTIONS");
options.ScriptingAPI.FileReferences.Add(Utilities.RemovePathRelativeParts(Path.Combine(Globals.EngineRoot, "Source", "Platforms", "DotNet", "Newtonsoft.Json.dll")));
var newtonsoftJsonPath = options.Platform?.HasDynamicCodeExecutionSupport ?? true ? "Newtonsoft.Json.dll" : "AOT/Newtonsoft.Json.dll";
options.ScriptingAPI.FileReferences.Add(Utilities.RemovePathRelativeParts(Path.Combine(Globals.EngineRoot, "Source", "Platforms", "DotNet", newtonsoftJsonPath)));
options.ScriptingAPI.SystemReferences.Add("System.ComponentModel.TypeConverter");
}
}

View File

@@ -142,7 +142,7 @@ namespace Flax.Build
}
/// <summary>
/// Returns true if this build target should use separate (aka main-only) executable file and separate runtime (in shared library). Used on platforms that don't support linking again executable file but only shared library (see HasExecutableFileReferenceSupport).
/// Returns true if this build target should use separate (aka main-only) executable file and separate runtime (in shared library). Used on platforms that don't support linking (or symbol lookup) against executable file but only shared library (see HasExecutableFileReferenceSupport).
/// </summary>
public virtual bool UseSeparateMainExecutable(BuildOptions buildOptions)
{

View File

@@ -320,6 +320,17 @@ namespace Flax.Build.Platforms
{
}
/// <summary>
/// Gets linker argument to reference a specific shared library file.
/// </summary>
/// <param name="graph">The graph.</param>
/// <param name="options">The options.</param>
/// <param name="library">The shared library file path.</param>
protected virtual string GetSharedLibraryLinkArg(TaskGraph graph, BuildOptions options, string library)
{
return string.Format("\"-l{0}\"", GetLibName(library));
}
/// <inheritdoc />
public override CompileOutput CompileCppFiles(TaskGraph graph, BuildOptions options, List<string> sourceFiles, string outputPath)
{
@@ -527,7 +538,8 @@ namespace Flax.Build.Platforms
// Input libraries
var libraryPaths = new HashSet<string>();
foreach (var library in linkEnvironment.InputLibraries)
var dynamicLibExt = Platform.SharedLibraryFileExtension;
foreach (var library in linkEnvironment.InputLibraries.Concat(options.Libraries))
{
var dir = Path.GetDirectoryName(library);
var ext = Path.GetExtension(library);
@@ -539,37 +551,12 @@ namespace Flax.Build.Platforms
{
// Skip executable
}
else if (ext == ".so")
else if (ext == dynamicLibExt)
{
// Link against dynamic library
task.PrerequisiteFiles.Add(library);
libraryPaths.Add(dir);
args.Add(string.Format("\"-l{0}\"", GetLibName(library)));
}
else
{
task.PrerequisiteFiles.Add(library);
args.Add(string.Format("\"{0}\"", GetLibName(library)));
}
}
foreach (var library in options.Libraries)
{
var dir = Path.GetDirectoryName(library);
var ext = Path.GetExtension(library);
if (string.IsNullOrEmpty(dir))
{
args.Add(string.Format("\"-l{0}\"", library));
}
else if (string.IsNullOrEmpty(ext))
{
// Skip executable
}
else if (ext == ".so")
{
// Link against dynamic library
task.PrerequisiteFiles.Add(library);
libraryPaths.Add(dir);
args.Add(string.Format("\"-l{0}\"", GetLibName(library)));
args.Add(GetSharedLibraryLinkArg(graph, options, library));
}
else
{
@@ -580,8 +567,7 @@ namespace Flax.Build.Platforms
// Input files (link static libraries last)
task.PrerequisiteFiles.AddRange(linkEnvironment.InputFiles);
foreach (var file in linkEnvironment.InputFiles.Where(x => !x.EndsWith(".a"))
.Concat(linkEnvironment.InputFiles.Where(x => x.EndsWith(".a"))))
foreach (var file in linkEnvironment.InputFiles.Where(x => !x.EndsWith(".a")).Concat(linkEnvironment.InputFiles.Where(x => x.EndsWith(".a"))))
{
args.Add(string.Format("\"{0}\"", file.Replace('\\', '/')));
}

View File

@@ -144,24 +144,51 @@ namespace Flax.Build
return new TwoWayEnumerator<T>(source.GetEnumerator());
}
/// <summary>
/// File copy modes.
/// </summary>
public enum CopyMode
{
/// <summary>
/// Copies the file to the destination, fails if it already exists.
/// </summary>
New,
/// <summary>
/// If destination file exists, it will be overriden.
/// </summary>
OverrideIfExists,
/// <summary>
/// If destination file exists, has the same size and is newer than source file, it won't be overriden (avoids unnecessary copies).
/// </summary>
OverrideIfNewer,
}
/// <summary>
/// Copies the file.
/// </summary>
/// <param name="srcFilePath">The source file path.</param>
/// <param name="dstFilePath">The destination file path.</param>
/// <param name="overwrite"><see langword="true" /> if the destination file can be overwritten; otherwise, <see langword="false" />.</param>
public static void FileCopy(string srcFilePath, string dstFilePath, bool overwrite = true)
/// <param name="mode">Copy operation modes.</param>
public static void FileCopy(string srcFilePath, string dstFilePath, CopyMode mode = CopyMode.OverrideIfExists)
{
if (string.IsNullOrEmpty(srcFilePath))
throw new ArgumentNullException(nameof(srcFilePath));
if (string.IsNullOrEmpty(dstFilePath))
throw new ArgumentNullException(nameof(dstFilePath));
if (mode == CopyMode.OverrideIfNewer &&
File.Exists(dstFilePath) &&
File.GetLastWriteTime(srcFilePath) <= File.GetLastWriteTime(dstFilePath) &&
new FileInfo(dstFilePath).Length == new FileInfo(srcFilePath).Length)
return;
Log.Verbose(srcFilePath + " -> " + dstFilePath);
try
{
File.Copy(srcFilePath, dstFilePath, overwrite);
File.Copy(srcFilePath, dstFilePath, mode != CopyMode.New);
}
catch (Exception ex)
{
@@ -173,6 +200,17 @@ namespace Flax.Build
}
}
/// <summary>
/// Copies the file.
/// </summary>
/// <param name="srcFilePath">The source file path.</param>
/// <param name="dstFilePath">The destination file path.</param>
/// <param name="overwrite"><see langword="true" /> if the destination file can be overwritten; otherwise, <see langword="false" />.</param>
public static void FileCopy(string srcFilePath, string dstFilePath, bool overwrite)
{
FileCopy(srcFilePath, dstFilePath, overwrite ? CopyMode.OverrideIfExists : CopyMode.New);
}
/// <summary>
/// Copies the directories.
/// </summary>