From 7cbafcd86bc2bef96b71ece22295d235fc6b6d28 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Fri, 31 Mar 2023 14:41:42 +0200 Subject: [PATCH] Implement C# AOT process for .NET 7 for Windows platform --- Source/Editor/Cooker/CookingData.h | 26 ++ Source/Editor/Cooker/GameCooker.cpp | 14 +- .../Platform/Android/AndroidPlatformTools.cpp | 2 - .../Cooker/Platform/GDK/GDKPlatformTools.cpp | 4 +- .../Cooker/Platform/GDK/GDKPlatformTools.h | 2 +- .../Cooker/Platform/UWP/UWPPlatformTools.cpp | 4 +- .../Cooker/Platform/UWP/UWPPlatformTools.h | 2 +- Source/Editor/Cooker/PlatformTools.h | 11 +- .../Cooker/Steps/CompileScriptsStep.cpp | 2 +- Source/Editor/Cooker/Steps/DeployDataStep.cpp | 89 ++-- .../Cooker/Steps/PrecompileAssembliesStep.cpp | 73 +++- .../Cooker/Steps/PrecompileAssembliesStep.h | 4 +- Source/Editor/Utilities/EditorUtilities.cpp | 46 +++ Source/Editor/Utilities/EditorUtilities.h | 3 + Source/Engine/Scripting/Runtime/DotNet.cpp | 32 +- .../Platforms/DotNet/AOT/Newtonsoft.Json.dll | 3 + .../UWP/Binaries/Newtonsoft.Json.dll | 3 - Source/ThirdParty/nethost/nethost.Build.cs | 13 +- .../Flax.Build/Build/DotNet/DotNetAOT.cs | 389 ++++++++++++++++++ Source/Tools/Flax.Build/Build/Platform.cs | 2 +- .../Deps/Dependencies/NewtonsoftJson.cs | 6 +- .../Tools/Flax.Build/Utilities/MonoCecil.cs | 61 +++ .../Tools/Flax.Build/Utilities/Utilities.cs | 23 +- 23 files changed, 734 insertions(+), 80 deletions(-) create mode 100644 Source/Platforms/DotNet/AOT/Newtonsoft.Json.dll delete mode 100644 Source/Platforms/UWP/Binaries/Newtonsoft.Json.dll create mode 100644 Source/Tools/Flax.Build/Build/DotNet/DotNetAOT.cs diff --git a/Source/Editor/Cooker/CookingData.h b/Source/Editor/Cooker/CookingData.h index e0422e679..487c796bf 100644 --- a/Source/Editor/Cooker/CookingData.h +++ b/Source/Editor/Cooker/CookingData.h @@ -154,6 +154,32 @@ API_ENUM() enum class BuildConfiguration extern FLAXENGINE_API const Char* ToString(const BuildConfiguration configuration); +/// +/// .NET Ahead of Time Compilation (AOT) modes. +/// +enum class DotNetAOTModes +{ + /// + /// AOT is not used. + /// + None, + + /// + /// Use .NET Native IL Compiler (shorten as ILC) to convert all C# assemblies in native platform executable binary. + /// + ILC, + + /// + /// Use Mono AOT to cross-compile all used C# assemblies into native platform shared libraries. + /// + MonoAOTDynamic, + + /// + /// Use Mono AOT to cross-compile all used C# assemblies into native platform static libraries which can be linked into a single shared library. + /// + MonoAOTStatic, +}; + #define BUILD_STEP_CANCEL_CHECK if (GameCooker::IsCancelRequested()) return true /// diff --git a/Source/Editor/Cooker/GameCooker.cpp b/Source/Editor/Cooker/GameCooker.cpp index aac9e744a..f2aa6fc4a 100644 --- a/Source/Editor/Cooker/GameCooker.cpp +++ b/Source/Editor/Cooker/GameCooker.cpp @@ -168,6 +168,16 @@ const Char* ToString(const BuildConfiguration configuration) } } +bool PlatformTools::IsNativeCodeFile(CookingData& data, const String& file) +{ + const String filename = StringUtils::GetFileName(file); + if (filename.Contains(TEXT(".CSharp")) || + filename.Contains(TEXT("Newtonsoft.Json"))) + return false; + // TODO: maybe use Mono.Cecil via Flax.Build to read assembly image metadata and check if it contains C#? + return true; +} + bool CookingData::AssetTypeStatistics::operator<(const AssetTypeStatistics& other) const { if (ContentSize != other.ContentSize) @@ -636,9 +646,9 @@ bool GameCookerImpl::Build() // Build Started CallEvent(GameCooker::EventType::BuildStarted); + data.Tools->OnBuildStarted(data); for (int32 stepIndex = 0; stepIndex < Steps.Count(); stepIndex++) Steps[stepIndex]->OnBuildStarted(data); - data.Tools->OnBuildStarted(data); data.InitProgress(Steps.Count()); // Execute all steps in a sequence @@ -705,9 +715,9 @@ bool GameCookerImpl::Build() } IsRunning = false; CancelFlag = 0; - data.Tools->OnBuildEnded(data, failed); for (int32 stepIndex = 0; stepIndex < Steps.Count(); stepIndex++) Steps[stepIndex]->OnBuildEnded(data, failed); + data.Tools->OnBuildEnded(data, failed); CallEvent(failed ? GameCooker::EventType::BuildFailed : GameCooker::EventType::BuildDone); Delete(Data); Data = nullptr; diff --git a/Source/Editor/Cooker/Platform/Android/AndroidPlatformTools.cpp b/Source/Editor/Cooker/Platform/Android/AndroidPlatformTools.cpp index b4fe0fe6e..dda674c66 100644 --- a/Source/Editor/Cooker/Platform/Android/AndroidPlatformTools.cpp +++ b/Source/Editor/Cooker/Platform/Android/AndroidPlatformTools.cpp @@ -116,8 +116,6 @@ void AndroidPlatformTools::OnBuildStarted(CookingData& data) data.DataOutputPath /= TEXT("app/assets"); data.NativeCodeOutputPath /= TEXT("app/assets"); data.ManagedCodeOutputPath /= TEXT("app/assets"); - - PlatformTools::OnBuildStarted(data); } bool AndroidPlatformTools::OnPostProcess(CookingData& data) diff --git a/Source/Editor/Cooker/Platform/GDK/GDKPlatformTools.cpp b/Source/Editor/Cooker/Platform/GDK/GDKPlatformTools.cpp index 41c1a9722..28032ad07 100644 --- a/Source/Editor/Cooker/Platform/GDK/GDKPlatformTools.cpp +++ b/Source/Editor/Cooker/Platform/GDK/GDKPlatformTools.cpp @@ -37,9 +37,9 @@ GDKPlatformTools::GDKPlatformTools() } } -bool GDKPlatformTools::UseAOT() const +DotNetAOTModes GDKPlatformTools::UseAOT() const { - return true; + return DotNetAOTModes::MonoAOTDynamic; } bool GDKPlatformTools::OnScriptsStepDone(CookingData& data) diff --git a/Source/Editor/Cooker/Platform/GDK/GDKPlatformTools.h b/Source/Editor/Cooker/Platform/GDK/GDKPlatformTools.h index 350d95257..6f4a138a8 100644 --- a/Source/Editor/Cooker/Platform/GDK/GDKPlatformTools.h +++ b/Source/Editor/Cooker/Platform/GDK/GDKPlatformTools.h @@ -26,7 +26,7 @@ public: public: // [PlatformTools] - bool UseAOT() const override; + DotNetAOTModes UseAOT() const override; bool OnScriptsStepDone(CookingData& data) override; bool OnDeployBinaries(CookingData& data) override; void OnConfigureAOT(CookingData& data, AotConfig& config) override; diff --git a/Source/Editor/Cooker/Platform/UWP/UWPPlatformTools.cpp b/Source/Editor/Cooker/Platform/UWP/UWPPlatformTools.cpp index 9b0e54ea2..10aa13dd5 100644 --- a/Source/Editor/Cooker/Platform/UWP/UWPPlatformTools.cpp +++ b/Source/Editor/Cooker/Platform/UWP/UWPPlatformTools.cpp @@ -37,9 +37,9 @@ ArchitectureType UWPPlatformTools::GetArchitecture() const return _arch; } -bool UWPPlatformTools::UseAOT() const +DotNetAOTModes UWPPlatformTools::UseAOT() const { - return true; + return DotNetAOTModes::MonoAOTDynamic; } bool UWPPlatformTools::OnScriptsStepDone(CookingData& data) diff --git a/Source/Editor/Cooker/Platform/UWP/UWPPlatformTools.h b/Source/Editor/Cooker/Platform/UWP/UWPPlatformTools.h index ddd3a74a5..5ef9cb2fe 100644 --- a/Source/Editor/Cooker/Platform/UWP/UWPPlatformTools.h +++ b/Source/Editor/Cooker/Platform/UWP/UWPPlatformTools.h @@ -29,7 +29,7 @@ public: const Char* GetName() const override; PlatformType GetPlatform() const override; ArchitectureType GetArchitecture() const override; - bool UseAOT() const override; + DotNetAOTModes UseAOT() const override; bool OnScriptsStepDone(CookingData& data) override; bool OnDeployBinaries(CookingData& data) override; void OnConfigureAOT(CookingData& data, AotConfig& config) override; diff --git a/Source/Editor/Cooker/PlatformTools.h b/Source/Editor/Cooker/PlatformTools.h index bba16415b..75c705678 100644 --- a/Source/Editor/Cooker/PlatformTools.h +++ b/Source/Editor/Cooker/PlatformTools.h @@ -15,7 +15,6 @@ class TextureBase; class FLAXENGINE_API PlatformTools { public: - /// /// Finalizes an instance of the class. /// @@ -44,9 +43,9 @@ public: /// /// Gets the value indicating whenever platform requires AOT (needs C# assemblies to be precompiled). /// - virtual bool UseAOT() const + virtual DotNetAOTModes UseAOT() const { - return false; + return DotNetAOTModes::None; } /// @@ -75,13 +74,9 @@ public: /// The cooking data. /// The file path. /// True if it's a native file, otherwise false. - virtual bool IsNativeCodeFile(CookingData& data, const String& file) - { - return false; - } + virtual bool IsNativeCodeFile(CookingData& data, const String& file); public: - /// /// Called when game building starts. /// diff --git a/Source/Editor/Cooker/Steps/CompileScriptsStep.cpp b/Source/Editor/Cooker/Steps/CompileScriptsStep.cpp index c6b1a824f..d9cfa8bf9 100644 --- a/Source/Editor/Cooker/Steps/CompileScriptsStep.cpp +++ b/Source/Editor/Cooker/Steps/CompileScriptsStep.cpp @@ -204,7 +204,7 @@ bool CompileScriptsStep::Perform(CookingData& data) // Assume FlaxGame was prebuilt for target platform args += TEXT(" -SkipTargets=FlaxGame"); } - for (auto& define : data.CustomDefines) + for (const String& define : data.CustomDefines) { args += TEXT(" -D"); args += define; diff --git a/Source/Editor/Cooker/Steps/DeployDataStep.cpp b/Source/Editor/Cooker/Steps/DeployDataStep.cpp index 4e4227305..7947bb7a3 100644 --- a/Source/Editor/Cooker/Steps/DeployDataStep.cpp +++ b/Source/Editor/Cooker/Steps/DeployDataStep.cpp @@ -10,6 +10,7 @@ #include "Engine/Renderer/AntiAliasing/SMAA.h" #include "Engine/Engine/Globals.h" #include "Editor/Cooker/PlatformTools.h" +#include "Editor/Utilities/EditorUtilities.h" bool DeployDataStep::Perform(CookingData& data) { @@ -31,30 +32,20 @@ bool DeployDataStep::Perform(CookingData& data) FileSystem::CreateDirectory(contentDir); const String dstMono = data.DataOutputPath / TEXT("Mono"); #if USE_NETCORE - // TODO: Optionally copy all files needed for self-contained deployment { // Remove old Mono files FileSystem::DeleteDirectory(dstMono); FileSystem::DeleteFile(data.DataOutputPath / TEXT("MonoPosixHelper.dll")); } -#else - if (!FileSystem::DirectoryExists(dstMono)) + String dstDotnet = data.DataOutputPath / TEXT("Dotnet"); + const DotNetAOTModes aotMode = data.Tools->UseAOT(); + const bool usAOT = aotMode != DotNetAOTModes::None; + if (usAOT) { - // Deploy Mono files (from platform data folder) - const String srcMono = depsRoot / TEXT("Mono"); - if (!FileSystem::DirectoryExists(srcMono)) - { - data.Error(TEXT("Missing Mono runtime data files.")); - return true; - } - if (FileSystem::CopyDirectory(dstMono, srcMono, true)) - { - data.Error(TEXT("Failed to copy Mono runtime data files.")); - return true; - } + // Deploy Dotnet files into intermediate cooking directory for AOT + FileSystem::DeleteDirectory(dstDotnet); + dstDotnet = data.ManagedCodeOutputPath; } -#endif - const String dstDotnet = data.DataOutputPath / TEXT("Dotnet"); if (buildSettings.SkipDotnetPackaging && data.Tools->UseSystemDotnet()) { // Use system-installed .Net Runtime @@ -69,7 +60,7 @@ bool DeployDataStep::Perform(CookingData& data) { // Use prebuilt .Net installation for that platform LOG(Info, "Using .Net Runtime {} at {}", data.Tools->GetName(), srcDotnet); - if (FileSystem::CopyDirectory(dstDotnet, srcDotnet, true)) + if (EditorUtilities::CopyDirectoryIfNewer(dstDotnet, srcDotnet, true)) { data.Error(TEXT("Failed to copy .Net runtime data files.")); return true; @@ -92,7 +83,7 @@ bool DeployDataStep::Perform(CookingData& data) canUseSystemDotnet = PLATFORM_TYPE == PlatformType::Mac; break; } - if (canUseSystemDotnet) + if (canUseSystemDotnet && (aotMode == DotNetAOTModes::None || aotMode == DotNetAOTModes::ILC)) { // Ask Flax.Build to provide .Net SDK location for the current platform String sdks; @@ -130,6 +121,7 @@ bool DeployDataStep::Perform(CookingData& data) } Sorting::QuickSort(versions.Get(), versions.Count()); const String version = versions.Last(); + FileSystem::NormalizePath(srcDotnet); LOG(Info, "Using .Net Runtime {} at {}", version, srcDotnet); // Deploy runtime files @@ -137,8 +129,15 @@ bool DeployDataStep::Perform(CookingData& data) FileSystem::CopyFile(dstDotnet / TEXT("LICENSE.TXT"), srcDotnet / TEXT("LICENSE.TXT")); FileSystem::CopyFile(dstDotnet / TEXT("THIRD-PARTY-NOTICES.TXT"), srcDotnet / TEXT("ThirdPartyNotices.txt")); FileSystem::CopyFile(dstDotnet / TEXT("THIRD-PARTY-NOTICES.TXT"), srcDotnet / TEXT("THIRD-PARTY-NOTICES.TXT")); - failed |= FileSystem::CopyDirectory(dstDotnet / TEXT("host/fxr") / version, srcDotnet / TEXT("host/fxr") / version, true); - failed |= FileSystem::CopyDirectory(dstDotnet / TEXT("shared/Microsoft.NETCore.App") / version, srcDotnet / TEXT("shared/Microsoft.NETCore.App") / version, true); + if (usAOT) + { + failed |= EditorUtilities::CopyDirectoryIfNewer(dstDotnet, srcDotnet / TEXT("shared/Microsoft.NETCore.App") / version, true); + } + else + { + failed |= EditorUtilities::CopyDirectoryIfNewer(dstDotnet / TEXT("host/fxr") / version, srcDotnet / TEXT("host/fxr") / version, true); + failed |= EditorUtilities::CopyDirectoryIfNewer(dstDotnet / TEXT("shared/Microsoft.NETCore.App") / version, srcDotnet / TEXT("shared/Microsoft.NETCore.App") / version, true); + } if (failed) { data.Error(TEXT("Failed to copy .Net runtime data files.")); @@ -166,16 +165,41 @@ bool DeployDataStep::Perform(CookingData& data) data.Error(TEXT("Failed to get .Net SDK location for a current platform.")); return true; } + FileSystem::NormalizePath(srcDotnet); LOG(Info, "Using .Net Runtime {} at {}", TEXT("Host"), srcDotnet); // Deploy runtime files - const String packFolder = srcDotnet / TEXT("../../../"); + const Char* corlibPrivateName = TEXT("System.Private.CoreLib.dll"); + const bool srcDotnetFromEngine = srcDotnet.Contains(TEXT("Source/Platforms")); + String packFolder = srcDotnet / TEXT("../../../"); + String dstDotnetLibs = dstDotnet, srcDotnetLibs = srcDotnet; + StringUtils::PathRemoveRelativeParts(packFolder); + if (usAOT) + { + // AOT runtime files inside Engine Platform folder + packFolder /= TEXT("Dotnet"); + dstDotnetLibs /= TEXT("lib/net7.0"); + srcDotnetLibs = packFolder / TEXT("lib/net7.0"); + } + else if (srcDotnetFromEngine) + { + // Runtime files inside Engine Platform folder + dstDotnetLibs /= TEXT("lib/net7.0"); + srcDotnetLibs /= TEXT("lib/net7.0"); + } + else + { + // Runtime files inside Dotnet SDK folder + dstDotnetLibs /= TEXT("shared/Microsoft.NETCore.App"); + srcDotnetLibs /= TEXT("../lib/net7.0"); + } FileSystem::CopyFile(dstDotnet / TEXT("LICENSE.TXT"), packFolder / TEXT("LICENSE.txt")); FileSystem::CopyFile(dstDotnet / TEXT("LICENSE.TXT"), packFolder / TEXT("LICENSE.TXT")); FileSystem::CopyFile(dstDotnet / TEXT("THIRD-PARTY-NOTICES.TXT"), packFolder / TEXT("ThirdPartyNotices.txt")); FileSystem::CopyFile(dstDotnet / TEXT("THIRD-PARTY-NOTICES.TXT"), packFolder / TEXT("THIRD-PARTY-NOTICES.TXT")); - failed |= FileSystem::CopyDirectory(dstDotnet / TEXT("shared/Microsoft.NETCore.App"), srcDotnet / TEXT("../lib/net7.0"), true); - failed |= FileSystem::CopyFile(dstDotnet / TEXT("shared/Microsoft.NETCore.App") / TEXT("System.Private.CoreLib.dll"), srcDotnet / TEXT("System.Private.CoreLib.dll")); + failed |= EditorUtilities::CopyDirectoryIfNewer(dstDotnetLibs, srcDotnetLibs, true); + if (FileSystem::FileExists(srcDotnet / corlibPrivateName)) + failed |= EditorUtilities::CopyFileIfNewer(dstDotnetLibs / corlibPrivateName, srcDotnet / corlibPrivateName); switch (data.Platform) { case BuildPlatform::AndroidARM64: @@ -202,6 +226,23 @@ bool DeployDataStep::Perform(CookingData& data) } } } +#else + if (!FileSystem::DirectoryExists(dstMono)) + { + // Deploy Mono files (from platform data folder) + const String srcMono = depsRoot / TEXT("Mono"); + if (!FileSystem::DirectoryExists(srcMono)) + { + data.Error(TEXT("Missing Mono runtime data files.")); + return true; + } + if (FileSystem::CopyDirectory(dstMono, srcMono, true)) + { + data.Error(TEXT("Failed to copy Mono runtime data files.")); + return true; + } + } +#endif // Deploy engine data for the target platform if (data.Tools->OnDeployBinaries(data)) diff --git a/Source/Editor/Cooker/Steps/PrecompileAssembliesStep.cpp b/Source/Editor/Cooker/Steps/PrecompileAssembliesStep.cpp index 54bd4111d..549da870a 100644 --- a/Source/Editor/Cooker/Steps/PrecompileAssembliesStep.cpp +++ b/Source/Editor/Cooker/Steps/PrecompileAssembliesStep.cpp @@ -1,25 +1,82 @@ // Copyright (c) 2012-2023 Wojciech Figat. All rights reserved. #include "PrecompileAssembliesStep.h" -#include "Editor/Scripting/ScriptsBuilder.h" #include "Engine/Platform/FileSystem.h" +#include "Engine/Core/Config/BuildSettings.h" +#include "Engine/Engine/Globals.h" +#include "Editor/Scripting/ScriptsBuilder.h" #include "Editor/Cooker/PlatformTools.h" +#include "Editor/Utilities/EditorUtilities.h" + +void PrecompileAssembliesStep::OnBuildStarted(CookingData& data) +{ + const DotNetAOTModes aotMode = data.Tools->UseAOT(); + if (aotMode == DotNetAOTModes::None) + return; + + // Redirect C# assemblies to intermediate cooking directory (processed by ILC) + data.ManagedCodeOutputPath = data.CacheDirectory / TEXT("AOTAssemblies"); +} bool PrecompileAssembliesStep::Perform(CookingData& data) { - // Skip for some platforms - if (!data.Tools->UseAOT()) + const DotNetAOTModes aotMode = data.Tools->UseAOT(); + if (aotMode == DotNetAOTModes::None) + return false; + const auto& buildSettings = *BuildSettings::Get(); + if (buildSettings.SkipDotnetPackaging && data.Tools->UseSystemDotnet()) return false; LOG(Info, "Using AOT..."); - - // Useful references about AOT: - // http://www.mono-project.com/docs/advanced/runtime/docs/aot/ - // http://www.mono-project.com/docs/advanced/aot/ - const String infoMsg = TEXT("Running AOT"); data.StepProgress(infoMsg, 0); + // Override Newtonsoft.Json with AOT-version (one that doesn't use System.Reflection.Emit) + EditorUtilities::CopyFileIfNewer(data.ManagedCodeOutputPath / TEXT("Newtonsoft.Json.dll"), Globals::StartupFolder / TEXT("Source/Platforms/DotNet/AOT/Newtonsoft.Json.dll")); + FileSystem::DeleteFile(data.ManagedCodeOutputPath / TEXT("Newtonsoft.Json.xml")); + FileSystem::DeleteFile(data.ManagedCodeOutputPath / TEXT("Newtonsoft.Json.pdb")); + + // Run AOT by Flax.Build + const Char *platform, *architecture, *configuration = ::ToString(data.Configuration); + data.GetBuildPlatformName(platform, architecture); + const String logFile = data.CacheDirectory / TEXT("AotLog.txt"); + const Char* aotModeName = TEXT(""); + switch (aotMode) + { + case DotNetAOTModes::ILC: + aotModeName = TEXT("ILC"); + break; + case DotNetAOTModes::MonoAOTDynamic: + aotModeName = TEXT("MonoAOTDynamic"); + break; + case DotNetAOTModes::MonoAOTStatic: + aotModeName = TEXT("MonoAOTStatic"); + break; + } + auto args = String::Format( + TEXT("-log -logfile=\"{}\" -runDotNetAOT -mutex -platform={} -arch={} -configuration={} -aotMode={} -binaries=\"{}\" -intermediate=\"{}\""), + logFile, platform, architecture, configuration, aotModeName, data.DataOutputPath, data.ManagedCodeOutputPath); + for (const String& define : data.CustomDefines) + { + args += TEXT(" -D"); + args += define; + } + if (ScriptsBuilder::RunBuildTool(args)) + { + data.Error(TEXT("Failed to precompile game scripts.")); + return true; + } + + return false; + + // Useful references about AOT: + // https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/README.md + // https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/nativeaot.md + // https://github.com/dotnet/samples/tree/main/core/nativeaot/NativeLibrary + // http://www.mono-project.com/docs/advanced/runtime/docs/aot/ + // http://www.mono-project.com/docs/advanced/aot/ + // Setup + // TODO: remove old AotConfig, OnConfigureAOT, OnPerformAOT and OnPostProcessAOT PlatformTools::AotConfig config(data); data.Tools->OnConfigureAOT(data, config); diff --git a/Source/Editor/Cooker/Steps/PrecompileAssembliesStep.h b/Source/Editor/Cooker/Steps/PrecompileAssembliesStep.h index d7c89e7ed..dcfe2fc5d 100644 --- a/Source/Editor/Cooker/Steps/PrecompileAssembliesStep.h +++ b/Source/Editor/Cooker/Steps/PrecompileAssembliesStep.h @@ -5,8 +5,7 @@ #include "Editor/Cooker/GameCooker.h" /// -/// Optional step used only on selected platform that precompiles C# script assemblies. -/// Uses Mono Ahead of Time Compilation (AOT) feature. +/// Optional step used only on selected platform that precompiles C# script assemblies. Uses Ahead of Time Compilation (AOT) feature. /// /// class PrecompileAssembliesStep : public GameCooker::BuildStep @@ -14,5 +13,6 @@ class PrecompileAssembliesStep : public GameCooker::BuildStep public: // [BuildStep] + void OnBuildStarted(CookingData& data) override; bool Perform(CookingData& data) override; }; diff --git a/Source/Editor/Utilities/EditorUtilities.cpp b/Source/Editor/Utilities/EditorUtilities.cpp index ae59e2c32..91c6899aa 100644 --- a/Source/Editor/Utilities/EditorUtilities.cpp +++ b/Source/Editor/Utilities/EditorUtilities.cpp @@ -820,3 +820,49 @@ bool EditorUtilities::ReplaceInFile(const StringView& file, const StringView& fi text.Replace(findWhat.Get(), findWhat.Length(), replaceWith.Get(), replaceWith.Length()); return File::WriteAllText(file, text, Encoding::ANSI); } + +bool EditorUtilities::CopyFileIfNewer(const StringView& dst, const StringView& src) +{ + if (FileSystem::FileExists(dst) && + FileSystem::GetFileLastEditTime(src) <= FileSystem::GetFileLastEditTime(dst) && + FileSystem::GetFileSize(dst) == FileSystem::GetFileSize(src)) + return false; + return FileSystem::CopyFile(dst, src); +} + +bool EditorUtilities::CopyDirectoryIfNewer(const StringView& dst, const StringView& src, bool withSubDirectories) +{ + if (FileSystem::DirectoryExists(dst)) + { + // Copy all files + Array cache(32); + if (FileSystem::DirectoryGetFiles(cache, *src, TEXT("*"), DirectorySearchOption::TopDirectoryOnly)) + return true; + for (int32 i = 0; i < cache.Count(); i++) + { + String dstFile = String(dst) / StringUtils::GetFileName(cache[i]); + if (CopyFileIfNewer(*dstFile, *cache[i])) + return true; + } + + // Copy all subdirectories (if need to) + if (withSubDirectories) + { + cache.Clear(); + if (FileSystem::GetChildDirectories(cache, src)) + return true; + for (int32 i = 0; i < cache.Count(); i++) + { + String dstDir = String(dst) / StringUtils::GetFileName(cache[i]); + if (CopyDirectoryIfNewer(dstDir, cache[i], true)) + return true; + } + } + + return false; + } + else + { + return FileSystem::CopyDirectory(dst, src, withSubDirectories); + } +} diff --git a/Source/Editor/Utilities/EditorUtilities.h b/Source/Editor/Utilities/EditorUtilities.h index 37fb28a2a..cbb344266 100644 --- a/Source/Editor/Utilities/EditorUtilities.h +++ b/Source/Editor/Utilities/EditorUtilities.h @@ -82,4 +82,7 @@ public: /// The value to replace to. /// True if failed, otherwise false. static bool ReplaceInFile(const StringView& file, const StringView& findWhat, const StringView& replaceWith); + + static bool CopyFileIfNewer(const StringView& dst, const StringView& src); + static bool CopyDirectoryIfNewer(const StringView& dst, const StringView& src, bool withSubDirectories); }; diff --git a/Source/Engine/Scripting/Runtime/DotNet.cpp b/Source/Engine/Scripting/Runtime/DotNet.cpp index 420f804d8..f19fdf4d7 100644 --- a/Source/Engine/Scripting/Runtime/DotNet.cpp +++ b/Source/Engine/Scripting/Runtime/DotNet.cpp @@ -199,7 +199,7 @@ void RegisterNativeLibrary(const char* moduleName, const char* modulePath) CallStaticMethod(RegisterNativeLibraryPtr, moduleName, modulePath); } -bool InitHostfxr(const String& configPath, const String& libraryPath); +bool InitHostfxr(); void ShutdownHostfxr(); MAssembly* GetAssembly(void* assemblyHandle); @@ -263,15 +263,9 @@ void MCore::UnloadDomain(const StringAnsi& domainName) bool MCore::LoadEngine() { PROFILE_CPU(); - const ::String csharpLibraryPath = Globals::BinariesFolder / TEXT("FlaxEngine.CSharp.dll"); - const ::String csharpRuntimeConfigPath = Globals::BinariesFolder / TEXT("FlaxEngine.CSharp.runtimeconfig.json"); - if (!FileSystem::FileExists(csharpLibraryPath)) - LOG(Fatal, "Failed to initialize managed runtime, FlaxEngine.CSharp.dll is missing."); - if (!FileSystem::FileExists(csharpRuntimeConfigPath)) - LOG(Fatal, "Failed to initialize managed runtime, FlaxEngine.CSharp.runtimeconfig.json is missing."); // Initialize hostfxr - if (InitHostfxr(csharpRuntimeConfigPath, csharpLibraryPath)) + if (InitHostfxr()) return true; // Prepare managed side @@ -1484,14 +1478,20 @@ hostfxr_set_error_writer_fn hostfxr_set_error_writer; hostfxr_get_dotnet_environment_info_result_fn hostfxr_get_dotnet_environment_info_result; hostfxr_run_app_fn hostfxr_run_app; -bool InitHostfxr(const String& configPath, const String& libraryPath) +bool InitHostfxr() { - const FLAX_CORECLR_STRING& library_path = FLAX_CORECLR_STRING(libraryPath); + const ::String csharpLibraryPath = Globals::BinariesFolder / TEXT("FlaxEngine.CSharp.dll"); + const ::String csharpRuntimeConfigPath = Globals::BinariesFolder / TEXT("FlaxEngine.CSharp.runtimeconfig.json"); + if (!FileSystem::FileExists(csharpLibraryPath)) + LOG(Fatal, "Failed to initialize managed runtime, missing file: {0}", csharpLibraryPath); + if (!FileSystem::FileExists(csharpRuntimeConfigPath)) + LOG(Fatal, "Failed to initialize managed runtime, missing file: {0}", csharpRuntimeConfigPath); + const FLAX_CORECLR_STRING& libraryPath = FLAX_CORECLR_STRING(csharpLibraryPath); // Get path to hostfxr library get_hostfxr_parameters get_hostfxr_params; get_hostfxr_params.size = sizeof(hostfxr_initialize_parameters); - get_hostfxr_params.assembly_path = library_path.Get(); + get_hostfxr_params.assembly_path = libraryPath.Get(); FLAX_CORECLR_STRING dotnetRoot; // TODO: implement proper lookup for dotnet installation folder and handle standalone build of FlaxGame #if PLATFORM_MAC @@ -1552,10 +1552,10 @@ bool InitHostfxr(const String& configPath, const String& libraryPath) } // Initialize hosting component - const char_t* argv[1] = { library_path.Get() }; + const char_t* argv[1] = { libraryPath.Get() }; hostfxr_initialize_parameters init_params; init_params.size = sizeof(hostfxr_initialize_parameters); - init_params.host_path = library_path.Get(); + init_params.host_path = libraryPath.Get(); path = String(StringUtils::GetDirectoryName(path)) / TEXT("/../../../"); StringUtils::PathRemoveRelativeParts(path); dotnetRoot = FLAX_CORECLR_STRING(path); @@ -1708,6 +1708,10 @@ static MonoAssembly* OnMonoAssemblyLoad(const char* aname) if (!FileSystem::FileExists(path)) { path = Globals::ProjectFolder / String(TEXT("/Dotnet/shared/Microsoft.NETCore.App/")) / fileName; + if (!FileSystem::FileExists(path)) + { + path = Globals::ProjectFolder / String(TEXT("/Dotnet/")) / fileName; + } } // Load assembly @@ -1732,7 +1736,7 @@ static MonoAssembly* OnMonoAssemblyPreloadHook(MonoAssemblyName* aname, char** a return OnMonoAssemblyLoad(mono_assembly_name_get_name(aname)); } -bool InitHostfxr(const String& configPath, const String& libraryPath) +bool InitHostfxr() { #if DOTNET_HOST_MONO_DEBUG // Enable detailed Mono logging diff --git a/Source/Platforms/DotNet/AOT/Newtonsoft.Json.dll b/Source/Platforms/DotNet/AOT/Newtonsoft.Json.dll new file mode 100644 index 000000000..edeb7993a --- /dev/null +++ b/Source/Platforms/DotNet/AOT/Newtonsoft.Json.dll @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:389fcd3f232ffec36788771c2b8ad165ab770a193d796c12c5604bf84dc62999 +size 681472 diff --git a/Source/Platforms/UWP/Binaries/Newtonsoft.Json.dll b/Source/Platforms/UWP/Binaries/Newtonsoft.Json.dll deleted file mode 100644 index 1b8e69d5f..000000000 --- a/Source/Platforms/UWP/Binaries/Newtonsoft.Json.dll +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:32df88b715d10a5562a6a75ba061c3b1c3f32f88ca74c995b82f103cfb9a9000 -size 635392 diff --git a/Source/ThirdParty/nethost/nethost.Build.cs b/Source/ThirdParty/nethost/nethost.Build.cs index a8b9d4534..6e461def3 100644 --- a/Source/ThirdParty/nethost/nethost.Build.cs +++ b/Source/ThirdParty/nethost/nethost.Build.cs @@ -49,8 +49,17 @@ public class nethost : ThirdPartyModule case TargetPlatform.XboxOne: case TargetPlatform.XboxScarlett: case TargetPlatform.UWP: - options.OutputFiles.Add(Path.Combine(hostRuntime.Path, "nethost.lib")); - options.DependencyFiles.Add(Path.Combine(hostRuntime.Path, "nethost.dll")); + if (hostRuntime.Type == DotNetSdk.HostType.CoreCRL) + { + options.OutputFiles.Add(Path.Combine(hostRuntime.Path, "nethost.lib")); + options.DependencyFiles.Add(Path.Combine(hostRuntime.Path, "nethost.dll")); + } + else + { + options.PublicDefinitions.Add("USE_MONO_DYNAMIC_LIB"); + options.OutputFiles.Add(Path.Combine(hostRuntime.Path, "coreclr.import.lib")); + options.DependencyFiles.Add(Path.Combine(hostRuntime.Path, "coreclr.dll")); + } break; case TargetPlatform.Linux: options.OutputFiles.Add(Path.Combine(hostRuntime.Path, "libnethost.a")); diff --git a/Source/Tools/Flax.Build/Build/DotNet/DotNetAOT.cs b/Source/Tools/Flax.Build/Build/DotNet/DotNetAOT.cs new file mode 100644 index 000000000..bad71bfee --- /dev/null +++ b/Source/Tools/Flax.Build/Build/DotNet/DotNetAOT.cs @@ -0,0 +1,389 @@ +// Copyright (c) 2012-2023 Wojciech Figat. All rights reserved. + +using Mono.Cecil; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace Flax.Build +{ + /// + /// .NET Ahead of Time Compilation (AOT) modes. + /// + public enum DotNetAOTModes + { + /// + /// AOT is not used. + /// + None, + + /// + /// Use .NET Native IL Compiler (shorten as ILC) to convert all C# assemblies in native platform executable binary. + /// + ILC, + + /// + /// Use Mono AOT to cross-compile all used C# assemblies into native platform shared libraries. + /// + MonoAOTDynamic, + + /// + /// Use Mono AOT to cross-compile all used C# assemblies into native platform static libraries which can be linked into a single shared library. + /// + MonoAOTStatic, + } + + partial class Configuration + { + /// + /// AOT mode to use by -runDotNetAOT command. + /// + [CommandLine("aotMode", "")] + public static DotNetAOTModes AOTMode; + + /// + /// Executes AOT process as a part of the game cooking (called by PrecompileAssembliesStep in Editor). + /// + [CommandLine("runDotNetAOT", "")] + public static void RunDotNetAOT() + { + Log.Info("Running .NET AOT in mode " + AOTMode); + DotNetAOT.Run(); + } + } + + /// + /// The DotNet Ahead of Time Compilation (AOT) feature. + /// + public static class DotNetAOT + { + /// + /// Executes AOT process as a part of the game cooking (called by PrecompileAssembliesStep in Editor). + /// + public static void Run() + { + var platform = Configuration.BuildPlatforms[0]; + var arch = Configuration.BuildArchitectures[0]; + var configuration = Configuration.BuildConfigurations[0]; + if (!DotNetSdk.Instance.GetHostRuntime(platform, arch, out var hostRuntime)) + throw new Exception("Missing host runtime"); + var buildPlatform = Platform.GetPlatform(platform); + var buildToolchain = buildPlatform.GetToolchain(arch); + var dotnetAotDebug = Configuration.CustomDefines.Contains("DOTNET_AOT_DEBUG") || Environment.GetEnvironmentVariable("DOTNET_AOT_DEBUG") == "1"; + var aotMode = Configuration.AOTMode; + var outputPath = Configuration.BinariesFolder; // Provided by PrecompileAssembliesStep + var aotAssembliesPath = Configuration.IntermediateFolder; // Provided by PrecompileAssembliesStep + if (!Directory.Exists(outputPath)) + throw new Exception("Missing AOT output folder " + outputPath); + if (!Directory.Exists(aotAssembliesPath)) + throw new Exception("Missing AOT assemblies folder " + aotAssembliesPath); + var dotnetOutputPath = Path.Combine(outputPath, "Dotnet"); + if (!Directory.Exists(dotnetOutputPath)) + Directory.CreateDirectory(dotnetOutputPath); + + // Find input files + var inputFiles = Directory.GetFiles(aotAssembliesPath, "*.dll", SearchOption.TopDirectoryOnly).ToList(); + inputFiles.RemoveAll(x => x.EndsWith(".dll.dll") || Path.GetFileName(x) == "BuilderRulesCache.dll"); + for (int i = 0; i < inputFiles.Count; i++) + inputFiles[i] = Utilities.NormalizePath(inputFiles[i]); + inputFiles.Sort(); + + // Useful references about AOT: + // .NET Native IL Compiler (shorten as ILC) is used to convert IL into native platform binary + // https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/README.md + // https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/nativeaot.md + // https://github.com/dotnet/samples/tree/main/core/nativeaot/NativeLibrary + // http://www.mono-project.com/docs/advanced/runtime/docs/aot/ + // http://www.mono-project.com/docs/advanced/aot/ + + if (aotMode == DotNetAOTModes.ILC) + { + var runtimeIdentifier = DotNetSdk.GetHostRuntimeIdentifier(platform, arch); + var runtimeIdentifierParts = runtimeIdentifier.Split('-'); + var enableReflection = true; + var enableReflectionScan = true; + var enableStackTrace = true; + + var aotOutputPath = Path.Combine(aotAssembliesPath, "Output"); + if (!Directory.Exists(aotOutputPath)) + Directory.CreateDirectory(aotOutputPath); + + // TODO: run dotnet nuget installation to get 'runtime..Microsoft.DotNet.ILCompiler' package + //var ilcRoot = Path.Combine(DotNetSdk.Instance.RootPath, "sdk\\7.0.202\\Sdks\\Microsoft.DotNet.ILCompiler"); + var ilcRoot = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), $".nuget\\packages\\runtime.{runtimeIdentifier}.microsoft.dotnet.ilcompiler\\7.0.4"); + + // Build ILC args list + var ilcArgs = new StringBuilder(); + ilcArgs.AppendLine("--resilient"); // Ignore unresolved types, methods, and assemblies. Defaults to false + ilcArgs.AppendLine("--nativelib"); // Compile as static or shared library + if (configuration != TargetConfiguration.Debug) + ilcArgs.AppendLine("-O"); // Enable optimizations + if (configuration == TargetConfiguration.Release) + ilcArgs.AppendLine("--Ot"); // Enable optimizations, favor code speed + if (configuration != TargetConfiguration.Release) + ilcArgs.AppendLine("-g"); // Emit debugging information + string ilcTargetOs = runtimeIdentifierParts[0]; + if (ilcTargetOs == "win") + ilcTargetOs = "windows"; + ilcArgs.AppendLine("--targetos:" + ilcTargetOs); // Target OS for cross compilation + ilcArgs.AppendLine("--targetarch:" + runtimeIdentifierParts[1]); // Target architecture for cross compilation + var ilcOutputFileName = buildPlatform.SharedLibraryFilePrefix + "AOT" + buildPlatform.SharedLibraryFileExtension; + var ilcOutputPath = Path.Combine(aotOutputPath, ilcOutputFileName); + ilcArgs.AppendLine("-o:" + ilcOutputPath); // Output file path + foreach (var inputFile in inputFiles) + { + ilcArgs.AppendLine(inputFile); // Input file + ilcArgs.AppendLine("--root:" + inputFile); // Fully generate given assembly + } + ilcArgs.AppendLine("--nowarn:\"1701;1702;IL2121;1701;1702\""); // Disable specific warning messages + ilcArgs.AppendLine("--initassembly:System.Private.CoreLib"); // Assembly(ies) with a library initializer + ilcArgs.AppendLine("--initassembly:System.Private.TypeLoader"); + if (enableReflectionScan && enableReflection) + { + ilcArgs.AppendLine("--scanreflection"); // Scan IL for reflection patterns + } + if (enableReflection) + { + ilcArgs.AppendLine("--initassembly:System.Private.Reflection.Execution"); + } + else + { + ilcArgs.AppendLine("--initassembly:System.Private.DisabledReflection"); + ilcArgs.AppendLine("--reflectiondata:none"); + ilcArgs.AppendLine("--feature:System.Collections.Generic.DefaultComparers=false"); + ilcArgs.AppendLine("--feature:System.Reflection.IsReflectionExecutionAvailable=false"); + } + if (enableReflection || enableStackTrace) + ilcArgs.AppendLine("--initassembly:System.Private.StackTraceMetadata"); + if (enableStackTrace) + ilcArgs.AppendLine("--stacktracedata"); // Emit data to support generating stack trace strings at runtime + ilcArgs.AppendLine("--feature:System.Linq.Expressions.CanCompileToIL=false"); + ilcArgs.AppendLine("--feature:System.Linq.Expressions.CanEmitObjectArrayDelegate=false"); + ilcArgs.AppendLine("--feature:System.Linq.Expressions.CanCreateArbitraryDelegates=false"); + // TODO: reference files (-r) + var referenceFiles = new List(); + referenceFiles.AddRange(Directory.GetFiles(Path.Combine(ilcRoot, "framework"), "*.dll")); + referenceFiles.AddRange(Directory.GetFiles(Path.Combine(ilcRoot, "sdk"), "*.dll")); + referenceFiles.Sort(); + foreach (var referenceFile in referenceFiles) + { + ilcArgs.AppendLine("--r:" + referenceFile); // Reference file(s) for compilation + } + ilcArgs.AppendLine("--appcontextswitch:RUNTIME_IDENTIFIER=" + runtimeIdentifier); // System.AppContext switches to set (format: 'Key=Value') + ilcArgs.AppendLine("--appcontextswitch:Microsoft.Extensions.DependencyInjection.VerifyOpenGenericServiceTrimmability=true"); + ilcArgs.AppendLine("--appcontextswitch:System.ComponentModel.TypeConverter.EnableUnsafeBinaryFormatterInDesigntimeLicenseContextSerialization=false"); + ilcArgs.AppendLine("--appcontextswitch:System.Diagnostics.Tracing.EventSource.IsSupported=false"); + ilcArgs.AppendLine("--appcontextswitch:System.Reflection.Metadata.MetadataUpdater.IsSupported=false"); + ilcArgs.AppendLine("--appcontextswitch:System.Resources.ResourceManager.AllowCustomResourceTypes=false"); + ilcArgs.AppendLine("--appcontextswitch:System.Runtime.InteropServices.BuiltInComInterop.IsSupported=false"); + ilcArgs.AppendLine("--appcontextswitch:System.Runtime.InteropServices.EnableConsumingManagedCodeFromNativeHosting=false"); + ilcArgs.AppendLine("--appcontextswitch:System.Runtime.InteropServices.EnableCppCLIHostActivation=false"); + ilcArgs.AppendLine("--appcontextswitch:System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization=false"); + ilcArgs.AppendLine("--appcontextswitch:System.StartupHookProvider.IsSupported=false"); + ilcArgs.AppendLine("--appcontextswitch:System.Threading.Thread.EnableAutoreleasePool=false"); + ilcArgs.AppendLine("--appcontextswitch:System.Text.Encoding.EnableUnsafeUTF7Encoding=false"); + ilcArgs.AppendLine("--feature:Microsoft.Extensions.DependencyInjection.VerifyOpenGenericServiceTrimmability=true"); + ilcArgs.AppendLine("--feature:System.ComponentModel.TypeConverter.EnableUnsafeBinaryFormatterInDesigntimeLicenseContextSerialization=false"); + ilcArgs.AppendLine("--feature:System.Diagnostics.Tracing.EventSource.IsSupported=false"); + ilcArgs.AppendLine("--feature:System.Reflection.Metadata.MetadataUpdater.IsSupported=false"); + ilcArgs.AppendLine("--feature:System.Resources.ResourceManager.AllowCustomResourceTypes=false"); + ilcArgs.AppendLine("--feature:System.Runtime.InteropServices.BuiltInComInterop.IsSupported=false"); + ilcArgs.AppendLine("--feature:System.Runtime.InteropServices.EnableConsumingManagedCodeFromNativeHosting=false"); + ilcArgs.AppendLine("--feature:System.Runtime.InteropServices.EnableCppCLIHostActivation=false"); + ilcArgs.AppendLine("--feature:System.Runtime.Serialization.EnableUnsafeBinaryFormatterSerialization=false"); + ilcArgs.AppendLine("--feature:System.StartupHookProvider.IsSupported=false"); + ilcArgs.AppendLine("--feature:System.Threading.Thread.EnableAutoreleasePool=false"); + ilcArgs.AppendLine("--feature:System.Text.Encoding.EnableUnsafeUTF7Encoding=false"); + ilcArgs.AppendLine("--directpinvoke:System.Globalization.Native"); + ilcArgs.AppendLine("--directpinvoke:System.IO.Compression"); + if (buildPlatform is Platforms.WindowsPlatformBase) + { + // Windows-family + ilcArgs.AppendLine($"--directpinvokelist:{ilcRoot}\\build\\WindowsAPIs.txt"); + } + + // Developer debug options + if (dotnetAotDebug) + { + ilcArgs.AppendLine("--verbose"); // Enable verbose logging + ilcArgs.AppendLine("--metadatalog:" + Path.Combine(aotAssembliesPath, "DotnetAot.metadata.csv")); // Generate a metadata log file + ilcArgs.AppendLine("--exportsfile:" + Path.Combine(aotAssembliesPath, "DotnetAot.exports.txt")); // File to write exported method definitions + ilcArgs.AppendLine("--map:" + Path.Combine(aotAssembliesPath, "DotnetAot.map.xml")); // Generate a map file + ilcArgs.AppendLine("--mstat:" + Path.Combine(aotAssembliesPath, "DotnetAot.mstat")); // Generate an mstat file + ilcArgs.AppendLine("--dgmllog:" + Path.Combine(aotAssembliesPath, "DotnetAot.codegen.dgml.xml")); // Save result of dependency analysis as DGML + ilcArgs.AppendLine("--scandgmllog:" + Path.Combine(aotAssembliesPath, "DotnetAot.scan.dgml.xml")); // Save result of scanner dependency analysis as DGML + } + + // Run ILC + var ilcResponseFile = Path.Combine(aotAssembliesPath, "AOT.ilc.rsp"); + Utilities.WriteFileIfChanged(ilcResponseFile, string.Join(Environment.NewLine, ilcArgs)); + var ilcPath = Path.Combine(ilcRoot, "tools/ilc.exe"); + if (!File.Exists(ilcPath)) + throw new Exception("Missing ILC " + ilcPath); + Utilities.Run(ilcPath, string.Format("@\"{0}\"", ilcResponseFile), null, null, Utilities.RunOptions.AppMustExist | Utilities.RunOptions.ThrowExceptionOnError | Utilities.RunOptions.ConsoleLogOutput); + + // Copy to the destination folder + Utilities.FileCopy(ilcOutputPath, Path.Combine(outputPath, ilcOutputFileName)); + } + else if (aotMode == DotNetAOTModes.MonoAOTDynamic || aotMode == DotNetAOTModes.MonoAOTStatic) + { + var platformToolsRoot = Path.Combine(Globals.EngineRoot, "Source/Platforms", platform.ToString(), "Binaries/Tools"); + if (!Directory.Exists(platformToolsRoot)) + throw new Exception("Missing platform tools " + platformToolsRoot); + var dotnetLibPath = Path.Combine(aotAssembliesPath, "lib/net7.0"); + var monoAssembliesOutputPath = aotMode == DotNetAOTModes.MonoAOTDynamic ? dotnetOutputPath : null; + + // TODO: impl Mono AOT more generic way, not just Windows-only case + + // Build list of assemblies to process (use game assemblies as root to walk over used references from stdlib) + var assembliesPaths = new List(); + using (var assemblyResolver = new MonoCecil.BasicAssemblyResolver()) + { + assemblyResolver.SearchDirectories.Add(aotAssembliesPath); + assemblyResolver.SearchDirectories.Add(dotnetLibPath); + + foreach (var inputFile in inputFiles) + { + try + { + BuildAssembliesList(inputFile, assembliesPaths, assemblyResolver); + } + catch (Exception) + { + Log.Error($"Failed to load assembly '{inputFile}'"); + throw; + } + } + } + + // Setup options + var aotCompilerPath = Path.Combine(platformToolsRoot, "mono-aot-cross.exe"); + var monoAotMode = "full"; + var debugMode = configuration != TargetConfiguration.Release ? "soft-debug" : "nodebug"; + var aotCompilerArgs = $"--aot={monoAotMode},verbose,stats,print-skipped,{debugMode} -O=all"; + if (configuration != TargetConfiguration.Release) + aotCompilerArgs = "--debug " + aotCompilerArgs; + var envVars = new Dictionary(); + envVars["MONO_PATH"] = aotAssembliesPath + ";" + dotnetLibPath; + if (dotnetAotDebug) + { + envVars["MONO_LOG_LEVEL"] = "debug"; + } + + // Run compilation + var compileAssembly = (string assemblyPath) => + { + // Skip if output is already generated and is newer than a source assembly + var outputFilePath = assemblyPath + buildPlatform.SharedLibraryFileExtension; + if (!File.Exists(outputFilePath) || File.GetLastWriteTime(assemblyPath) > File.GetLastWriteTime(outputFilePath)) + { + Log.Error("Run AOT"); + if (dotnetAotDebug) + { + // Increase log readability when spamming log with verbose mode + Log.Info(""); + Log.Info(""); + } + + // Run cross-compiler compiler + Log.Info(" * " + assemblyPath); + Utilities.Run(aotCompilerPath, $"{aotCompilerArgs} \"{assemblyPath}\"", null, platformToolsRoot, Utilities.RunOptions.AppMustExist | Utilities.RunOptions.ThrowExceptionOnError | Utilities.RunOptions.ConsoleLogOutput, envVars); + } + var deployedFilePath = monoAssembliesOutputPath != null ? Path.Combine(monoAssembliesOutputPath, Path.GetFileName(outputFilePath)) : outputFilePath; + if (monoAssembliesOutputPath != null && (!File.Exists(deployedFilePath) || File.GetLastWriteTime(outputFilePath) > File.GetLastWriteTime(deployedFilePath))) + { + Log.Error("Copy files"); + + // Copy to the destination folder + Utilities.FileCopy(assemblyPath, Path.Combine(monoAssembliesOutputPath, Path.GetFileName(assemblyPath))); + Utilities.FileCopy(outputFilePath, deployedFilePath); + if (configuration == TargetConfiguration.Debug || !(buildPlatform is Platforms.WindowsPlatformBase)) + Utilities.FileCopy(outputFilePath + ".pdb", Path.Combine(monoAssembliesOutputPath, Path.GetFileName(outputFilePath + ".pdb"))); + } + }; + if (Configuration.MaxConcurrency > 1 && Configuration.ConcurrencyProcessorScale > 0.0f && !dotnetAotDebug) + { + // Multi-threaded + System.Threading.Tasks.Parallel.ForEach(assembliesPaths, compileAssembly); + } + else + { + // Single-threaded + foreach (var assemblyPath in assembliesPaths) + compileAssembly(assemblyPath); + } + } + else + { + throw new Exception(); + } + + // Deploy license files + Utilities.FileCopy(Path.Combine(aotAssembliesPath, "LICENSE.TXT"), Path.Combine(dotnetOutputPath, "LICENSE.TXT")); + Utilities.FileCopy(Path.Combine(aotAssembliesPath, "THIRD-PARTY-NOTICES.TXT"), Path.Combine(dotnetOutputPath, "THIRD-PARTY-NOTICES.TXT")); + } + + internal static void BuildAssembliesList(string assemblyPath, List outputList, IAssemblyResolver assemblyResolver) + { + // Skip if already processed + if (outputList.Contains(assemblyPath)) + return; + outputList.Add(assemblyPath); + + // Load assembly metadata + using (AssemblyDefinition assembly = AssemblyDefinition.ReadAssembly(assemblyPath, new ReaderParameters { ReadSymbols = false, AssemblyResolver = assemblyResolver })) + { + foreach (ModuleDefinition assemblyModule in assembly.Modules) + { + // Collected referenced assemblies + foreach (AssemblyNameReference assemblyReference in assemblyModule.AssemblyReferences) + { + BuildAssembliesList(assemblyPath, assemblyReference, outputList, assemblyResolver); + } + } + } + + // Move to the end of list + outputList.Remove(assemblyPath); + outputList.Add(assemblyPath); + } + + internal static void BuildAssembliesList(AssemblyDefinition assembly, List outputList, IAssemblyResolver assemblyResolver) + { + // Skip if already processed + var assemblyPath = Utilities.NormalizePath(assembly.MainModule.FileName); + if (outputList.Contains(assemblyPath)) + return; + outputList.Add(assemblyPath); + + foreach (ModuleDefinition assemblyModule in assembly.Modules) + { + // Collected referenced assemblies + foreach (AssemblyNameReference assemblyReference in assemblyModule.AssemblyReferences) + { + BuildAssembliesList(assemblyPath, assemblyReference, outputList, assemblyResolver); + } + } + + // Move to the end of list + outputList.Remove(assemblyPath); + outputList.Add(assemblyPath); + } + + internal static void BuildAssembliesList(string assemblyPath, AssemblyNameReference assemblyReference, List outputList, IAssemblyResolver assemblyResolver) + { + try + { + var reference = assemblyResolver.Resolve(assemblyReference); + BuildAssembliesList(reference, outputList, assemblyResolver); + } + catch (Exception) + { + Log.Error($"Failed to load assembly '{assemblyReference.FullName}' referenced by '{assemblyPath}'"); + throw; + } + } + } +} diff --git a/Source/Tools/Flax.Build/Build/Platform.cs b/Source/Tools/Flax.Build/Build/Platform.cs index ac9a64b00..16f114363 100644 --- a/Source/Tools/Flax.Build/Build/Platform.cs +++ b/Source/Tools/Flax.Build/Build/Platform.cs @@ -166,7 +166,7 @@ namespace Flax.Build public virtual string SharedLibraryFilePrefix => string.Empty; /// - /// Gets the statuc library files prefix. + /// Gets the static library files prefix. /// public virtual string StaticLibraryFilePrefix => string.Empty; diff --git a/Source/Tools/Flax.Build/Deps/Dependencies/NewtonsoftJson.cs b/Source/Tools/Flax.Build/Deps/Dependencies/NewtonsoftJson.cs index 0d8b2ec02..03fe0bda1 100644 --- a/Source/Tools/Flax.Build/Deps/Dependencies/NewtonsoftJson.cs +++ b/Source/Tools/Flax.Build/Deps/Dependencies/NewtonsoftJson.cs @@ -30,6 +30,7 @@ namespace Flax.Deps.Dependencies TargetPlatform.PS5, TargetPlatform.Switch, TargetPlatform.Mac, + TargetPlatform.iOS, }; default: return new TargetPlatform[0]; } @@ -83,13 +84,14 @@ namespace Flax.Deps.Dependencies { case TargetPlatform.UWP: case TargetPlatform.XboxOne: + case TargetPlatform.XboxScarlett: case TargetPlatform.PS4: case TargetPlatform.PS5: - case TargetPlatform.XboxScarlett: case TargetPlatform.Switch: + case TargetPlatform.iOS: { var file = "Newtonsoft.Json.dll"; - Utilities.FileCopy(Path.Combine(binFolder, file), Path.Combine(options.PlatformsFolder, platform.ToString(), "Binaries", file)); + Utilities.FileCopy(Path.Combine(binFolder, file), Path.Combine(options.PlatformsFolder, "DotNet/AOT", file)); break; } } diff --git a/Source/Tools/Flax.Build/Utilities/MonoCecil.cs b/Source/Tools/Flax.Build/Utilities/MonoCecil.cs index 43c1de4ba..00f9fbf9b 100644 --- a/Source/Tools/Flax.Build/Utilities/MonoCecil.cs +++ b/Source/Tools/Flax.Build/Utilities/MonoCecil.cs @@ -1,8 +1,13 @@ // Copyright (c) 2012-2023 Wojciech Figat. All rights reserved. using System; +using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Reflection; using Mono.Cecil; +using CustomAttributeNamedArgument = Mono.Cecil.CustomAttributeNamedArgument; +using ICustomAttributeProvider = Mono.Cecil.ICustomAttributeProvider; namespace Flax.Build { @@ -11,6 +16,62 @@ namespace Flax.Build /// internal static class MonoCecil { + public sealed class BasicAssemblyResolver : IAssemblyResolver + { + private readonly Dictionary _cache = new(); + + public HashSet SearchDirectories = new(); + + public AssemblyDefinition Resolve(AssemblyNameReference name) + { + return Resolve(name, new ReaderParameters()); + } + + public AssemblyDefinition Resolve(AssemblyNameReference name, ReaderParameters parameters) + { + if (_cache.TryGetValue(name.FullName, out var assembly)) + return assembly; + + if (parameters.AssemblyResolver == null) + parameters.AssemblyResolver = this; + foreach (var searchDirectory in SearchDirectories) + { + if (TryLoad(name, parameters, searchDirectory, out assembly)) + return assembly; + } + + throw new AssemblyResolutionException(name); + } + + public void Dispose() + { + foreach (var assembly in _cache.Values) + assembly.Dispose(); + _cache.Clear(); + } + + private bool TryLoad(AssemblyNameReference name, ReaderParameters parameters, string directory, out AssemblyDefinition assembly) + { + assembly = null; + + var file = Path.Combine(directory, name.Name + ".dll"); + if (!File.Exists(file)) + return false; + + try + { + assembly = ModuleDefinition.ReadModule(file, parameters).Assembly; + } + catch (BadImageFormatException) + { + return false; + } + + _cache[name.FullName] = assembly; + return true; + } + } + public static void CompilationError(string message) { Log.Error(message); diff --git a/Source/Tools/Flax.Build/Utilities/Utilities.cs b/Source/Tools/Flax.Build/Utilities/Utilities.cs index 5e336598d..d731dcec5 100644 --- a/Source/Tools/Flax.Build/Utilities/Utilities.cs +++ b/Source/Tools/Flax.Build/Utilities/Utilities.cs @@ -314,21 +314,26 @@ namespace Flax.Build /// ThrowExceptionOnError = 1 << 6, + /// + /// Logs program output to the console, otherwise only when using verbose log. + /// + ConsoleLogOutput = 1 << 7, + /// /// The default options. /// Default = AppMustExist, } - private static void StdOut(object sender, DataReceivedEventArgs e) + private static void StdLogInfo(object sender, DataReceivedEventArgs e) { if (e.Data != null) { - Log.Verbose(e.Data); + Log.Info(e.Data); } } - private static void StdErr(object sender, DataReceivedEventArgs e) + private static void StdLogVerbose(object sender, DataReceivedEventArgs e) { if (e.Data != null) { @@ -400,8 +405,16 @@ namespace Flax.Build { proc.StartInfo.RedirectStandardOutput = true; proc.StartInfo.RedirectStandardError = true; - proc.OutputDataReceived += StdOut; - proc.ErrorDataReceived += StdErr; + if (options.HasFlag(RunOptions.ConsoleLogOutput)) + { + proc.OutputDataReceived += StdLogInfo; + proc.ErrorDataReceived += StdLogInfo; + } + else + { + proc.OutputDataReceived += StdLogVerbose; + proc.ErrorDataReceived += StdLogVerbose; + } } if (envVars != null)