From d032f18b7121d42adb65e4e9e01cd6e207e7be98 Mon Sep 17 00:00:00 2001 From: Wojciech Figat Date: Fri, 14 Jan 2022 14:19:46 +0100 Subject: [PATCH] Add DotNet targets building --- .../Flax.Build/Build/DotNet/Builder.DotNet.cs | 399 ++++++++++++------ .../Tools/Flax.Build/Build/Graph/TaskGraph.cs | 22 + .../Build/NativeCpp/Builder.NativeCpp.cs | 5 - .../Tools/Flax.Build/Deploy/VCEnvironment.cs | 8 +- 4 files changed, 292 insertions(+), 142 deletions(-) diff --git a/Source/Tools/Flax.Build/Build/DotNet/Builder.DotNet.cs b/Source/Tools/Flax.Build/Build/DotNet/Builder.DotNet.cs index f291ba961..646247f61 100644 --- a/Source/Tools/Flax.Build/Build/DotNet/Builder.DotNet.cs +++ b/Source/Tools/Flax.Build/Build/DotNet/Builder.DotNet.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using Flax.Build.Graph; +using Flax.Deploy; namespace Flax.Build { @@ -12,15 +13,268 @@ namespace Flax.Build { private static void BuildTargetDotNet(RulesAssembly rules, TaskGraph graph, Target target, Platform platform, TargetConfiguration configuration) { - throw new NotImplementedException("TODO: building C# targets"); + // Check if use custom project file + if (!string.IsNullOrEmpty(target.CustomExternalProjectFilePath)) + { + // Use msbuild to compile it + var task = graph.Add(); + task.WorkingDirectory = Globals.Root; + task.InfoMessage = "Building " + Path.GetFileName(target.CustomExternalProjectFilePath); + task.Cost = 100; + task.CommandPath = VCEnvironment.MSBuildPath; + task.CommandArguments = string.Format("\"{0}\" /m /t:Build /p:Configuration=\"{1}\" /p:Platform=\"{2}\" {3} /nologo", target.CustomExternalProjectFilePath, configuration.ToString(), "AnyCPU", VCEnvironment.Verbosity); + return; + } + + // Warn if target has no valid modules + if (target.Modules.Count == 0) + Log.Warning(string.Format("Target {0} has no modules to build", target.Name)); + + // Pick a project + var project = Globals.Project; + if (target is ProjectTarget projectTarget) + project = projectTarget.Project; + if (project == null) + throw new Exception($"Cannot build target {target.Name}. The project file is missing (.flaxproj located in the folder above)."); + + // Setup build environment for the target + var targetBuildOptions = GetBuildOptions(target, platform, null, TargetArchitecture.AnyCPU, configuration, project.ProjectFolderPath); + using (new ProfileEventScope("PreBuild")) + { + // Pre build + target.PreBuild(graph, targetBuildOptions); + PreBuild?.Invoke(graph, targetBuildOptions); + + // Ensure that target build directories exist + if (!target.IsPreBuilt && !Directory.Exists(targetBuildOptions.IntermediateFolder)) + Directory.CreateDirectory(targetBuildOptions.IntermediateFolder); + if (!target.IsPreBuilt && !Directory.Exists(targetBuildOptions.OutputFolder)) + Directory.CreateDirectory(targetBuildOptions.OutputFolder); + } + + // Setup building common data container + var buildData = new BuildData + { + Project = project, + Graph = graph, + Rules = rules, + Target = target, + TargetOptions = targetBuildOptions, + Platform = platform, + Architecture = TargetArchitecture.AnyCPU, + Configuration = configuration, + }; + + // Collect all modules + using (new ProfileEventScope("CollectModules")) + { + foreach (var moduleName in target.Modules) + { + var module = rules.GetModule(moduleName); + if (module != null) + { + CollectModules(buildData, module, true); + } + else + { + Log.Warning(string.Format("Missing module {0} (or invalid name specified)", moduleName)); + } + } + } + + // Build all modules from target binary modules but in order of collecting (from independent to more dependant ones) + var sourceFiles = new List(); + using (new ProfileEventScope("BuildModules")) + { + foreach (var module in buildData.ModulesOrderList) + { + if (buildData.BinaryModules.Any(x => x.Contains(module))) + { + var moduleOptions = BuildModule(buildData, module); + + // Get source files + sourceFiles.AddRange(moduleOptions.SourceFiles.Where(x => x.EndsWith(".cs"))); + + // Merge module into target environment + foreach (var e in moduleOptions.OutputFiles) + buildData.TargetOptions.LinkEnv.InputFiles.Add(e); + foreach (var e in moduleOptions.DependencyFiles) + buildData.TargetOptions.DependencyFiles.Add(e); + foreach (var e in moduleOptions.OptionalDependencyFiles) + buildData.TargetOptions.OptionalDependencyFiles.Add(e); + buildData.TargetOptions.Libraries.AddRange(moduleOptions.Libraries); + buildData.TargetOptions.DelayLoadLibraries.AddRange(moduleOptions.DelayLoadLibraries); + buildData.TargetOptions.ScriptingAPI.Add(moduleOptions.ScriptingAPI); + } + } + } + + // Build + var outputTargetFilePath = target.GetOutputFilePath(targetBuildOptions); + var outputPath = Path.GetDirectoryName(outputTargetFilePath); + using (new ProfileEventScope("Build")) + { + // Cleanup source files + sourceFiles.RemoveAll(x => x.EndsWith(BuildFilesPostfix)); + sourceFiles.Sort(); + + // Build assembly + BuildDotNet(graph, buildData, targetBuildOptions, target.OutputName, sourceFiles); + } + + // Deploy files + if (!target.IsPreBuilt) + { + using (new ProfileEventScope("DeployFiles")) + { + foreach (var srcFile in targetBuildOptions.OptionalDependencyFiles.Where(File.Exists).Union(targetBuildOptions.DependencyFiles)) + { + var dstFile = Path.Combine(outputPath, Path.GetFileName(srcFile)); + graph.AddCopyFile(dstFile, srcFile); + } + } + } + + using (new ProfileEventScope("PostBuild")) + { + // Post build + PostBuild?.Invoke(graph, targetBuildOptions); + target.PostBuild(graph, targetBuildOptions); + } + } + + private static void BuildDotNet(TaskGraph graph, BuildData buildData, NativeCpp.BuildOptions buildOptions, string name, List sourceFiles, HashSet fileReferences = null) + { + // Setup build options + var buildPlatform = Platform.BuildTargetPlatform; + var outputPath = Path.GetDirectoryName(buildData.Target.GetOutputFilePath(buildOptions)); + var outputFile = Path.Combine(outputPath, name + ".dll"); + var outputDocFile = Path.Combine(outputPath, name + ".xml"); + string monoRoot, monoPath, cscPath; + switch (buildPlatform) + { + case TargetPlatform.Windows: + { + monoRoot = Path.Combine(Globals.EngineRoot, "Source", "Platforms", "Editor", "Windows", "Mono"); + + // Prefer installed Roslyn C# compiler over Mono one + monoPath = null; + cscPath = Path.Combine(Path.GetDirectoryName(Deploy.VCEnvironment.MSBuildPath), "Roslyn", "csc.exe"); + + if (!File.Exists(cscPath)) + { + // Fallback to Mono binaries + monoPath = Path.Combine(monoRoot, "bin", "mono.exe"); + cscPath = Path.Combine(monoRoot, "lib", "mono", "4.5", "csc.exe"); + } + break; + } + case TargetPlatform.Linux: + monoRoot = Path.Combine(Globals.EngineRoot, "Source", "Platforms", "Editor", "Linux", "Mono"); + monoPath = Path.Combine(monoRoot, "bin", "mono"); + cscPath = Path.Combine(monoRoot, "lib", "mono", "4.5", "csc.exe"); + break; + case TargetPlatform.Mac: + monoRoot = Path.Combine(Globals.EngineRoot, "Source", "Platforms", "Editor", "Mac", "Mono"); + monoPath = Path.Combine(monoRoot, "bin", "mono"); + cscPath = Path.Combine(monoRoot, "lib", "mono", "4.5", "csc.exe"); + break; + default: throw new InvalidPlatformException(buildPlatform); + } + var referenceAssemblies = Path.Combine(monoRoot, "lib", "mono", "4.5-api"); + if (fileReferences == null) + fileReferences = buildOptions.ScriptingAPI.FileReferences; + else + fileReferences.AddRange(buildOptions.ScriptingAPI.FileReferences); + + // Setup C# compiler arguments + var args = new List(); + args.Clear(); + args.Add("/nologo"); + args.Add("/target:library"); + args.Add("/platform:AnyCPU"); + args.Add("/debug+"); + args.Add("/debug:portable"); + args.Add("/errorreport:prompt"); + args.Add("/preferreduilang:en-US"); + args.Add("/highentropyva+"); + args.Add("/deterministic"); + args.Add("/nostdlib+"); + args.Add("/errorendlocation"); + args.Add("/utf8output"); + args.Add("/warn:4"); + args.Add("/unsafe"); + args.Add("/fullpaths"); + args.Add("/langversion:7.3"); + if (buildOptions.ScriptingAPI.IgnoreMissingDocumentationWarnings) + args.Add("-nowarn:1591"); + args.Add(buildData.Configuration == TargetConfiguration.Debug ? "/optimize-" : "/optimize+"); + args.Add(string.Format("/out:\"{0}\"", outputFile)); + args.Add(string.Format("/doc:\"{0}\"", outputDocFile)); + if (buildOptions.ScriptingAPI.Defines.Count != 0) + args.Add("/define:" + string.Join(";", buildOptions.ScriptingAPI.Defines)); + if (buildData.Configuration == TargetConfiguration.Debug) + args.Add("/define:DEBUG"); + args.Add(string.Format("/reference:\"{0}{1}mscorlib.dll\"", referenceAssemblies, Path.DirectorySeparatorChar)); + foreach (var reference in buildOptions.ScriptingAPI.SystemReferences) + args.Add(string.Format("/reference:\"{0}{2}{1}.dll\"", referenceAssemblies, reference, Path.DirectorySeparatorChar)); + foreach (var reference in fileReferences) + args.Add(string.Format("/reference:\"{0}\"", reference)); + foreach (var sourceFile in sourceFiles) + args.Add("\"" + sourceFile + "\""); + + // Generate response file with source files paths and compilation arguments + string responseFile = Path.Combine(buildOptions.IntermediateFolder, name + ".response"); + Utilities.WriteFileIfChanged(responseFile, string.Join(Environment.NewLine, args)); + + // Create C# compilation task + var task = graph.Add(); + task.PrerequisiteFiles.Add(responseFile); + task.PrerequisiteFiles.AddRange(sourceFiles); + task.PrerequisiteFiles.AddRange(fileReferences); + task.ProducedFiles.Add(outputFile); + task.WorkingDirectory = buildData.TargetOptions.WorkingDirectory; + task.InfoMessage = "Compiling " + outputFile; + task.Cost = task.PrerequisiteFiles.Count; + + if (monoPath != null) + { + task.CommandPath = monoPath; + task.CommandArguments = $"\"{cscPath}\" /noconfig @\"{responseFile}\""; + } + else + { + // The "/shared" flag enables the compiler server support: + // https://github.com/dotnet/roslyn/blob/main/docs/compilers/Compiler%20Server.md + + task.CommandPath = cscPath; + task.CommandArguments = $"/noconfig /shared @\"{responseFile}\""; + } + + // Copy referenced assemblies + foreach (var srcFile in buildOptions.ScriptingAPI.FileReferences) + { + var dstFile = Path.Combine(outputPath, Path.GetFileName(srcFile)); + if (dstFile == srcFile || graph.HasCopyTask(dstFile, srcFile)) + continue; + graph.AddCopyFile(dstFile, srcFile); + + var srcPdb = Path.ChangeExtension(srcFile, "pdb"); + if (File.Exists(srcPdb)) + graph.AddCopyFile(Path.ChangeExtension(dstFile, "pdb"), srcPdb); + + var srcXml = Path.ChangeExtension(srcFile, "xml"); + if (File.Exists(srcXml)) + graph.AddCopyFile(Path.ChangeExtension(dstFile, "xml"), srcXml); + } } private static void BuildTargetBindings(TaskGraph graph, BuildData buildData) { - var workspaceRoot = buildData.TargetOptions.WorkingDirectory; - var args = new List(); - var referencesToCopy = new HashSet>(); + var sourceFiles = new List(); + var fileReferences = new HashSet(); var buildOptions = buildData.TargetOptions; + var outputPath = Path.GetDirectoryName(buildData.Target.GetOutputFilePath(buildOptions)); foreach (var binaryModule in buildData.BinaryModules) { if (binaryModule.All(x => !x.BuildCSharp)) @@ -32,7 +286,7 @@ namespace Flax.Build var project = GetModuleProject(binaryModule.First(), buildData); // Get source files - var sourceFiles = new List(); + sourceFiles.Clear(); foreach (var module in binaryModule) sourceFiles.AddRange(buildData.Modules[module].SourceFiles.Where(x => x.EndsWith(".cs"))); sourceFiles.RemoveAll(x => x.EndsWith(BuildFilesPostfix)); @@ -41,45 +295,8 @@ namespace Flax.Build sourceFiles.Add(moduleGen); sourceFiles.Sort(); - // Setup build options - var buildPlatform = Platform.BuildPlatform.Target; - var outputPath = Path.GetDirectoryName(buildData.Target.GetOutputFilePath(buildOptions)); - var outputFile = Path.Combine(outputPath, binaryModuleName + ".CSharp.dll"); - var outputDocFile = Path.Combine(outputPath, binaryModuleName + ".CSharp.xml"); - string monoRoot, monoPath, cscPath; - switch (buildPlatform) - { - case TargetPlatform.Windows: - { - monoRoot = Path.Combine(Globals.EngineRoot, "Source", "Platforms", "Editor", "Windows", "Mono"); - - // Prefer installed Roslyn C# compiler over Mono one - monoPath = null; - cscPath = Path.Combine(Path.GetDirectoryName(Deploy.VCEnvironment.MSBuildPath), "Roslyn", "csc.exe"); - - if (!File.Exists(cscPath)) - { - // Fallback to Mono binaries - monoPath = Path.Combine(monoRoot, "bin", "mono.exe"); - cscPath = Path.Combine(monoRoot, "lib", "mono", "4.5", "csc.exe"); - } - break; - } - case TargetPlatform.Linux: - monoRoot = Path.Combine(Globals.EngineRoot, "Source", "Platforms", "Editor", "Linux", "Mono"); - monoPath = Path.Combine(monoRoot, "bin", "mono"); - cscPath = Path.Combine(monoRoot, "lib", "mono", "4.5", "csc.exe"); - break; - case TargetPlatform.Mac: - monoRoot = Path.Combine(Globals.EngineRoot, "Source", "Platforms", "Editor", "Mac", "Mono"); - monoPath = Path.Combine(monoRoot, "bin", "mono"); - cscPath = Path.Combine(monoRoot, "lib", "mono", "4.5", "csc.exe"); - break; - default: throw new InvalidPlatformException(buildPlatform); - } - var referenceAssemblies = Path.Combine(monoRoot, "lib", "mono", "4.5-api"); - var references = new HashSet(buildOptions.ScriptingAPI.FileReferences); - + // Get references + fileReferences.Clear(); foreach (var module in binaryModule) { if (!buildData.Modules.TryGetValue(module, out var moduleBuildOptions)) @@ -101,7 +318,7 @@ namespace Flax.Build continue; // Reference module output binary - references.Add(Path.Combine(outputPath, dependencyModule.BinaryModuleName + ".CSharp.dll")); + fileReferences.Add(Path.Combine(outputPath, dependencyModule.BinaryModuleName + ".CSharp.dll")); } foreach (var e in buildData.ReferenceBuilds) { @@ -110,7 +327,7 @@ namespace Flax.Build if (q.Name == dependencyModule.BinaryModuleName && !string.IsNullOrEmpty(q.ManagedPath)) { // Reference binary module build build for referenced target - references.Add(q.ManagedPath); + fileReferences.Add(q.ManagedPath); break; } } @@ -119,96 +336,10 @@ namespace Flax.Build } } - // Setup C# compiler arguments - args.Clear(); - args.Add("/nologo"); - args.Add("/target:library"); - args.Add("/platform:AnyCPU"); - args.Add("/debug+"); - args.Add("/debug:portable"); - args.Add("/errorreport:prompt"); - args.Add("/preferreduilang:en-US"); - args.Add("/highentropyva+"); - args.Add("/deterministic"); - args.Add("/nostdlib+"); - args.Add("/errorendlocation"); - args.Add("/utf8output"); - args.Add("/warn:4"); - args.Add("/unsafe"); - args.Add("/fullpaths"); - args.Add("/langversion:7.3"); - if (buildOptions.ScriptingAPI.IgnoreMissingDocumentationWarnings) - args.Add("-nowarn:1591"); - args.Add(buildData.Configuration == TargetConfiguration.Debug ? "/optimize-" : "/optimize+"); - args.Add(string.Format("/out:\"{0}\"", outputFile)); - args.Add(string.Format("/doc:\"{0}\"", outputDocFile)); - if (buildOptions.ScriptingAPI.Defines.Count != 0) - args.Add("/define:" + string.Join(";", buildOptions.ScriptingAPI.Defines)); - if (buildData.Configuration == TargetConfiguration.Debug) - args.Add("/define:DEBUG"); - args.Add(string.Format("/reference:\"{0}{1}mscorlib.dll\"", referenceAssemblies, Path.DirectorySeparatorChar)); - foreach (var reference in buildOptions.ScriptingAPI.SystemReferences) - args.Add(string.Format("/reference:\"{0}{2}{1}.dll\"", referenceAssemblies, reference, Path.DirectorySeparatorChar)); - foreach (var reference in references) - args.Add(string.Format("/reference:\"{0}\"", reference)); - foreach (var sourceFile in sourceFiles) - args.Add("\"" + sourceFile + "\""); - - // Generate response file with source files paths and compilation arguments - string responseFile = Path.Combine(buildOptions.IntermediateFolder, binaryModuleName + ".CSharp.response"); - Utilities.WriteFileIfChanged(responseFile, string.Join(Environment.NewLine, args)); - - // Create C# compilation task - var task = graph.Add(); - task.PrerequisiteFiles.Add(responseFile); - task.PrerequisiteFiles.AddRange(sourceFiles); - task.PrerequisiteFiles.AddRange(references); - task.ProducedFiles.Add(outputFile); - task.WorkingDirectory = workspaceRoot; - task.InfoMessage = "Compiling " + outputFile; - task.Cost = task.PrerequisiteFiles.Count; - - if (monoPath != null) - { - task.CommandPath = monoPath; - task.CommandArguments = $"\"{cscPath}\" /noconfig @\"{responseFile}\""; - } - else - { - // The "/shared" flag enables the compiler server support: - // https://github.com/dotnet/roslyn/blob/main/docs/compilers/Compiler%20Server.md - - task.CommandPath = cscPath; - task.CommandArguments = $"/noconfig /shared @\"{responseFile}\""; - } - - // Copy referenced assemblies - foreach (var reference in buildOptions.ScriptingAPI.FileReferences) - { - var dstFile = Path.Combine(outputPath, Path.GetFileName(reference)); - if (dstFile == reference) - continue; - - referencesToCopy.Add(new KeyValuePair(dstFile, reference)); - } + // Build assembly + BuildDotNet(graph, buildData, buildOptions, binaryModuleName + ".CSharp", sourceFiles, fileReferences); } } - - // Copy files (using hash set to prevent copying the same file twice when building multiple scripting modules using the same files) - foreach (var e in referencesToCopy) - { - var dst = e.Key; - var src = e.Value; - graph.AddCopyFile(dst, src); - - var srcPdb = Path.ChangeExtension(src, "pdb"); - if (File.Exists(srcPdb)) - graph.AddCopyFile(Path.ChangeExtension(dst, "pdb"), srcPdb); - - var srcXml = Path.ChangeExtension(src, "xml"); - if (File.Exists(srcXml)) - graph.AddCopyFile(Path.ChangeExtension(dst, "xml"), srcXml); - } } } } diff --git a/Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs b/Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs index ffa7410da..9d307cebd 100644 --- a/Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs +++ b/Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs @@ -94,6 +94,28 @@ namespace Flax.Build.Graph return task; } + /// + /// Checks if that copy task is already added in a graph. Use it for logic that might copy the same file multiple times. + /// + /// The destination file path. + /// The source file path. + /// True if has copy task already scheduled in a task graph, otherwise false.. + public bool HasCopyTask(string dstFile, string srcFile) + { + for (int i = Tasks.Count - 1; i >= 0; i--) + { + var t = Tasks[i]; + if (t.Cost == 1 && + t.PrerequisiteFiles.Count == 1 && t.PrerequisiteFiles[0] == srcFile && + t.ProducedFiles.Count == 1 && t.ProducedFiles[0] == dstFile) + { + // Already scheduled for copy + return true; + } + } + return false; + } + /// /// Builds the cache for the task graph. Cached the map for files to provide O(1) lookup for producing task. /// diff --git a/Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs b/Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs index fc45d6a0b..89aadb3e1 100644 --- a/Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs +++ b/Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs @@ -616,9 +616,7 @@ namespace Flax.Build // Warn if target has no valid modules if (target.Modules.Count == 0) - { Log.Warning(string.Format("Target {0} has no modules to build", target.Name)); - } // Pick a project var project = Globals.Project; @@ -951,10 +949,7 @@ namespace Flax.Build // Warn if target has no valid modules if (target.Modules.Count == 0) - { Log.Warning(string.Format("Target {0} has no modules to build", target.Name)); - } - // Pick a project var project = Globals.Project; if (target is ProjectTarget projectTarget) diff --git a/Source/Tools/Flax.Build/Deploy/VCEnvironment.cs b/Source/Tools/Flax.Build/Deploy/VCEnvironment.cs index 6f1a764c8..17bdbf190 100644 --- a/Source/Tools/Flax.Build/Deploy/VCEnvironment.cs +++ b/Source/Tools/Flax.Build/Deploy/VCEnvironment.cs @@ -184,6 +184,8 @@ namespace Flax.Deploy return false; } + internal static string Verbosity => Configuration.Verbose ? " " : "/verbosity:minimal "; + /// /// Runs msbuild.exe with the specified arguments. /// @@ -203,7 +205,7 @@ namespace Flax.Deploy throw new Exception(string.Format("Project {0} does not exist!", project)); } - string cmdLine = string.Format("\"{0}\" /m /t:Build /p:Configuration=\"{1}\" /p:Platform=\"{2}\" /verbosity:minimal /nologo", project, buildConfig, buildPlatform); + string cmdLine = string.Format("\"{0}\" /m /t:Build /p:Configuration=\"{1}\" /p:Platform=\"{2}\" {3} /nologo", project, buildConfig, buildPlatform, Verbosity); int result = Utilities.Run(msBuild, cmdLine); if (result != 0) { @@ -230,7 +232,7 @@ namespace Flax.Deploy throw new Exception(string.Format("Unable to build solution {0}. Solution file not found.", solutionFile)); } - string cmdLine = string.Format("\"{0}\" /m /t:Build /p:Configuration=\"{1}\" /p:Platform=\"{2}\" /verbosity:minimal /nologo", solutionFile, buildConfig, buildPlatform); + string cmdLine = string.Format("\"{0}\" /m /t:Build /p:Configuration=\"{1}\" /p:Platform=\"{2}\" {3} /nologo", solutionFile, buildConfig, buildPlatform, Verbosity); int result = Utilities.Run(msBuild, cmdLine); if (result != 0) { @@ -255,7 +257,7 @@ namespace Flax.Deploy throw new Exception(string.Format("Unable to clean solution {0}. Solution file not found.", solutionFile)); } - string cmdLine = string.Format("\"{0}\" /t:Clean /verbosity:minimal /nologo", solutionFile); + string cmdLine = string.Format("\"{0}\" /t:Clean {1} /nologo", solutionFile, Verbosity); Utilities.Run(msBuild, cmdLine); }