GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Merge branch 'flax_build_perf_improvements' of https://github.com/GoaLitiuM/FlaxEngine into GoaLitiuM-flax_build_perf_improvements

Browse Source
This commit is contained in:
Wojtek Figat
2023-03-15 17:57:16 +01:00
parent 4f10f852c7 e11587e72e
commit 33ac9c8811
9 changed files with 177 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Source/Tools/Flax.Build/Build/Builder.Rules.cs
View File

@@ -270,8 +270,9 @@ namespace Flax.Build
private static void FindRules(string directory, List<string> result)
{
// Optional way:
//result.AddRange(Directory.GetFiles(directory, '*' + BuildFilesPostfix, SearchOption.AllDirectories));
result.AddRange(Directory.GetFiles(directory, '*' + BuildFilesPostfix, SearchOption.AllDirectories));
/*
var files = Directory.GetFiles(directory);
for (int i = 0; i < files.Length; i++)
{
@@ -286,6 +287,7 @@ namespace Flax.Build
{
FindRules(directories[i], result);
}
*/
}
}
}

41
Source/Tools/Flax.Build/Build/FileCache.cs Normal file
View File

@@ -0,0 +1,41 @@
using System;
using System.Collections.Generic;
using System.IO;
namespace Flax.Build
{
/// <summary>
/// Cache filesystem related queries like File.Exists and File.GetLastWriteTime.
/// </summary>
public static class FileCache
{
private static Dictionary<string, FileInfo> fileInfoCache = new Dictionary<string, FileInfo>();
public static void FileRemoveFromCache(string path)
{
//fileInfoCache[path].Refresh();
fileInfoCache.Remove(path);
}
public static bool Exists(string path)
{
if (fileInfoCache.TryGetValue(path, out var fileInfo))
return fileInfo.Exists;
fileInfo = new FileInfo(path);
fileInfoCache.Add(path, fileInfo);
return fileInfo.Exists;
}
public static DateTime GetLastWriteTime(string path)
{
if (fileInfoCache.TryGetValue(path, out var fileInfo))
return fileInfo.LastWriteTime;
fileInfo = new FileInfo(path);
fileInfoCache.Add(path, fileInfo);
return fileInfo.LastWriteTime;
}
}
}

26
Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs
View File

@@ -21,6 +21,7 @@ namespace Flax.Build.Graph
private readonly List<BuildResultCache> _prevBuildCache = new List<BuildResultCache>();
private readonly List<string> _prevBuildCacheFiles = new List<string>();
private readonly Dictionary<string, int> _prevBuildCacheFileIndices = new Dictionary<string, int>();
/// <summary>
/// The workspace folder of the task graph.
@@ -297,9 +298,10 @@ namespace Flax.Build.Graph
var lastWrite = new DateTime(reader.ReadInt64());
var isValid = true;
if (File.Exists(file))
var cacheFile = true;
if (FileCache.Exists(file))
{
if (File.GetLastWriteTime(file) > lastWrite)
if (FileCache.GetLastWriteTime(file) > lastWrite)
{
isValid = false;
}
@@ -308,10 +310,16 @@ namespace Flax.Build.Graph
{
isValid = false;
}
else
cacheFile = false;
filesDates[i] = lastWrite;
filesValid[i] = isValid;
_prevBuildCacheFiles.Add(file);
_prevBuildCacheFileIndices.Add(file, i);
if (!isValid || !cacheFile)
FileCache.FileRemoveFromCache(file);
}
int taskCount = reader.ReadInt32();
@@ -468,16 +476,22 @@ namespace Flax.Build.Graph
fileIndices.Clear();
foreach (var file in files)
{
int fileIndex = _prevBuildCacheFiles.IndexOf(file);
if (fileIndex == -1)
if (!_prevBuildCacheFileIndices.TryGetValue(file, out int fileIndex))
{
fileIndex = _prevBuildCacheFiles.Count;
_prevBuildCacheFiles.Add(file);
_prevBuildCacheFileIndices.Add(file, fileIndex);
}
fileIndices.Add(fileIndex);
}
if (!task.HasValidCachedResults)
{
foreach (var file in task.ProducedFiles)
FileCache.FileRemoveFromCache(file);
}
_prevBuildCache.Add(new BuildResultCache
{
CmdLine = cmdLine,
@@ -501,8 +515,8 @@ namespace Flax.Build.Graph
// Last File Write
DateTime lastWrite;
if (File.Exists(file))
lastWrite = File.GetLastWriteTime(file);
if (FileCache.Exists(file))
lastWrite = FileCache.GetLastWriteTime(file);
else
lastWrite = DateTime.MinValue;
writer.Write(lastWrite.Ticks);

41
Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs
View File

@@ -7,7 +7,8 @@ using System.Linq;
using Flax.Build.Bindings;
using Flax.Build.Graph;
using Flax.Build.NativeCpp;
using Newtonsoft.Json;
using System.Text.Json.Serialization;
using System.Text.Json;
namespace Flax.Build
{
@@ -56,34 +57,52 @@ namespace Flax.Build
{
public string Name;
[NonSerialized]
[JsonIgnore]
public string NativePath;
[JsonProperty("NativePath")]
[JsonPropertyName("NativePath")]
public string NativePathProcessed;
[NonSerialized]
[JsonIgnore]
public string ManagedPath;
[JsonProperty("ManagedPath")]
[JsonPropertyName("ManagedPath")]
public string ManagedPathProcessed;
}
public class BuildTargetReferenceInfo
{
[NonSerialized]
[JsonIgnore]
public string ProjectPath;
[JsonProperty("ProjectPath")]
[JsonPropertyName("ProjectPath")]
public string ProjectPathProcessed;
[NonSerialized]
[JsonIgnore]
public string Path;
[JsonProperty("Path")]
[JsonPropertyName("Path")]
public string PathProcessed;
}
[JsonSourceGenerationOptions(IncludeFields = true)]
[JsonSerializable(typeof(BuildTargetBinaryModuleInfo))]
internal partial class BuildTargetBinaryModuleInfoSourceGenerationContext : JsonSerializerContext
{
}
[JsonSourceGenerationOptions(IncludeFields = true)]
[JsonSerializable(typeof(BuildTargetReferenceInfo))]
internal partial class BuildTargetReferenceInfoSourceGenerationContext : JsonSerializerContext
{
}
[JsonSourceGenerationOptions(IncludeFields = true)]
[JsonSerializable(typeof(BuildTargetInfo))]
internal partial class BuildTargetInfoSourceGenerationContext : JsonSerializerContext
{
}
public class BuildTargetInfo
{
public string Name;
@@ -985,7 +1004,7 @@ namespace Flax.Build
buildData.BuildInfo.AddReferencedBuilds(ref i, project.ProjectFolderPath, buildData.ReferenceBuilds);
if (!buildData.Target.IsPreBuilt)
Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonConvert.SerializeObject(buildData.BuildInfo, Formatting.Indented));
Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonSerializer.Serialize<BuildTargetInfo>(buildData.BuildInfo, new JsonSerializerOptions() { WriteIndented = true, IncludeFields = true, TypeInfoResolver = BuildTargetInfoSourceGenerationContext.Default }));
}
// Deploy files
@@ -1184,7 +1203,7 @@ namespace Flax.Build
buildData.BuildInfo.AddReferencedBuilds(ref i, project.ProjectFolderPath, buildData.ReferenceBuilds);
if (!buildData.Target.IsPreBuilt)
Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonConvert.SerializeObject(buildData.BuildInfo, Formatting.Indented));
Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonSerializer.Serialize<BuildTargetInfo>(buildData.BuildInfo, new JsonSerializerOptions() { WriteIndented = true, IncludeFields = true, TypeInfoResolver = BuildTargetInfoSourceGenerationContext.Default }));
}
// Deploy files

40
Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs
View File

@@ -17,8 +17,6 @@ namespace Flax.Build.NativeCpp
private static Dictionary<string, string[]> AllIncludesCache = new();
private static Dictionary<string, DateTime> DirectIncludesTimestampCache = new();
private static Dictionary<string, DateTime> AllIncludesTimestampCache = new();
private static Dictionary<string, bool> FileExistsCache = new();
private static Dictionary<string, DateTime> FileTimestampCache = new();
private static readonly string IncludeToken = "include";
private static string CachePath;
@@ -141,26 +139,6 @@ namespace Flax.Build.NativeCpp
}
}
private static bool FileExists(string path)
{
if (FileExistsCache.TryGetValue(path, out bool result))
return result;
result = File.Exists(path);
FileExistsCache.Add(path, result);
return result;
}
private static DateTime FileLastWriteTime(string path)
{
if (FileTimestampCache.TryGetValue(path, out DateTime result))
return result;
result = File.GetLastWriteTime(path);
FileTimestampCache.Add(path, result);
return result;
}
/// <summary>
/// Finds all included files by the source file (including dependencies).
/// </summary>
@@ -176,7 +154,7 @@ namespace Flax.Build.NativeCpp
{
if (AllIncludesTimestampCache.TryGetValue(sourceFile, out var cachedTimestamp))
{
lastModified = FileLastWriteTime(sourceFile);
lastModified = FileCache.GetLastWriteTime(sourceFile);
if (lastModified == cachedTimestamp)
return result;
}
@@ -185,7 +163,7 @@ namespace Flax.Build.NativeCpp
AllIncludesTimestampCache.Remove(sourceFile);
}
if (!FileExists(sourceFile))
if (!FileCache.Exists(sourceFile))
throw new Exception(string.Format("Cannot scan file \"{0}\" for includes because it does not exist.", sourceFile));
//using (new ProfileEventScope("FindAllIncludedFiles"))
@@ -231,7 +209,7 @@ namespace Flax.Build.NativeCpp
private static string[] GetDirectIncludes(string sourceFile)
{
if (!FileExists(sourceFile))
if (!FileCache.Exists(sourceFile))
return Array.Empty<string>();
DateTime? lastModified = null;
@@ -241,7 +219,7 @@ namespace Flax.Build.NativeCpp
{
if (DirectIncludesTimestampCache.TryGetValue(sourceFile, out var cachedTimestamp))
{
lastModified = FileLastWriteTime(sourceFile);
lastModified = FileCache.GetLastWriteTime(sourceFile);
if (lastModified == cachedTimestamp)
return result;
}
@@ -337,11 +315,11 @@ namespace Flax.Build.NativeCpp
// Relative to the workspace root
var includedFilePath = Path.Combine(Globals.Root, "Source", includedFile);
if (!FileExists(includedFilePath))
if (!FileCache.Exists(includedFilePath))
{
// Relative to the source file
includedFilePath = Path.Combine(sourceFileFolder, includedFile);
if (!FileExists(includedFilePath))
if (!FileCache.Exists(includedFilePath))
{
// Relative to any of the included project workspaces
var project = Globals.Project;
@@ -349,7 +327,7 @@ namespace Flax.Build.NativeCpp
foreach (var reference in project.References)
{
includedFilePath = Path.Combine(reference.Project.ProjectFolderPath, "Source", includedFile);
if (FileExists(includedFilePath))
if (FileCache.Exists(includedFilePath))
{
isValid = true;
break;
@@ -360,7 +338,7 @@ namespace Flax.Build.NativeCpp
if (!isValid && isLibraryInclude)
{
includedFilePath = Path.Combine(Globals.Root, "Source", "ThirdParty", includedFile);
if (FileExists(includedFilePath))
if (FileCache.Exists(includedFilePath))
{
isValid = true;
}
@@ -387,7 +365,7 @@ namespace Flax.Build.NativeCpp
result = includedFiles.ToArray();
DirectIncludesCache.Add(sourceFile, result);
if (!DirectIncludesTimestampCache.ContainsKey(sourceFile))
DirectIncludesTimestampCache.Add(sourceFile, lastModified ?? FileLastWriteTime(sourceFile));
DirectIncludesTimestampCache.Add(sourceFile, lastModified ?? FileCache.GetLastWriteTime(sourceFile));
return result;
}
}

1
Source/Tools/Flax.Build/Build/Plugins/NetworkingPlugin.cs
View File

@@ -508,7 +508,6 @@ namespace Flax.Build.Plugins
task.CommandPath = null;
task.InfoMessage = $"Generating networking code for {Path.GetFileName(assemblyPath)}...";
task.Cost = 50;
task.DisableCache = true;
task.DependentTasks = new HashSet<Task>();
task.DependentTasks.Add(buildTask);
}