GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Merge branch 'flax_build_perf_improvements' of https://github.com/GoaLitiuM/FlaxEngine into GoaLitiuM-flax_build_perf_improvements

Browse Source
This commit is contained in:
Wojtek Figat
2023-03-15 17:57:16 +01:00
parent 4f10f852c7 e11587e72e
commit 33ac9c8811
9 changed files with 177 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cache.cs
View File

@@ -253,7 +253,7 @@ namespace Flax.Build.Bindings
{
// Version
writer.Write(CacheVersion);
writer.Write(File.GetLastWriteTime(Assembly.GetExecutingAssembly().Location).Ticks);
writer.Write(FileCache.GetLastWriteTime(Assembly.GetExecutingAssembly().Location).Ticks);
// Build options
writer.Write(moduleOptions.IntermediateFolder);
@@ -270,7 +270,7 @@ namespace Flax.Build.Bindings
{
var headerFile = headerFiles[i];
writer.Write(headerFile);
writer.Write(File.GetLastWriteTime(headerFile).Ticks);
writer.Write(FileCache.GetLastWriteTime(headerFile).Ticks);
}
// Info
@@ -281,7 +281,7 @@ namespace Flax.Build.Bindings
private static bool LoadCache(ref ModuleInfo moduleInfo, BuildOptions moduleOptions, List<string> headerFiles)
{
var path = GetCachePath(moduleInfo.Module, moduleOptions);
if (!File.Exists(path))
if (!FileCache.Exists(path))
return false;
try
{
@@ -292,7 +292,7 @@ namespace Flax.Build.Bindings
var version = reader.ReadInt32();
if (version != CacheVersion)
return false;
if (File.GetLastWriteTime(Assembly.GetExecutingAssembly().Location).Ticks != reader.ReadInt64())
if (FileCache.GetLastWriteTime(Assembly.GetExecutingAssembly().Location).Ticks != reader.ReadInt64())
return false;
// Build options
@@ -320,7 +320,7 @@ namespace Flax.Build.Bindings
var headerFile = headerFiles[i];
if (headerFile != reader.ReadString())
return false;
if (File.GetLastWriteTime(headerFile).Ticks > reader.ReadInt64())
if (FileCache.GetLastWriteTime(headerFile).Ticks > reader.ReadInt64())
return false;
}

4
Source/Tools/Flax.Build/Build/Builder.Rules.cs
View File

@@ -270,8 +270,9 @@ namespace Flax.Build
private static void FindRules(string directory, List<string> result)
{
// Optional way:
//result.AddRange(Directory.GetFiles(directory, '*' + BuildFilesPostfix, SearchOption.AllDirectories));
result.AddRange(Directory.GetFiles(directory, '*' + BuildFilesPostfix, SearchOption.AllDirectories));
/*
var files = Directory.GetFiles(directory);
for (int i = 0; i < files.Length; i++)
{
@@ -286,6 +287,7 @@ namespace Flax.Build
{
FindRules(directories[i], result);
}
*/
}
}
}

41
Source/Tools/Flax.Build/Build/FileCache.cs Normal file
View File

@@ -0,0 +1,41 @@
using System;
using System.Collections.Generic;
using System.IO;
namespace Flax.Build
{
/// <summary>
/// Cache filesystem related queries like File.Exists and File.GetLastWriteTime.
/// </summary>
public static class FileCache
{
private static Dictionary<string, FileInfo> fileInfoCache = new Dictionary<string, FileInfo>();
public static void FileRemoveFromCache(string path)
{
//fileInfoCache[path].Refresh();
fileInfoCache.Remove(path);
}
public static bool Exists(string path)
{
if (fileInfoCache.TryGetValue(path, out var fileInfo))
return fileInfo.Exists;
fileInfo = new FileInfo(path);
fileInfoCache.Add(path, fileInfo);
return fileInfo.Exists;
}
public static DateTime GetLastWriteTime(string path)
{
if (fileInfoCache.TryGetValue(path, out var fileInfo))
return fileInfo.LastWriteTime;
fileInfo = new FileInfo(path);
fileInfoCache.Add(path, fileInfo);
return fileInfo.LastWriteTime;
}
}
}

26
Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs
View File

@@ -21,6 +21,7 @@ namespace Flax.Build.Graph
private readonly List<BuildResultCache> _prevBuildCache = new List<BuildResultCache>();
private readonly List<string> _prevBuildCacheFiles = new List<string>();
private readonly Dictionary<string, int> _prevBuildCacheFileIndices = new Dictionary<string, int>();
/// <summary>
/// The workspace folder of the task graph.
@@ -297,9 +298,10 @@ namespace Flax.Build.Graph
var lastWrite = new DateTime(reader.ReadInt64());
var isValid = true;
if (File.Exists(file))
var cacheFile = true;
if (FileCache.Exists(file))
{
if (File.GetLastWriteTime(file) > lastWrite)
if (FileCache.GetLastWriteTime(file) > lastWrite)
{
isValid = false;
}
@@ -308,10 +310,16 @@ namespace Flax.Build.Graph
{
isValid = false;
}
else
cacheFile = false;
filesDates[i] = lastWrite;
filesValid[i] = isValid;
_prevBuildCacheFiles.Add(file);
_prevBuildCacheFileIndices.Add(file, i);
if (!isValid || !cacheFile)
FileCache.FileRemoveFromCache(file);
}
int taskCount = reader.ReadInt32();
@@ -468,16 +476,22 @@ namespace Flax.Build.Graph
fileIndices.Clear();
foreach (var file in files)
{
int fileIndex = _prevBuildCacheFiles.IndexOf(file);
if (fileIndex == -1)
if (!_prevBuildCacheFileIndices.TryGetValue(file, out int fileIndex))
{
fileIndex = _prevBuildCacheFiles.Count;
_prevBuildCacheFiles.Add(file);
_prevBuildCacheFileIndices.Add(file, fileIndex);
}
fileIndices.Add(fileIndex);
}
if (!task.HasValidCachedResults)
{
foreach (var file in task.ProducedFiles)
FileCache.FileRemoveFromCache(file);
}
_prevBuildCache.Add(new BuildResultCache
{
CmdLine = cmdLine,
@@ -501,8 +515,8 @@ namespace Flax.Build.Graph
// Last File Write
DateTime lastWrite;
if (File.Exists(file))
lastWrite = File.GetLastWriteTime(file);
if (FileCache.Exists(file))
lastWrite = FileCache.GetLastWriteTime(file);
else
lastWrite = DateTime.MinValue;
writer.Write(lastWrite.Ticks);

41
Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs
View File

@@ -7,7 +7,8 @@ using System.Linq;
using Flax.Build.Bindings;
using Flax.Build.Graph;
using Flax.Build.NativeCpp;
using Newtonsoft.Json;
using System.Text.Json.Serialization;
using System.Text.Json;
namespace Flax.Build
{
@@ -56,34 +57,52 @@ namespace Flax.Build
{
public string Name;
[NonSerialized]
[JsonIgnore]
public string NativePath;
[JsonProperty("NativePath")]
[JsonPropertyName("NativePath")]
public string NativePathProcessed;
[NonSerialized]
[JsonIgnore]
public string ManagedPath;
[JsonProperty("ManagedPath")]
[JsonPropertyName("ManagedPath")]
public string ManagedPathProcessed;
}
public class BuildTargetReferenceInfo
{
[NonSerialized]
[JsonIgnore]
public string ProjectPath;
[JsonProperty("ProjectPath")]
[JsonPropertyName("ProjectPath")]
public string ProjectPathProcessed;
[NonSerialized]
[JsonIgnore]
public string Path;
[JsonProperty("Path")]
[JsonPropertyName("Path")]
public string PathProcessed;
}
[JsonSourceGenerationOptions(IncludeFields = true)]
[JsonSerializable(typeof(BuildTargetBinaryModuleInfo))]
internal partial class BuildTargetBinaryModuleInfoSourceGenerationContext : JsonSerializerContext
{
}
[JsonSourceGenerationOptions(IncludeFields = true)]
[JsonSerializable(typeof(BuildTargetReferenceInfo))]
internal partial class BuildTargetReferenceInfoSourceGenerationContext : JsonSerializerContext
{
}
[JsonSourceGenerationOptions(IncludeFields = true)]
[JsonSerializable(typeof(BuildTargetInfo))]
internal partial class BuildTargetInfoSourceGenerationContext : JsonSerializerContext
{
}
public class BuildTargetInfo
{
public string Name;
@@ -985,7 +1004,7 @@ namespace Flax.Build
buildData.BuildInfo.AddReferencedBuilds(ref i, project.ProjectFolderPath, buildData.ReferenceBuilds);
if (!buildData.Target.IsPreBuilt)
Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonConvert.SerializeObject(buildData.BuildInfo, Formatting.Indented));
Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonSerializer.Serialize<BuildTargetInfo>(buildData.BuildInfo, new JsonSerializerOptions() { WriteIndented = true, IncludeFields = true, TypeInfoResolver = BuildTargetInfoSourceGenerationContext.Default }));
}
// Deploy files
@@ -1184,7 +1203,7 @@ namespace Flax.Build
buildData.BuildInfo.AddReferencedBuilds(ref i, project.ProjectFolderPath, buildData.ReferenceBuilds);
if (!buildData.Target.IsPreBuilt)
Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonConvert.SerializeObject(buildData.BuildInfo, Formatting.Indented));
Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonSerializer.Serialize<BuildTargetInfo>(buildData.BuildInfo, new JsonSerializerOptions() { WriteIndented = true, IncludeFields = true, TypeInfoResolver = BuildTargetInfoSourceGenerationContext.Default }));
}
// Deploy files

40
Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs
View File

@@ -17,8 +17,6 @@ namespace Flax.Build.NativeCpp
private static Dictionary<string, string[]> AllIncludesCache = new();
private static Dictionary<string, DateTime> DirectIncludesTimestampCache = new();
private static Dictionary<string, DateTime> AllIncludesTimestampCache = new();
private static Dictionary<string, bool> FileExistsCache = new();
private static Dictionary<string, DateTime> FileTimestampCache = new();
private static readonly string IncludeToken = "include";
private static string CachePath;
@@ -141,26 +139,6 @@ namespace Flax.Build.NativeCpp
}
}
private static bool FileExists(string path)
{
if (FileExistsCache.TryGetValue(path, out bool result))
return result;
result = File.Exists(path);
FileExistsCache.Add(path, result);
return result;
}
private static DateTime FileLastWriteTime(string path)
{
if (FileTimestampCache.TryGetValue(path, out DateTime result))
return result;
result = File.GetLastWriteTime(path);
FileTimestampCache.Add(path, result);
return result;
}
/// <summary>
/// Finds all included files by the source file (including dependencies).
/// </summary>
@@ -176,7 +154,7 @@ namespace Flax.Build.NativeCpp
{
if (AllIncludesTimestampCache.TryGetValue(sourceFile, out var cachedTimestamp))
{
lastModified = FileLastWriteTime(sourceFile);
lastModified = FileCache.GetLastWriteTime(sourceFile);
if (lastModified == cachedTimestamp)
return result;
}
@@ -185,7 +163,7 @@ namespace Flax.Build.NativeCpp
AllIncludesTimestampCache.Remove(sourceFile);
}
if (!FileExists(sourceFile))
if (!FileCache.Exists(sourceFile))
throw new Exception(string.Format("Cannot scan file \"{0}\" for includes because it does not exist.", sourceFile));
//using (new ProfileEventScope("FindAllIncludedFiles"))
@@ -231,7 +209,7 @@ namespace Flax.Build.NativeCpp
private static string[] GetDirectIncludes(string sourceFile)
{
if (!FileExists(sourceFile))
if (!FileCache.Exists(sourceFile))
return Array.Empty<string>();
DateTime? lastModified = null;
@@ -241,7 +219,7 @@ namespace Flax.Build.NativeCpp
{
if (DirectIncludesTimestampCache.TryGetValue(sourceFile, out var cachedTimestamp))
{
lastModified = FileLastWriteTime(sourceFile);
lastModified = FileCache.GetLastWriteTime(sourceFile);
if (lastModified == cachedTimestamp)
return result;
}
@@ -337,11 +315,11 @@ namespace Flax.Build.NativeCpp
// Relative to the workspace root
var includedFilePath = Path.Combine(Globals.Root, "Source", includedFile);
if (!FileExists(includedFilePath))
if (!FileCache.Exists(includedFilePath))
{
// Relative to the source file
includedFilePath = Path.Combine(sourceFileFolder, includedFile);
if (!FileExists(includedFilePath))
if (!FileCache.Exists(includedFilePath))
{
// Relative to any of the included project workspaces
var project = Globals.Project;
@@ -349,7 +327,7 @@ namespace Flax.Build.NativeCpp
foreach (var reference in project.References)
{
includedFilePath = Path.Combine(reference.Project.ProjectFolderPath, "Source", includedFile);
if (FileExists(includedFilePath))
if (FileCache.Exists(includedFilePath))
{
isValid = true;
break;
@@ -360,7 +338,7 @@ namespace Flax.Build.NativeCpp
if (!isValid && isLibraryInclude)
{
includedFilePath = Path.Combine(Globals.Root, "Source", "ThirdParty", includedFile);
if (FileExists(includedFilePath))
if (FileCache.Exists(includedFilePath))
{
isValid = true;
}
@@ -387,7 +365,7 @@ namespace Flax.Build.NativeCpp
result = includedFiles.ToArray();
DirectIncludesCache.Add(sourceFile, result);
if (!DirectIncludesTimestampCache.ContainsKey(sourceFile))
DirectIncludesTimestampCache.Add(sourceFile, lastModified ?? FileLastWriteTime(sourceFile));
DirectIncludesTimestampCache.Add(sourceFile, lastModified ?? FileCache.GetLastWriteTime(sourceFile));
return result;
}
}

1
Source/Tools/Flax.Build/Build/Plugins/NetworkingPlugin.cs
View File

@@ -508,7 +508,6 @@ namespace Flax.Build.Plugins
task.CommandPath = null;
task.InfoMessage = $"Generating networking code for {Path.GetFileName(assemblyPath)}...";
task.Cost = 50;
task.DisableCache = true;
task.DependentTasks = new HashSet<Task>();
task.DependentTasks.Add(buildTask);
}

3
Source/Tools/Flax.Build/Flax.Build.csproj
View File

@@ -41,9 +41,6 @@
<Reference Include="System.Text.Encoding.CodePages">
<HintPath>..\..\..\Source\Platforms\DotNet\System.Text.Encoding.CodePages.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>..\..\..\Source\Platforms\DotNet\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="Mono.Cecil">
<HintPath>..\..\..\Source\Platforms\DotNet\Mono.Cecil.dll</HintPath>
</Reference>

101
Source/Tools/Flax.Build/ProjectInfo.cs
View File

@@ -4,11 +4,12 @@ using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using Newtonsoft.Json;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Flax.Build
{
public class FlaxVersionConverter : JsonConverter
public class FlaxVersionConverter : JsonConverter<Version>
{
/// <summary>
/// Writes the JSON representation of the object.
@@ -16,20 +17,9 @@ namespace Flax.Build
/// <param name="writer">The <see cref="JsonWriter"/> to write to.</param>
/// <param name="value">The value.</param>
/// <param name="serializer">The calling serializer.</param>
public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer)
public override void Write(Utf8JsonWriter writer, Version value, JsonSerializerOptions options)
{
if (value == null)
{
writer.WriteNull();
}
else if (value is Version)
{
writer.WriteValue(value.ToString());
}
else
{
throw new JsonSerializationException("Expected Version object value");
}
writer.WriteStringValue(value.ToString());
}
/// <summary>
@@ -40,27 +30,27 @@ namespace Flax.Build
/// <param name="existingValue">The existing property value of the JSON that is being converted.</param>
/// <param name="serializer">The calling serializer.</param>
/// <returns>The object value.</returns>
public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
public override Version? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType == JsonToken.Null)
if (reader.TokenType == JsonTokenType.Null)
{
return null;
}
else
{
if (reader.TokenType == JsonToken.StartObject)
if (reader.TokenType == JsonTokenType.StartObject)
{
try
{
reader.Read();
Dictionary<string, int> values = new Dictionary<string, int>();
while (reader.TokenType == JsonToken.PropertyName)
while (reader.TokenType == JsonTokenType.PropertyName)
{
var key = reader.Value as string;
var key = reader.GetString();
reader.Read();
var val = (long)reader.Value;
var val = reader.GetInt32();
reader.Read();
values.Add(key, (int)val);
values.Add(key, val);
}
int major = 0, minor = 0, build = 0;
@@ -73,24 +63,24 @@ namespace Flax.Build
}
catch (Exception ex)
{
throw new Exception(String.Format("Error parsing version string: {0}", reader.Value), ex);
throw new Exception(String.Format("Error parsing version string: {0}", reader.GetString()), ex);
}
}
else if (reader.TokenType == JsonToken.String)
else if (reader.TokenType == JsonTokenType.String)
{
try
{
Version v = new Version((string)reader.Value!);
Version v = new Version((string)reader.GetString()!);
return v;
}
catch (Exception ex)
{
throw new Exception(String.Format("Error parsing version string: {0}", reader.Value), ex);
throw new Exception(String.Format("Error parsing version string: {0}", reader.GetString()), ex);
}
}
else
{
throw new Exception(String.Format("Unexpected token or value when parsing version. Token: {0}, Value: {1}", reader.TokenType, reader.Value));
throw new Exception(String.Format("Unexpected token or value when parsing version. Token: {0}, Value: {1}", reader.TokenType, reader.GetString()));
}
}
}
@@ -108,6 +98,51 @@ namespace Flax.Build
}
}
public class ConfigurationDictionaryConverter : System.Text.Json.Serialization.JsonConverter<Dictionary<string, string>>
{
public override Dictionary<string, string>? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var dictionary = new Dictionary<string, string>();
while (reader.Read())
{
if (reader.TokenType == JsonTokenType.EndObject)
return dictionary;
if (reader.TokenType != JsonTokenType.PropertyName)
throw new Exception();
string key = reader.GetString();
reader.Read();
string value;
if (reader.TokenType == JsonTokenType.True)
value = "true";
else if (reader.TokenType == JsonTokenType.False)
value = "false";
else
value = reader.GetString();
dictionary.Add(key, value);
}
throw new Exception();
}
public override void Write(Utf8JsonWriter writer, Dictionary<string, string> dictionary, JsonSerializerOptions options)
{
writer.WriteStartObject();
foreach ((string key, string value) in dictionary)
{
var propertyName = key.ToString();
writer.WritePropertyName(options.PropertyNamingPolicy?.ConvertName(propertyName) ?? propertyName);
writer.WriteStringValue(value);
}
writer.WriteEndObject();
}
}
[JsonSourceGenerationOptions(IncludeFields = true)]
[JsonSerializable(typeof(ProjectInfo))]
internal partial class ProjectInfoSourceGenerationContext : JsonSerializerContext
{
}
/// <summary>
/// Contains information about Flax project.
@@ -129,7 +164,7 @@ namespace Flax.Build
/// <summary>
/// The referenced project.
/// </summary>
[NonSerialized]
[JsonIgnore]
public ProjectInfo Project;
/// <inheritdoc />
@@ -147,13 +182,13 @@ namespace Flax.Build
/// <summary>
/// The project file path.
/// </summary>
[NonSerialized]
[JsonIgnore]
public string ProjectPath;
/// <summary>
/// The project root folder path.
/// </summary>
[NonSerialized]
[JsonIgnore]
public string ProjectFolderPath;
/// <summary>
@@ -199,6 +234,7 @@ namespace Flax.Build
/// <summary>
/// The custom build configuration entries loaded from project file.
/// </summary>
[System.Text.Json.Serialization.JsonConverter(typeof(ConfigurationDictionaryConverter))]
public Dictionary<string, string> Configuration;
/// <summary>
@@ -255,7 +291,7 @@ namespace Flax.Build
/// </summary>
public void Save()
{
var contents = JsonConvert.SerializeObject(this, new JsonSerializerSettings() { Converters = new[] { new FlaxVersionConverter() } });
var contents = JsonSerializer.Serialize<ProjectInfo>(this, new JsonSerializerOptions() { Converters = { new FlaxVersionConverter() }, TypeInfoResolver = ProjectInfoSourceGenerationContext.Default });
File.WriteAllText(ProjectPath, contents);
}
@@ -281,7 +317,8 @@ namespace Flax.Build
// Load
Log.Verbose("Loading project file from \"" + path + "\"...");
var contents = File.ReadAllText(path);
var project = JsonConvert.DeserializeObject<ProjectInfo>(contents, new JsonSerializerSettings() { Converters = new[] { new FlaxVersionConverter() } });
var project = JsonSerializer.Deserialize<ProjectInfo>(contents.AsSpan(),
new JsonSerializerOptions() { Converters = { new FlaxVersionConverter() }, IncludeFields = true, TypeInfoResolver = ProjectInfoSourceGenerationContext.Default });
project.ProjectPath = path;
project.ProjectFolderPath = Path.GetDirectoryName(path);