From 5c9a27a6d615294a1a94b51a422bf457cf0b9d5b Mon Sep 17 00:00:00 2001 From: Ari Vuollet Date: Tue, 27 Dec 2022 20:59:38 +0200 Subject: [PATCH] Serialize C++ includes cache results Almost half the build tool runtime is spent scanning includes in C++-files, now the results are cached and invalidated when files last write timestamp changes. --- Source/Tools/Flax.Build/Build/Builder.cs | 11 + .../Build/NativeCpp/IncludesCache.cs | 188 +++++++++++++++++- 2 files changed, 191 insertions(+), 8 deletions(-) diff --git a/Source/Tools/Flax.Build/Build/Builder.cs b/Source/Tools/Flax.Build/Build/Builder.cs index 5139ed621..d81205161 100644 --- a/Source/Tools/Flax.Build/Build/Builder.cs +++ b/Source/Tools/Flax.Build/Build/Builder.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using Flax.Build.Graph; +using Flax.Build.NativeCpp; namespace Flax.Build { @@ -256,6 +257,11 @@ namespace Flax.Build if (targets.Length == 0) Log.Warning("No targets to build"); + using (new ProfileEventScope("LoadIncludesCache")) + { + IncludesCache.LoadCache(); + } + // Create task graph for building all targets var graph = new TaskGraph(project.ProjectFolderPath); foreach (var target in targets) @@ -393,6 +399,11 @@ namespace Flax.Build } } + using (new ProfileEventScope("SaveIncludesCache")) + { + IncludesCache.SaveCache(); + } + foreach (var target in targets) { target.PostBuild(); diff --git a/Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs b/Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs index fa4001158..a65252eab 100644 --- a/Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs +++ b/Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.Serialization.Formatters.Binary; using System.Text; namespace Flax.Build.NativeCpp @@ -13,9 +14,151 @@ namespace Flax.Build.NativeCpp /// public static class IncludesCache { - private static readonly Dictionary DirectIncludesCache = new Dictionary(); - private static readonly Dictionary AllIncludesCache = new Dictionary(); + private static Dictionary DirectIncludesCache = new Dictionary(); + private static Dictionary AllIncludesCache = new Dictionary(); + private static Dictionary DirectIncludesTimestampCache = new Dictionary(); + private static Dictionary AllIncludesTimestampCache = new Dictionary(); + private static Dictionary FileExistsCache = new Dictionary(); + private static Dictionary FileTimestampCache = new Dictionary(); private static readonly string IncludeToken = "include"; + private static string CachePath; + + public static void LoadCache() + { + CachePath = Path.Combine(Globals.Root, Configuration.IntermediateFolder, "IncludesCache.cache"); + if (!File.Exists(CachePath)) + return; + + using (var stream = new FileStream(CachePath, FileMode.Open)) + using (var reader = new BinaryReader(stream)) + { + int version = reader.ReadInt32(); + + // DirectIncludesCache + { + int count = reader.ReadInt32(); + for (int i = 0; i < count; i++) + { + string key = reader.ReadString(); + string[] values = new string[reader.ReadInt32()]; + for (int j = 0; j < values.Length; j++) + values[j] = reader.ReadString(); + + DirectIncludesCache.Add(key, values); + } + } + + // AllIncludesCache + { + int count = reader.ReadInt32(); + for (int i = 0; i < count; i++) + { + string key = reader.ReadString(); + string[] values = new string[reader.ReadInt32()]; + for (int j = 0; j < values.Length; j++) + values[j] = reader.ReadString(); + + AllIncludesCache.Add(key, values); + } + } + + // DirectIncludesTimestampCache + { + int count = reader.ReadInt32(); + for (int i = 0; i < count; i++) + { + string key = reader.ReadString(); + DateTime value = new DateTime(reader.ReadInt64()); + DirectIncludesTimestampCache.Add(key, value); + } + } + + // AllIncludesTimestampCache + { + int count = reader.ReadInt32(); + for (int i = 0; i < count; i++) + { + string key = reader.ReadString(); + DateTime value = new DateTime(reader.ReadInt64()); + AllIncludesTimestampCache.Add(key, value); + } + } + } + } + + public static void SaveCache() + { + using (var stream = new FileStream(CachePath, FileMode.Create)) + using (var writer = new BinaryWriter(stream)) + { + // Version + writer.Write(1); + + // DirectIncludesCache + { + writer.Write(DirectIncludesCache.Count); + foreach (KeyValuePair kvp in DirectIncludesCache) + { + writer.Write(kvp.Key); + writer.Write(kvp.Value.Length); + foreach (var value in kvp.Value) + writer.Write(value); + } + } + + // AllIncludesCache + { + writer.Write(AllIncludesCache.Count); + foreach (KeyValuePair kvp in AllIncludesCache) + { + writer.Write(kvp.Key); + writer.Write(kvp.Value.Length); + foreach (var value in kvp.Value) + writer.Write(value); + } + } + + // DirectIncludesTimestampCache + { + writer.Write(DirectIncludesTimestampCache.Count); + foreach (KeyValuePair kvp in DirectIncludesTimestampCache) + { + writer.Write(kvp.Key); + writer.Write(kvp.Value.Ticks); + } + } + + // AllIncludesTimestampCache + { + writer.Write(AllIncludesTimestampCache.Count); + foreach (KeyValuePair kvp in AllIncludesTimestampCache) + { + writer.Write(kvp.Key); + writer.Write(kvp.Value.Ticks); + } + } + } + } + + private static bool FileExists(string path) + { + if (FileExistsCache.TryGetValue(path, out bool result)) + return result; + + result = File.Exists(path); + FileExistsCache.Add(path, result); + return result; + } + + private static DateTime FileLastWriteTime(string path) + { + if (FileTimestampCache.TryGetValue(path, out DateTime result)) + return result; + + result = File.GetLastWriteTime(path); + FileTimestampCache.Add(path, result); + return result; + } /// /// Finds all included files by the source file (including dependencies). @@ -24,12 +167,24 @@ namespace Flax.Build.NativeCpp /// The list of included files by this file. Not null nut may be empty. public static string[] FindAllIncludedFiles(string sourceFile) { + DateTime? lastModified = null; + // Try hit the cache string[] result; if (AllIncludesCache.TryGetValue(sourceFile, out result)) - return result; + { + if (AllIncludesTimestampCache.TryGetValue(sourceFile, out var cachedTimestamp)) + { + lastModified = FileLastWriteTime(sourceFile); + if (lastModified == cachedTimestamp) + return result; + } - if (!File.Exists(sourceFile)) + AllIncludesCache.Remove(sourceFile); + AllIncludesTimestampCache.Remove(sourceFile); + } + + if (!FileExists(sourceFile)) throw new Exception(string.Format("Cannot scan file \"{0}\" for includes because it does not exist.", sourceFile)); //using (new ProfileEventScope("FindAllIncludedFiles")) @@ -44,6 +199,9 @@ namespace Flax.Build.NativeCpp result = includedFiles.ToArray(); AllIncludesCache.Add(sourceFile, result); + if (!AllIncludesTimestampCache.ContainsKey(sourceFile)) + AllIncludesTimestampCache.Add(sourceFile, lastModified ?? File.GetLastWriteTime(sourceFile)); + /*Log.Info("File includes for " + sourceFile); foreach (var e in result) { @@ -72,10 +230,22 @@ namespace Flax.Build.NativeCpp private static string[] GetDirectIncludes(string sourceFile) { + DateTime? lastModified = null; + // Try hit the cache string[] result; if (DirectIncludesCache.TryGetValue(sourceFile, out result)) - return result; + { + if (DirectIncludesTimestampCache.TryGetValue(sourceFile, out var cachedTimestamp)) + { + lastModified = FileLastWriteTime(sourceFile); + if (lastModified == cachedTimestamp) + return result; + } + + DirectIncludesCache.Remove(sourceFile); + DirectIncludesTimestampCache.Remove(sourceFile); + } // Find all files included directly var includedFiles = new HashSet(); @@ -152,11 +322,11 @@ namespace Flax.Build.NativeCpp // Relative to the workspace root var includedFilePath = Path.Combine(Globals.Root, "Source", includedFile); - if (!File.Exists(includedFilePath)) + if (!FileExists(includedFilePath)) { // Relative to the source file includedFilePath = Path.Combine(sourceFileFolder, includedFile); - if (!File.Exists(includedFilePath)) + if (!FileExists(includedFilePath)) { // Relative to any of the included project workspaces var project = Globals.Project; @@ -164,7 +334,7 @@ namespace Flax.Build.NativeCpp foreach (var reference in project.References) { includedFilePath = Path.Combine(reference.Project.ProjectFolderPath, "Source", includedFile); - if (File.Exists(includedFilePath)) + if (FileExists(includedFilePath)) { isValid = true; break; @@ -191,6 +361,8 @@ namespace Flax.Build.NativeCpp // Process result result = includedFiles.ToArray(); DirectIncludesCache.Add(sourceFile, result); + if (!DirectIncludesTimestampCache.ContainsKey(sourceFile)) + DirectIncludesTimestampCache.Add(sourceFile, lastModified ?? FileLastWriteTime(sourceFile)); return result; } }