diff --git a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cache.cs b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cache.cs index d1aba9162..72896f186 100644 --- a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cache.cs +++ b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cache.cs @@ -253,7 +253,7 @@ namespace Flax.Build.Bindings { // Version writer.Write(CacheVersion); - writer.Write(File.GetLastWriteTime(Assembly.GetExecutingAssembly().Location).Ticks); + writer.Write(FileCache.GetLastWriteTime(Assembly.GetExecutingAssembly().Location).Ticks); // Build options writer.Write(moduleOptions.IntermediateFolder); @@ -270,7 +270,7 @@ namespace Flax.Build.Bindings { var headerFile = headerFiles[i]; writer.Write(headerFile); - writer.Write(File.GetLastWriteTime(headerFile).Ticks); + writer.Write(FileCache.GetLastWriteTime(headerFile).Ticks); } // Info @@ -281,7 +281,7 @@ namespace Flax.Build.Bindings private static bool LoadCache(ref ModuleInfo moduleInfo, BuildOptions moduleOptions, List headerFiles) { var path = GetCachePath(moduleInfo.Module, moduleOptions); - if (!File.Exists(path)) + if (!FileCache.Exists(path)) return false; try { @@ -292,7 +292,7 @@ namespace Flax.Build.Bindings var version = reader.ReadInt32(); if (version != CacheVersion) return false; - if (File.GetLastWriteTime(Assembly.GetExecutingAssembly().Location).Ticks != reader.ReadInt64()) + if (FileCache.GetLastWriteTime(Assembly.GetExecutingAssembly().Location).Ticks != reader.ReadInt64()) return false; // Build options @@ -320,7 +320,7 @@ namespace Flax.Build.Bindings var headerFile = headerFiles[i]; if (headerFile != reader.ReadString()) return false; - if (File.GetLastWriteTime(headerFile).Ticks > reader.ReadInt64()) + if (FileCache.GetLastWriteTime(headerFile).Ticks > reader.ReadInt64()) return false; } diff --git a/Source/Tools/Flax.Build/Build/Builder.Rules.cs b/Source/Tools/Flax.Build/Build/Builder.Rules.cs index 52f0a49b1..5acc97976 100644 --- a/Source/Tools/Flax.Build/Build/Builder.Rules.cs +++ b/Source/Tools/Flax.Build/Build/Builder.Rules.cs @@ -270,8 +270,9 @@ namespace Flax.Build private static void FindRules(string directory, List result) { // Optional way: - //result.AddRange(Directory.GetFiles(directory, '*' + BuildFilesPostfix, SearchOption.AllDirectories)); + result.AddRange(Directory.GetFiles(directory, '*' + BuildFilesPostfix, SearchOption.AllDirectories)); + /* var files = Directory.GetFiles(directory); for (int i = 0; i < files.Length; i++) { @@ -286,6 +287,7 @@ namespace Flax.Build { FindRules(directories[i], result); } + */ } } } diff --git a/Source/Tools/Flax.Build/Build/FileCache.cs b/Source/Tools/Flax.Build/Build/FileCache.cs new file mode 100644 index 000000000..a11b53415 --- /dev/null +++ b/Source/Tools/Flax.Build/Build/FileCache.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace Flax.Build +{ + /// + /// Cache filesystem related queries like File.Exists and File.GetLastWriteTime. + /// + public static class FileCache + { + private static Dictionary fileInfoCache = new Dictionary(); + + public static void FileRemoveFromCache(string path) + { + //fileInfoCache[path].Refresh(); + fileInfoCache.Remove(path); + } + + public static bool Exists(string path) + { + if (fileInfoCache.TryGetValue(path, out var fileInfo)) + return fileInfo.Exists; + + fileInfo = new FileInfo(path); + fileInfoCache.Add(path, fileInfo); + return fileInfo.Exists; + } + + public static DateTime GetLastWriteTime(string path) + { + + if (fileInfoCache.TryGetValue(path, out var fileInfo)) + return fileInfo.LastWriteTime; + + fileInfo = new FileInfo(path); + fileInfoCache.Add(path, fileInfo); + return fileInfo.LastWriteTime; + } + } +} diff --git a/Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs b/Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs index 5e092d155..c1ac6a258 100644 --- a/Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs +++ b/Source/Tools/Flax.Build/Build/Graph/TaskGraph.cs @@ -21,6 +21,7 @@ namespace Flax.Build.Graph private readonly List _prevBuildCache = new List(); private readonly List _prevBuildCacheFiles = new List(); + private readonly Dictionary _prevBuildCacheFileIndices = new Dictionary(); /// /// The workspace folder of the task graph. @@ -297,9 +298,10 @@ namespace Flax.Build.Graph var lastWrite = new DateTime(reader.ReadInt64()); var isValid = true; - if (File.Exists(file)) + var cacheFile = true; + if (FileCache.Exists(file)) { - if (File.GetLastWriteTime(file) > lastWrite) + if (FileCache.GetLastWriteTime(file) > lastWrite) { isValid = false; } @@ -308,10 +310,16 @@ namespace Flax.Build.Graph { isValid = false; } + else + cacheFile = false; filesDates[i] = lastWrite; filesValid[i] = isValid; _prevBuildCacheFiles.Add(file); + _prevBuildCacheFileIndices.Add(file, i); + + if (!isValid || !cacheFile) + FileCache.FileRemoveFromCache(file); } int taskCount = reader.ReadInt32(); @@ -468,16 +476,22 @@ namespace Flax.Build.Graph fileIndices.Clear(); foreach (var file in files) { - int fileIndex = _prevBuildCacheFiles.IndexOf(file); - if (fileIndex == -1) + if (!_prevBuildCacheFileIndices.TryGetValue(file, out int fileIndex)) { fileIndex = _prevBuildCacheFiles.Count; _prevBuildCacheFiles.Add(file); + _prevBuildCacheFileIndices.Add(file, fileIndex); } fileIndices.Add(fileIndex); } + if (!task.HasValidCachedResults) + { + foreach (var file in task.ProducedFiles) + FileCache.FileRemoveFromCache(file); + } + _prevBuildCache.Add(new BuildResultCache { CmdLine = cmdLine, @@ -501,8 +515,8 @@ namespace Flax.Build.Graph // Last File Write DateTime lastWrite; - if (File.Exists(file)) - lastWrite = File.GetLastWriteTime(file); + if (FileCache.Exists(file)) + lastWrite = FileCache.GetLastWriteTime(file); else lastWrite = DateTime.MinValue; writer.Write(lastWrite.Ticks); diff --git a/Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs b/Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs index 086cff297..225e46ba7 100644 --- a/Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs +++ b/Source/Tools/Flax.Build/Build/NativeCpp/Builder.NativeCpp.cs @@ -7,7 +7,8 @@ using System.Linq; using Flax.Build.Bindings; using Flax.Build.Graph; using Flax.Build.NativeCpp; -using Newtonsoft.Json; +using System.Text.Json.Serialization; +using System.Text.Json; namespace Flax.Build { @@ -56,34 +57,52 @@ namespace Flax.Build { public string Name; - [NonSerialized] + [JsonIgnore] public string NativePath; - [JsonProperty("NativePath")] + [JsonPropertyName("NativePath")] public string NativePathProcessed; - [NonSerialized] + [JsonIgnore] public string ManagedPath; - [JsonProperty("ManagedPath")] + [JsonPropertyName("ManagedPath")] public string ManagedPathProcessed; } public class BuildTargetReferenceInfo { - [NonSerialized] + [JsonIgnore] public string ProjectPath; - [JsonProperty("ProjectPath")] + [JsonPropertyName("ProjectPath")] public string ProjectPathProcessed; - [NonSerialized] + [JsonIgnore] public string Path; - [JsonProperty("Path")] + [JsonPropertyName("Path")] public string PathProcessed; } + [JsonSourceGenerationOptions(IncludeFields = true)] + [JsonSerializable(typeof(BuildTargetBinaryModuleInfo))] + internal partial class BuildTargetBinaryModuleInfoSourceGenerationContext : JsonSerializerContext + { + } + + [JsonSourceGenerationOptions(IncludeFields = true)] + [JsonSerializable(typeof(BuildTargetReferenceInfo))] + internal partial class BuildTargetReferenceInfoSourceGenerationContext : JsonSerializerContext + { + } + + [JsonSourceGenerationOptions(IncludeFields = true)] + [JsonSerializable(typeof(BuildTargetInfo))] + internal partial class BuildTargetInfoSourceGenerationContext : JsonSerializerContext + { + } + public class BuildTargetInfo { public string Name; @@ -985,7 +1004,7 @@ namespace Flax.Build buildData.BuildInfo.AddReferencedBuilds(ref i, project.ProjectFolderPath, buildData.ReferenceBuilds); if (!buildData.Target.IsPreBuilt) - Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonConvert.SerializeObject(buildData.BuildInfo, Formatting.Indented)); + Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonSerializer.Serialize(buildData.BuildInfo, new JsonSerializerOptions() { WriteIndented = true, IncludeFields = true, TypeInfoResolver = BuildTargetInfoSourceGenerationContext.Default })); } // Deploy files @@ -1184,7 +1203,7 @@ namespace Flax.Build buildData.BuildInfo.AddReferencedBuilds(ref i, project.ProjectFolderPath, buildData.ReferenceBuilds); if (!buildData.Target.IsPreBuilt) - Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonConvert.SerializeObject(buildData.BuildInfo, Formatting.Indented)); + Utilities.WriteFileIfChanged(Path.Combine(outputPath, target.Name + ".Build.json"), JsonSerializer.Serialize(buildData.BuildInfo, new JsonSerializerOptions() { WriteIndented = true, IncludeFields = true, TypeInfoResolver = BuildTargetInfoSourceGenerationContext.Default })); } // Deploy files diff --git a/Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs b/Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs index c986cb89a..abda26072 100644 --- a/Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs +++ b/Source/Tools/Flax.Build/Build/NativeCpp/IncludesCache.cs @@ -17,8 +17,6 @@ namespace Flax.Build.NativeCpp private static Dictionary AllIncludesCache = new(); private static Dictionary DirectIncludesTimestampCache = new(); private static Dictionary AllIncludesTimestampCache = new(); - private static Dictionary FileExistsCache = new(); - private static Dictionary FileTimestampCache = new(); private static readonly string IncludeToken = "include"; private static string CachePath; @@ -141,26 +139,6 @@ namespace Flax.Build.NativeCpp } } - private static bool FileExists(string path) - { - if (FileExistsCache.TryGetValue(path, out bool result)) - return result; - - result = File.Exists(path); - FileExistsCache.Add(path, result); - return result; - } - - private static DateTime FileLastWriteTime(string path) - { - if (FileTimestampCache.TryGetValue(path, out DateTime result)) - return result; - - result = File.GetLastWriteTime(path); - FileTimestampCache.Add(path, result); - return result; - } - /// /// Finds all included files by the source file (including dependencies). /// @@ -176,7 +154,7 @@ namespace Flax.Build.NativeCpp { if (AllIncludesTimestampCache.TryGetValue(sourceFile, out var cachedTimestamp)) { - lastModified = FileLastWriteTime(sourceFile); + lastModified = FileCache.GetLastWriteTime(sourceFile); if (lastModified == cachedTimestamp) return result; } @@ -185,7 +163,7 @@ namespace Flax.Build.NativeCpp AllIncludesTimestampCache.Remove(sourceFile); } - if (!FileExists(sourceFile)) + if (!FileCache.Exists(sourceFile)) throw new Exception(string.Format("Cannot scan file \"{0}\" for includes because it does not exist.", sourceFile)); //using (new ProfileEventScope("FindAllIncludedFiles")) @@ -231,7 +209,7 @@ namespace Flax.Build.NativeCpp private static string[] GetDirectIncludes(string sourceFile) { - if (!FileExists(sourceFile)) + if (!FileCache.Exists(sourceFile)) return Array.Empty(); DateTime? lastModified = null; @@ -241,7 +219,7 @@ namespace Flax.Build.NativeCpp { if (DirectIncludesTimestampCache.TryGetValue(sourceFile, out var cachedTimestamp)) { - lastModified = FileLastWriteTime(sourceFile); + lastModified = FileCache.GetLastWriteTime(sourceFile); if (lastModified == cachedTimestamp) return result; } @@ -337,11 +315,11 @@ namespace Flax.Build.NativeCpp // Relative to the workspace root var includedFilePath = Path.Combine(Globals.Root, "Source", includedFile); - if (!FileExists(includedFilePath)) + if (!FileCache.Exists(includedFilePath)) { // Relative to the source file includedFilePath = Path.Combine(sourceFileFolder, includedFile); - if (!FileExists(includedFilePath)) + if (!FileCache.Exists(includedFilePath)) { // Relative to any of the included project workspaces var project = Globals.Project; @@ -349,7 +327,7 @@ namespace Flax.Build.NativeCpp foreach (var reference in project.References) { includedFilePath = Path.Combine(reference.Project.ProjectFolderPath, "Source", includedFile); - if (FileExists(includedFilePath)) + if (FileCache.Exists(includedFilePath)) { isValid = true; break; @@ -360,7 +338,7 @@ namespace Flax.Build.NativeCpp if (!isValid && isLibraryInclude) { includedFilePath = Path.Combine(Globals.Root, "Source", "ThirdParty", includedFile); - if (FileExists(includedFilePath)) + if (FileCache.Exists(includedFilePath)) { isValid = true; } @@ -387,7 +365,7 @@ namespace Flax.Build.NativeCpp result = includedFiles.ToArray(); DirectIncludesCache.Add(sourceFile, result); if (!DirectIncludesTimestampCache.ContainsKey(sourceFile)) - DirectIncludesTimestampCache.Add(sourceFile, lastModified ?? FileLastWriteTime(sourceFile)); + DirectIncludesTimestampCache.Add(sourceFile, lastModified ?? FileCache.GetLastWriteTime(sourceFile)); return result; } } diff --git a/Source/Tools/Flax.Build/Build/Plugins/NetworkingPlugin.cs b/Source/Tools/Flax.Build/Build/Plugins/NetworkingPlugin.cs index 94d38490a..9dd19c3b9 100644 --- a/Source/Tools/Flax.Build/Build/Plugins/NetworkingPlugin.cs +++ b/Source/Tools/Flax.Build/Build/Plugins/NetworkingPlugin.cs @@ -508,7 +508,6 @@ namespace Flax.Build.Plugins task.CommandPath = null; task.InfoMessage = $"Generating networking code for {Path.GetFileName(assemblyPath)}..."; task.Cost = 50; - task.DisableCache = true; task.DependentTasks = new HashSet(); task.DependentTasks.Add(buildTask); } diff --git a/Source/Tools/Flax.Build/Flax.Build.csproj b/Source/Tools/Flax.Build/Flax.Build.csproj index 626b81502..9c837ee89 100644 --- a/Source/Tools/Flax.Build/Flax.Build.csproj +++ b/Source/Tools/Flax.Build/Flax.Build.csproj @@ -41,9 +41,6 @@ ..\..\..\Source\Platforms\DotNet\System.Text.Encoding.CodePages.dll - - ..\..\..\Source\Platforms\DotNet\Newtonsoft.Json.dll - ..\..\..\Source\Platforms\DotNet\Mono.Cecil.dll diff --git a/Source/Tools/Flax.Build/ProjectInfo.cs b/Source/Tools/Flax.Build/ProjectInfo.cs index 78cb48107..7830f59c1 100644 --- a/Source/Tools/Flax.Build/ProjectInfo.cs +++ b/Source/Tools/Flax.Build/ProjectInfo.cs @@ -4,11 +4,12 @@ using System; using System.IO; using System.Collections.Generic; using System.Linq; -using Newtonsoft.Json; +using System.Text.Json; +using System.Text.Json.Serialization; namespace Flax.Build { - public class FlaxVersionConverter : JsonConverter + public class FlaxVersionConverter : JsonConverter { /// /// Writes the JSON representation of the object. @@ -16,20 +17,9 @@ namespace Flax.Build /// The to write to. /// The value. /// The calling serializer. - public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + public override void Write(Utf8JsonWriter writer, Version value, JsonSerializerOptions options) { - if (value == null) - { - writer.WriteNull(); - } - else if (value is Version) - { - writer.WriteValue(value.ToString()); - } - else - { - throw new JsonSerializationException("Expected Version object value"); - } + writer.WriteStringValue(value.ToString()); } /// @@ -40,27 +30,27 @@ namespace Flax.Build /// The existing property value of the JSON that is being converted. /// The calling serializer. /// The object value. - public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) + public override Version? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - if (reader.TokenType == JsonToken.Null) + if (reader.TokenType == JsonTokenType.Null) { return null; } else { - if (reader.TokenType == JsonToken.StartObject) + if (reader.TokenType == JsonTokenType.StartObject) { try { reader.Read(); Dictionary values = new Dictionary(); - while (reader.TokenType == JsonToken.PropertyName) + while (reader.TokenType == JsonTokenType.PropertyName) { - var key = reader.Value as string; + var key = reader.GetString(); reader.Read(); - var val = (long)reader.Value; + var val = reader.GetInt32(); reader.Read(); - values.Add(key, (int)val); + values.Add(key, val); } int major = 0, minor = 0, build = 0; @@ -73,24 +63,24 @@ namespace Flax.Build } catch (Exception ex) { - throw new Exception(String.Format("Error parsing version string: {0}", reader.Value), ex); + throw new Exception(String.Format("Error parsing version string: {0}", reader.GetString()), ex); } } - else if (reader.TokenType == JsonToken.String) + else if (reader.TokenType == JsonTokenType.String) { try { - Version v = new Version((string)reader.Value!); + Version v = new Version((string)reader.GetString()!); return v; } catch (Exception ex) { - throw new Exception(String.Format("Error parsing version string: {0}", reader.Value), ex); + throw new Exception(String.Format("Error parsing version string: {0}", reader.GetString()), ex); } } else { - throw new Exception(String.Format("Unexpected token or value when parsing version. Token: {0}, Value: {1}", reader.TokenType, reader.Value)); + throw new Exception(String.Format("Unexpected token or value when parsing version. Token: {0}, Value: {1}", reader.TokenType, reader.GetString())); } } } @@ -108,6 +98,51 @@ namespace Flax.Build } } + public class ConfigurationDictionaryConverter : System.Text.Json.Serialization.JsonConverter> + { + public override Dictionary? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dictionary = new Dictionary(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + return dictionary; + if (reader.TokenType != JsonTokenType.PropertyName) + throw new Exception(); + + string key = reader.GetString(); + reader.Read(); + + string value; + if (reader.TokenType == JsonTokenType.True) + value = "true"; + else if (reader.TokenType == JsonTokenType.False) + value = "false"; + else + value = reader.GetString(); + dictionary.Add(key, value); + } + throw new Exception(); + } + + public override void Write(Utf8JsonWriter writer, Dictionary dictionary, JsonSerializerOptions options) + { + writer.WriteStartObject(); + foreach ((string key, string value) in dictionary) + { + var propertyName = key.ToString(); + writer.WritePropertyName(options.PropertyNamingPolicy?.ConvertName(propertyName) ?? propertyName); + writer.WriteStringValue(value); + } + writer.WriteEndObject(); + } + } + + [JsonSourceGenerationOptions(IncludeFields = true)] + [JsonSerializable(typeof(ProjectInfo))] + internal partial class ProjectInfoSourceGenerationContext : JsonSerializerContext + { + } /// /// Contains information about Flax project. @@ -129,7 +164,7 @@ namespace Flax.Build /// /// The referenced project. /// - [NonSerialized] + [JsonIgnore] public ProjectInfo Project; /// @@ -147,13 +182,13 @@ namespace Flax.Build /// /// The project file path. /// - [NonSerialized] + [JsonIgnore] public string ProjectPath; /// /// The project root folder path. /// - [NonSerialized] + [JsonIgnore] public string ProjectFolderPath; /// @@ -199,6 +234,7 @@ namespace Flax.Build /// /// The custom build configuration entries loaded from project file. /// + [System.Text.Json.Serialization.JsonConverter(typeof(ConfigurationDictionaryConverter))] public Dictionary Configuration; /// @@ -255,7 +291,7 @@ namespace Flax.Build /// public void Save() { - var contents = JsonConvert.SerializeObject(this, new JsonSerializerSettings() { Converters = new[] { new FlaxVersionConverter() } }); + var contents = JsonSerializer.Serialize(this, new JsonSerializerOptions() { Converters = { new FlaxVersionConverter() }, TypeInfoResolver = ProjectInfoSourceGenerationContext.Default }); File.WriteAllText(ProjectPath, contents); } @@ -281,7 +317,8 @@ namespace Flax.Build // Load Log.Verbose("Loading project file from \"" + path + "\"..."); var contents = File.ReadAllText(path); - var project = JsonConvert.DeserializeObject(contents, new JsonSerializerSettings() { Converters = new[] { new FlaxVersionConverter() } }); + var project = JsonSerializer.Deserialize(contents.AsSpan(), + new JsonSerializerOptions() { Converters = { new FlaxVersionConverter() }, IncludeFields = true, TypeInfoResolver = ProjectInfoSourceGenerationContext.Default }); project.ProjectPath = path; project.ProjectFolderPath = Path.GetDirectoryName(path);