GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add support for multi-threaded scripting API headers parsing

Browse Source
This commit is contained in:
Wojtek Figat
2021-02-15 15:04:23 +01:00
parent cdd68c9a89
commit 5714741a5d
4 changed files with 331 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
Source/Tools/Flax.Build/Bindings/BindingsGenerator.Parsing.cs
View File

@@ -17,6 +17,7 @@ namespace Flax.Build.Bindings
public Stack<ApiTypeInfo> ScopeTypeStack;
public Stack<AccessLevel> ScopeAccessStack;
public Dictionary<string, string> PreprocessorDefines;
public List<string> StringCache;
public ApiTypeInfo ValidScopeInfoFromStack
{
@@ -46,14 +47,12 @@ namespace Flax.Build.Bindings
}
}
private static List<string> _commentCache;
private static string[] ParseComment(ref ParsingContext context)
{
if (_commentCache == null)
_commentCache = new List<string>();
if (context.StringCache == null)
context.StringCache = new List<string>();
else
_commentCache.Clear();
context.StringCache.Clear();
int tokensCount = 0;
bool isValid = true;
@@ -77,7 +76,7 @@ namespace Flax.Build.Bindings
if (commentLine.StartsWith("// "))
commentLine = "/// " + commentLine.Substring(3);
_commentCache.Insert(0, commentLine);
context.StringCache.Insert(0, commentLine);
break;
}
default:
@@ -90,14 +89,14 @@ namespace Flax.Build.Bindings
for (var i = 0; i < tokensCount; i++)
context.Tokenizer.NextToken(true, true);
if (_commentCache.Count == 1)
if (context.StringCache.Count == 1)
{
// Ensure to have summary begin/end pair
_commentCache.Insert(0, "/// <summary>");
_commentCache.Add("/// </summary>");
context.StringCache.Insert(0, "/// <summary>");
context.StringCache.Add("/// </summary>");
}
return _commentCache.ToArray();
return context.StringCache.ToArray();
}
private struct TagParameter

126
Source/Tools/Flax.Build/Bindings/BindingsGenerator.cs
View File

@@ -3,6 +3,8 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Flax.Build.NativeCpp;
using BuildData = Flax.Build.Builder.BuildData;
@@ -98,85 +100,94 @@ namespace Flax.Build.Bindings
}
}
// TODO: use async tasks or thread pool to load and parse multiple header files at once using multi-threading
// Find and load files with API tags
string[] headerFilesContents = null;
using (new ProfileEventScope("LoadHeaderFiles"))
// Parse bindings
Log.Verbose($"Parsing API bindings for {module.Name} ({moduleInfo.Name})");
int concurrency = Math.Min(Math.Max(1, (int)(Environment.ProcessorCount * Configuration.ConcurrencyProcessorScale)), Configuration.MaxConcurrency);
concurrency = 1; // Disable concurrency for parsing (the gain is unnoticeable or even worse in some cases)
if (concurrency == 1 || headerFiles.Count < 2 * concurrency)
{
// Single-threaded
for (int i = 0; i < headerFiles.Count; i++)
{
var contents = File.ReadAllText(headerFiles[i]);
using (new ProfileEventScope(Path.GetFileName(headerFiles[i])))
{
ParseModuleInnerAsync(moduleInfo, moduleOptions, headerFiles, i);
}
}
}
else
{
// Sort by file size to improve performance (parse larger files first)
headerFiles.Sort((a, b) => new System.IO.FileInfo(b).Length.CompareTo(new System.IO.FileInfo(a).Length));
// Multi-threaded
ThreadPool.GetMinThreads(out var workerThreads, out var completionPortThreads);
if (workerThreads != concurrency)
ThreadPool.SetMaxThreads(concurrency, completionPortThreads);
Parallel.For(0, headerFiles.Count, (i, state) =>
{
using (new ProfileEventScope(Path.GetFileName(headerFiles[i])))
{
ParseModuleInnerAsync(moduleInfo, moduleOptions, headerFiles, i);
}
});
}
// Initialize API
using (new ProfileEventScope("Init"))
{
moduleInfo.Init(buildData);
}
return moduleInfo;
}
private static void ParseModuleInnerAsync(ModuleInfo moduleInfo, BuildOptions moduleOptions, List<string> headerFiles, int workIndex)
{
// Find and load files with API tags
bool hasApi = false;
string headerFileContents = File.ReadAllText(headerFiles[workIndex]);
for (int j = 0; j < ApiTokens.SearchTags.Length; j++)
{
if (contents.Contains(ApiTokens.SearchTags[j]))
if (headerFileContents.Contains(ApiTokens.SearchTags[j]))
{
if (headerFilesContents == null)
headerFilesContents = new string[headerFiles.Count];
headerFilesContents[i] = contents;
hasApi = true;
break;
}
}
}
}
if (headerFilesContents == null)
return moduleInfo;
// Skip if none of the headers was modified after last time generated C++ file was edited
// TODO: skip parsing if module has API cache file -> then load it from disk
/*if (!forceGenerate)
{
var lastGenerateTime = File.GetLastWriteTime(bindings.GeneratedCppFilePath);
var anyModified = false;
for (int i = 0; i < headerFiles.Count; i++)
{
if (File.GetLastWriteTime(headerFiles[i]) > lastGenerateTime)
{
anyModified = true;
break;
}
}
if (!anyModified)
if (!hasApi)
return;
}*/
Log.Verbose($"Parsing API bindings for {module.Name} ({moduleInfo.Name})");
// Process all header files to generate the module API code reflection
var context = new ParsingContext
{
CurrentAccessLevel = AccessLevel.Public,
ScopeTypeStack = new Stack<ApiTypeInfo>(),
ScopeAccessStack = new Stack<AccessLevel>(),
PreprocessorDefines = new Dictionary<string, string>(),
};
for (int i = 0; i < headerFiles.Count; i++)
{
if (headerFilesContents[i] == null)
continue;
// Process header file to generate the module API code reflection
var fileInfo = new FileInfo
{
Parent = null,
Children = new List<ApiTypeInfo>(),
Name = headerFiles[i],
Name = headerFiles[workIndex],
Namespace = moduleInfo.Name,
};
lock (moduleInfo)
{
moduleInfo.AddChild(fileInfo);
}
try
{
// Tokenize the source
var tokenizer = new Tokenizer();
tokenizer.Tokenize(headerFilesContents[i]);
tokenizer.Tokenize(headerFileContents);
// Init the context
context.Tokenizer = tokenizer;
context.File = fileInfo;
context.ScopeInfo = null;
context.ScopeTypeStack.Clear();
context.ScopeAccessStack.Clear();
context.PreprocessorDefines.Clear();
var context = new ParsingContext
{
File = fileInfo,
Tokenizer = tokenizer,
ScopeInfo = null,
CurrentAccessLevel = AccessLevel.Public,
ScopeTypeStack = new Stack<ApiTypeInfo>(),
ScopeAccessStack = new Stack<AccessLevel>(),
PreprocessorDefines = new Dictionary<string, string>(),
};
context.EnterScope(fileInfo);
// Process the source code
@@ -410,12 +421,6 @@ namespace Flax.Build.Bindings
}
}
// Initialize API
moduleInfo.Init(buildData);
return moduleInfo;
}
private static string ReplacePreProcessorDefines(string text, IEnumerable<string> defines)
{
foreach (var define in defines)
@@ -594,7 +599,6 @@ namespace Flax.Build.Bindings
{
foreach (var fieldInfo in structureInfo.Fields)
{
// TODO: support bit-fields in structure fields
if (fieldInfo.Type.IsBitField)
throw new NotImplementedException($"TODO: support bit-fields in structure fields (found field {fieldInfo} in structure {structureInfo.Name})");

9
Source/Tools/Flax.Build/Bindings/FileInfo.cs
View File

@@ -7,7 +7,7 @@ namespace Flax.Build.Bindings
/// <summary>
/// The native file information for bindings generator.
/// </summary>
public class FileInfo : ApiTypeInfo, IComparable<FileInfo>
public class FileInfo : ApiTypeInfo, IComparable, IComparable<FileInfo>
{
public override void AddChild(ApiTypeInfo apiTypeInfo)
{
@@ -26,5 +26,12 @@ namespace Flax.Build.Bindings
{
return System.IO.Path.GetFileName(Name);
}
public int CompareTo(object obj)
{
if (obj is ApiTypeInfo apiTypeInfo)
return Name.CompareTo(apiTypeInfo.Name);
return 0;
}
}
}

9
Source/Tools/Flax.Build/Bindings/ModuleInfo.cs
View File

@@ -13,5 +13,14 @@ namespace Flax.Build.Bindings
{
return "module " + Name;
}
/// <inheritdoc />
public override void Init(Builder.BuildData buildData)
{
base.Init(buildData);
// Sort module files to prevent bindings rebuild due to order changes (list might be created in async)
Children.Sort();
}
}
}