From 8bcb984180701874f18e9bd41b0cc0ae0aa76f0f Mon Sep 17 00:00:00 2001 From: ExMatics HydrogenC <33123710+HydrogenC@users.noreply.github.com> Date: Sun, 19 Nov 2023 17:02:07 +0800 Subject: [PATCH 01/15] Implement SetNodeTransform --- Source/Engine/Level/Actors/AnimatedModel.cpp | 21 ++++++++++++++++++++ Source/Engine/Level/Actors/AnimatedModel.h | 16 +++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/Source/Engine/Level/Actors/AnimatedModel.cpp b/Source/Engine/Level/Actors/AnimatedModel.cpp index 88f7e6876..6c67620ae 100644 --- a/Source/Engine/Level/Actors/AnimatedModel.cpp +++ b/Source/Engine/Level/Actors/AnimatedModel.cpp @@ -172,6 +172,27 @@ void AnimatedModel::GetNodeTransformation(const StringView& nodeName, Matrix& no GetNodeTransformation(SkinnedModel ? SkinnedModel->FindNode(nodeName) : -1, nodeTransformation, worldSpace); } +void AnimatedModel::SetNodeTransformation(int32 nodeIndex, const Matrix& nodeTransformation, bool worldSpace) +{ + if (GraphInstance.NodesPose.IsEmpty()) + const_cast(this)->PreInitSkinningData(); // Ensure to have valid nodes pose to return + CHECK(nodeIndex >= 0 && nodeIndex < GraphInstance.NodesPose.Count()); + GraphInstance.NodesPose[nodeIndex] = nodeTransformation; + if (worldSpace) + { + Matrix world; + _transform.GetWorld(world); + Matrix invWorld; + Matrix::Invert(world, invWorld); + GraphInstance.NodesPose[nodeIndex] = GraphInstance.NodesPose[nodeIndex] * invWorld; + } + OnAnimationUpdated(); +} +void AnimatedModel::SetNodeTransformation(const StringView& nodeName, const Matrix& nodeTransformation, bool worldSpace) +{ + SetNodeTransformation(SkinnedModel ? SkinnedModel->FindNode(nodeName) : -1, nodeTransformation, worldSpace); +} + int32 AnimatedModel::FindClosestNode(const Vector3& location, bool worldSpace) const { if (GraphInstance.NodesPose.IsEmpty()) diff --git a/Source/Engine/Level/Actors/AnimatedModel.h b/Source/Engine/Level/Actors/AnimatedModel.h index 0c5c4d73b..3a4575bce 100644 --- a/Source/Engine/Level/Actors/AnimatedModel.h +++ b/Source/Engine/Level/Actors/AnimatedModel.h @@ -229,6 +229,22 @@ public: /// True if convert matrices into world-space, otherwise returned values will be in local-space of the actor. API_FUNCTION() void GetNodeTransformation(const StringView& nodeName, API_PARAM(Out) Matrix& nodeTransformation, bool worldSpace = false) const; + /// + /// Gets the node final transformation. + /// + /// The index of the skinned model skeleton node. + /// The final node transformation matrix. + /// True if convert matrices from world-space, otherwise values will be in local-space of the actor. + API_FUNCTION() void SetNodeTransformation(int32 nodeIndex, const Matrix& nodeTransformation, bool worldSpace = false); + + /// + /// Gets the node final transformation. + /// + /// The name of the skinned model skeleton node. + /// The final node transformation matrix. + /// True if convert matrices from world-space, otherwise values will be in local-space of the actor. + API_FUNCTION() void SetNodeTransformation(const StringView& nodeName, const Matrix& nodeTransformation, bool worldSpace = false); + /// /// Finds the closest node to a given location. /// From ddcb792767fe198a2ffa194e926ff721a0626ba7 Mon Sep 17 00:00:00 2001 From: ExMatics HydrogenC <33123710+HydrogenC@users.noreply.github.com> Date: Sun, 19 Nov 2023 17:07:42 +0800 Subject: [PATCH 02/15] Improve documentation --- Source/Engine/Level/Actors/AnimatedModel.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Engine/Level/Actors/AnimatedModel.h b/Source/Engine/Level/Actors/AnimatedModel.h index 3a4575bce..029e17b62 100644 --- a/Source/Engine/Level/Actors/AnimatedModel.h +++ b/Source/Engine/Level/Actors/AnimatedModel.h @@ -230,7 +230,7 @@ public: API_FUNCTION() void GetNodeTransformation(const StringView& nodeName, API_PARAM(Out) Matrix& nodeTransformation, bool worldSpace = false) const; /// - /// Gets the node final transformation. + /// Sets the node final transformation. If multiple nodes are to be set within a frame, do not use set worldSpace to true, and do the conversion yourself to avoid recalculation of inv matrices. /// /// The index of the skinned model skeleton node. /// The final node transformation matrix. @@ -238,7 +238,7 @@ public: API_FUNCTION() void SetNodeTransformation(int32 nodeIndex, const Matrix& nodeTransformation, bool worldSpace = false); /// - /// Gets the node final transformation. + /// Sets the node final transformation. If multiple nodes are to be set within a frame, do not use set worldSpace to true, and do the conversion yourself to avoid recalculation of inv matrices. /// /// The name of the skinned model skeleton node. /// The final node transformation matrix. From b14d88f8f8d06c95a19a44b80cfaef33b06f4b02 Mon Sep 17 00:00:00 2001 From: Chandler Cox Date: Mon, 20 Nov 2023 07:31:16 -0600 Subject: [PATCH 03/15] Add git submodule init to git cloning. --- Source/Editor/Windows/PluginsWindow.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Source/Editor/Windows/PluginsWindow.cs b/Source/Editor/Windows/PluginsWindow.cs index 268c22581..372505795 100644 --- a/Source/Editor/Windows/PluginsWindow.cs +++ b/Source/Editor/Windows/PluginsWindow.cs @@ -391,6 +391,25 @@ namespace FlaxEditor.Windows } Editor.Log("Plugin project has been cloned."); + + try + { + // Start git submodule clone + var settings = new CreateProcessSettings + { + FileName = "git", + WorkingDirectory = clonePath, + Arguments = "submodule update --init", + ShellExecute = false, + LogOutput = true, + }; + Platform.CreateProcess(ref settings); + } + catch (Exception e) + { + Editor.LogError($"Failed Git submodule process. {e}"); + return; + } // Find project config file. Could be different then what the user named the folder. var files = Directory.GetFiles(clonePath); From 9a091799fc143f141ba30fa87655968796fafde4 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Tue, 21 Nov 2023 00:00:43 +0100 Subject: [PATCH 04/15] Fix crash when debugging BT node state while tree is not running --- Source/Engine/AI/Behavior.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Source/Engine/AI/Behavior.cpp b/Source/Engine/AI/Behavior.cpp index a5b24fc8b..ab853e2a3 100644 --- a/Source/Engine/AI/Behavior.cpp +++ b/Source/Engine/AI/Behavior.cpp @@ -170,7 +170,11 @@ void Behavior::OnDisable() bool Behavior::GetNodeDebugRelevancy(const BehaviorTreeNode* node, const Behavior* behavior) { - return node && behavior && node->_executionIndex != -1 && behavior->_knowledge.RelevantNodes.Get(node->_executionIndex); + return node && + behavior && + node->_executionIndex >= 0 && + node->_executionIndex < behavior->_knowledge.RelevantNodes.Count() && + behavior->_knowledge.RelevantNodes.Get(node->_executionIndex); } String Behavior::GetNodeDebugInfo(const BehaviorTreeNode* node, Behavior* behavior) @@ -179,7 +183,7 @@ String Behavior::GetNodeDebugInfo(const BehaviorTreeNode* node, Behavior* behavi return String::Empty; BehaviorUpdateContext context; Platform::MemoryClear(&context, sizeof(context)); - if (behavior && node->_executionIndex != -1 && behavior->_knowledge.RelevantNodes.Get(node->_executionIndex)) + if (GetNodeDebugRelevancy(node, behavior)) { // Pass behavior and knowledge data only for relevant nodes to properly access it context.Behavior = behavior; From 4e37aafe6a5612dd8ac5ae0e235d06f6848ad6fd Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Tue, 21 Nov 2023 00:01:46 +0100 Subject: [PATCH 05/15] Fix BT logic flow when start/stopping behavior --- Source/Engine/AI/Behavior.cpp | 8 +++++++- Source/Engine/AI/BehaviorKnowledge.cpp | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Source/Engine/AI/Behavior.cpp b/Source/Engine/AI/Behavior.cpp index ab853e2a3..98fb9ba22 100644 --- a/Source/Engine/AI/Behavior.cpp +++ b/Source/Engine/AI/Behavior.cpp @@ -114,14 +114,19 @@ void Behavior::UpdateAsync() void Behavior::StartLogic() { + if (_result == BehaviorUpdateResult::Running) + return; PROFILE_CPU(); - // Ensure to have tree loaded on begin play + // Ensure to have tree loaded on play CHECK(Tree && !Tree->WaitForLoaded()); BehaviorTree* tree = Tree.Get(); CHECK(tree->Graph.Root); + // Setup state _result = BehaviorUpdateResult::Running; + _accumulatedTime = 0.0f; + _totalTime = 0; // Init knowledge _knowledge.InitMemory(tree); @@ -135,6 +140,7 @@ void Behavior::StopLogic(BehaviorUpdateResult result) _accumulatedTime = 0.0f; _totalTime = 0; _result = result; + _knowledge.FreeMemory(); } void Behavior::ResetLogic() diff --git a/Source/Engine/AI/BehaviorKnowledge.cpp b/Source/Engine/AI/BehaviorKnowledge.cpp index ffc011818..a73a1cfd4 100644 --- a/Source/Engine/AI/BehaviorKnowledge.cpp +++ b/Source/Engine/AI/BehaviorKnowledge.cpp @@ -83,7 +83,7 @@ bool AccessVariant(Variant& instance, const StringAnsiView& member, Variant& val } } #endif - else + else if (typeName.HasChars()) { LOG(Warning, "Missing scripting type \'{0}\'", String(typeName)); } From ea81ac6a7152af436ec7785c775fb04b0a1385f2 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Tue, 21 Nov 2023 11:10:05 +0100 Subject: [PATCH 06/15] Fix crash in Multi Blend 2D node in Anim Graph when using single animation on a triangle --- Source/Engine/Animations/Graph/AnimGroup.Animation.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Source/Engine/Animations/Graph/AnimGroup.Animation.cpp b/Source/Engine/Animations/Graph/AnimGroup.Animation.cpp index 5cceb31e3..6f0d7661b 100644 --- a/Source/Engine/Animations/Graph/AnimGroup.Animation.cpp +++ b/Source/Engine/Animations/Graph/AnimGroup.Animation.cpp @@ -1342,7 +1342,12 @@ void AnimGraphExecutor::ProcessGroupAnimation(Box* boxBase, Node* nodeBase, Valu { const bool xAxis = Math::IsZero(v0.X) && Math::IsZero(v1.X); const bool yAxis = Math::IsZero(v0.Y) && Math::IsZero(v1.Y); - if (xAxis || yAxis) + if (xAxis && yAxis) + { + // Single animation + value = SampleAnimation(node, loop, data.Length, startTimePos, bucket.TimePosition, newTimePos, aAnim, aData.W); + } + else if (xAxis || yAxis) { if (yAxis) { From 0db259e300b8204130a1fe891d8e07cc6f01abde Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Tue, 21 Nov 2023 12:24:55 +0100 Subject: [PATCH 07/15] Add `CustomScenes` feature to draw a fixed set of scenes within `SceneRenderTask` --- Source/Engine/Graphics/RenderTask.cpp | 23 +++++++++++++++++++---- Source/Engine/Graphics/RenderTask.h | 15 +++++++++++++-- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/Source/Engine/Graphics/RenderTask.cpp b/Source/Engine/Graphics/RenderTask.cpp index 48e89f59e..8a10a0444 100644 --- a/Source/Engine/Graphics/RenderTask.cpp +++ b/Source/Engine/Graphics/RenderTask.cpp @@ -8,11 +8,12 @@ #include "Engine/Core/Collections/Sorting.h" #include "Engine/Debug/DebugLog.h" #include "Engine/Level/Level.h" +#include "Engine/Level/Scene/Scene.h" #include "Engine/Level/Actors/Camera.h" +#include "Engine/Level/Actors/PostFxVolume.h" #include "Engine/Renderer/Renderer.h" #include "Engine/Render2D/Render2D.h" #include "Engine/Engine/Engine.h" -#include "Engine/Level/Actors/PostFxVolume.h" #include "Engine/Profiler/Profiler.h" #include "Engine/Renderer/RenderList.h" #include "Engine/Threading/Threading.h" @@ -202,15 +203,21 @@ void SceneRenderTask::CollectPostFxVolumes(RenderContext& renderContext) { Level::CollectPostFxVolumes(renderContext); } - if (EnumHasAllFlags(ActorsSource , ActorsSources::CustomActors)) + if (EnumHasAllFlags(ActorsSource, ActorsSources::CustomActors)) { for (Actor* a : CustomActors) { auto* postFxVolume = dynamic_cast(a); if (postFxVolume && a->GetIsActive()) - { postFxVolume->Collect(renderContext); - } + } + } + if (EnumHasAllFlags(ActorsSource, ActorsSources::CustomScenes)) + { + for (Scene* scene : CustomScenes) + { + if (scene && scene->IsActiveInHierarchy()) + scene->Rendering.CollectPostFxVolumes(renderContext); } } } @@ -282,6 +289,14 @@ void SceneRenderTask::OnCollectDrawCalls(RenderContextBatch& renderContextBatch, ASSERT_LOW_LAYER(_customActorsScene); _customActorsScene->Draw(renderContextBatch, (SceneRendering::DrawCategory)category); } + if (EnumHasAllFlags(ActorsSource, ActorsSources::CustomScenes)) + { + for (Scene* scene : CustomScenes) + { + if (scene && scene->IsActiveInHierarchy()) + scene->Rendering.Draw(renderContextBatch, (SceneRendering::DrawCategory)category); + } + } if (EnumHasAllFlags(ActorsSource, ActorsSources::Scenes)) { Level::DrawActors(renderContextBatch, category); diff --git a/Source/Engine/Graphics/RenderTask.h b/Source/Engine/Graphics/RenderTask.h index 9123751bc..864d95214 100644 --- a/Source/Engine/Graphics/RenderTask.h +++ b/Source/Engine/Graphics/RenderTask.h @@ -21,6 +21,7 @@ class PostProcessEffect; struct RenderContext; class Camera; class Actor; +class Scene; /// /// Allows to perform custom rendering using graphics pipeline. @@ -174,6 +175,11 @@ API_ENUM(Attributes="Flags") enum class ActorsSources /// CustomActors = 2, + /// + /// The scenes from the custom collection. + /// + CustomScenes = 4, + /// /// The actors from the loaded scenes and custom collection. /// @@ -267,9 +273,14 @@ public: public: /// - /// The custom set of actors to render. + /// The custom set of actors to render. Used when ActorsSources::CustomActors flag is active. /// - Array CustomActors; + API_FIELD() Array CustomActors; + + /// + /// The custom set of scenes to render. Used when ActorsSources::CustomScenes flag is active. + /// + API_FIELD() Array CustomScenes; /// /// Adds the custom actor to the rendering. From d01990e3bb77bd7ee6158912f5883a2c2961f3f1 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Tue, 21 Nov 2023 12:34:06 +0100 Subject: [PATCH 08/15] Fix deadlock in editor when using snap to the group with actor that has empty bounds #1971 --- Source/Editor/Gizmo/TransformGizmo.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/Editor/Gizmo/TransformGizmo.cs b/Source/Editor/Gizmo/TransformGizmo.cs index ef7558b8d..4a3fa39ba 100644 --- a/Source/Editor/Gizmo/TransformGizmo.cs +++ b/Source/Editor/Gizmo/TransformGizmo.cs @@ -111,7 +111,8 @@ namespace FlaxEditor.Gizmo if (isSelected) { GetSelectedObjectsBounds(out var selectionBounds, out _); - ray.Position = ray.GetPoint(selectionBounds.Size.Y * 0.5f); + var offset = Mathf.Max(selectionBounds.Size.Y * 0.5f, 1.0f); + ray.Position = ray.GetPoint(offset); continue; } From eba4b9bcc13a3a6307838fa7431af398d96f6140 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Tue, 21 Nov 2023 12:34:31 +0100 Subject: [PATCH 09/15] Add immediate game viewport sync after aspect ratio change in Editor --- Source/Editor/Windows/GameWindow.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/Editor/Windows/GameWindow.cs b/Source/Editor/Windows/GameWindow.cs index 8c6f5dc06..4a8c11176 100644 --- a/Source/Editor/Windows/GameWindow.cs +++ b/Source/Editor/Windows/GameWindow.cs @@ -386,6 +386,7 @@ namespace FlaxEditor.Windows { _viewport.Bounds = new Rectangle(Width * (1 - scaleWidth) / 2, 0, Width * scaleWidth, Height); } + _viewport.SyncBackbufferSize(); PerformLayout(); } From d99a92fd134e95a71f2e307bdbe08f67cfabdbb1 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Thu, 23 Nov 2023 13:46:43 +0100 Subject: [PATCH 10/15] Fix generic types including to be recursive in C++ bindings codegen #1966 --- .../Bindings/BindingsGenerator.Cpp.cs | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cpp.cs b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cpp.cs index 991ff196b..63cf9d2f8 100644 --- a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cpp.cs +++ b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Cpp.cs @@ -145,6 +145,20 @@ namespace Flax.Build.Bindings return $"(void*){result}"; } + + private static void GenerateCppAddFileReference(BuildData buildData, ApiTypeInfo caller, TypeInfo typeInfo, ApiTypeInfo apiType) + { + CppReferencesFiles.Add(apiType?.File); + if (typeInfo.GenericArgs != null) + { + for (int i = 0; i < typeInfo.GenericArgs.Count; i++) + { + var g = typeInfo.GenericArgs[i]; + GenerateCppAddFileReference(buildData, caller, g, FindApiTypeInfo(buildData, g, caller)); + } + } + } + public static string GenerateCppWrapperNativeToVariant(BuildData buildData, TypeInfo typeInfo, ApiTypeInfo caller, string value) { if (typeInfo.Type == "Variant") @@ -640,15 +654,7 @@ namespace Flax.Build.Bindings // Register any API types usage apiType = FindApiTypeInfo(buildData, typeInfo, caller); - CppReferencesFiles.Add(apiType?.File); - if (typeInfo.GenericArgs != null) - { - for (int i = 0; i < typeInfo.GenericArgs.Count; i++) - { - var t = FindApiTypeInfo(buildData, typeInfo.GenericArgs[i], caller); - CppReferencesFiles.Add(t?.File); - } - } + GenerateCppAddFileReference(buildData, caller, typeInfo, apiType); // Use dynamic array as wrapper container for fixed-size native arrays if (typeInfo.IsArray) @@ -1795,15 +1801,7 @@ namespace Flax.Build.Bindings return true; // Add includes to properly compile bindings (eg. SoftObjectReference) - CppReferencesFiles.Add(apiTypeInfo?.File); - if (typeInfo.GenericArgs != null) - { - for (int i = 0; i < typeInfo.GenericArgs.Count; i++) - { - var t = FindApiTypeInfo(buildData, typeInfo.GenericArgs[i], caller); - CppReferencesFiles.Add(t?.File); - } - } + GenerateCppAddFileReference(buildData, caller, typeInfo, apiTypeInfo); return false; } From d5075d845c674183cb00e61045e1e4d35f237096 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Thu, 23 Nov 2023 17:37:55 +0100 Subject: [PATCH 11/15] Fix content storage usage with relative paths #1966 --- Source/Engine/Content/BinaryAsset.cpp | 30 ++++++++++++------- Source/Engine/Content/Content.cpp | 9 +++--- .../Content/Storage/ContentStorageManager.cpp | 11 +++++++ .../Content/Storage/ContentStorageManager.h | 3 ++ 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/Source/Engine/Content/BinaryAsset.cpp b/Source/Engine/Content/BinaryAsset.cpp index b04603a96..9519f73df 100644 --- a/Source/Engine/Content/BinaryAsset.cpp +++ b/Source/Engine/Content/BinaryAsset.cpp @@ -323,26 +323,29 @@ bool BinaryAsset::SaveToAsset(const StringView& path, AssetInitData& data, bool { // Ensure path is in a valid format String pathNorm(path); - FileSystem::NormalizePath(pathNorm); + ContentStorageManager::FormatPath(pathNorm); + const StringView filePath = pathNorm; // Find target storage container and the asset - auto storage = ContentStorageManager::TryGetStorage(pathNorm); - auto asset = Content::GetAsset(pathNorm); + auto storage = ContentStorageManager::TryGetStorage(filePath); + auto asset = Content::GetAsset(filePath); auto binaryAsset = dynamic_cast(asset); if (asset && !binaryAsset) { LOG(Warning, "Cannot write to the non-binary asset location."); return true; } + if (!binaryAsset && !storage && FileSystem::FileExists(filePath)) + { + // Force-resolve storage (asset at that path could be not yet loaded into registry) + storage = ContentStorageManager::GetStorage(filePath); + } // Check if can perform write operation to the asset container - if (storage) + if (storage && !storage->AllowDataModifications()) { - if (!storage->AllowDataModifications()) - { - LOG(Warning, "Cannot write to the asset storage container."); - return true; - } + LOG(Warning, "Cannot write to the asset storage container."); + return true; } // Initialize data container @@ -352,6 +355,11 @@ bool BinaryAsset::SaveToAsset(const StringView& path, AssetInitData& data, bool // Use the same asset ID data.Header.ID = binaryAsset->GetID(); } + else if (storage && storage->GetEntriesCount()) + { + // Use the same file ID + data.Header.ID = storage->GetEntry(0).ID; + } else { // Randomize ID @@ -373,8 +381,8 @@ bool BinaryAsset::SaveToAsset(const StringView& path, AssetInitData& data, bool } else { - ASSERT(pathNorm.HasChars()); - result = FlaxStorage::Create(pathNorm, data, silentMode); + ASSERT(filePath.HasChars()); + result = FlaxStorage::Create(filePath, data, silentMode); } if (binaryAsset) binaryAsset->_isSaving = false; diff --git a/Source/Engine/Content/Content.cpp b/Source/Engine/Content/Content.cpp index bd27abc5a..910e16d66 100644 --- a/Source/Engine/Content/Content.cpp +++ b/Source/Engine/Content/Content.cpp @@ -449,18 +449,19 @@ Asset* Content::LoadAsync(const StringView& path, const ScriptingTypeHandle& typ { // Ensure path is in a valid format String pathNorm(path); - StringUtils::PathRemoveRelativeParts(pathNorm); + ContentStorageManager::FormatPath(pathNorm); + const StringView filePath = pathNorm; #if USE_EDITOR - if (!FileSystem::FileExists(pathNorm)) + if (!FileSystem::FileExists(filePath)) { - LOG(Error, "Missing file \'{0}\'", pathNorm); + LOG(Error, "Missing file \'{0}\'", filePath); return nullptr; } #endif AssetInfo assetInfo; - if (GetAssetInfo(pathNorm, assetInfo)) + if (GetAssetInfo(filePath, assetInfo)) { return LoadAsync(assetInfo.ID, type); } diff --git a/Source/Engine/Content/Storage/ContentStorageManager.cpp b/Source/Engine/Content/Storage/ContentStorageManager.cpp index d3e18e9d0..61e73a3f2 100644 --- a/Source/Engine/Content/Storage/ContentStorageManager.cpp +++ b/Source/Engine/Content/Storage/ContentStorageManager.cpp @@ -6,6 +6,7 @@ #include "Engine/Core/Log.h" #include "Engine/Engine/Engine.h" #include "Engine/Engine/EngineService.h" +#include "Engine/Engine/Globals.h" #include "Engine/Platform/FileSystem.h" #include "Engine/Profiler/ProfilerCPU.h" #include "Engine/Threading/TaskGraph.h" @@ -185,6 +186,16 @@ void ContentStorageManager::EnsureUnlocked() Locker.Unlock(); } +void ContentStorageManager::FormatPath(String& path) +{ + StringUtils::PathRemoveRelativeParts(path); + if (FileSystem::IsRelative(path)) + { + // Convert local-project paths into absolute format which is used by Content Storage system + path = Globals::ProjectFolder / path; + } +} + bool ContentStorageManager::IsFlaxStoragePath(const String& path) { auto extension = FileSystem::GetExtension(path).ToLower(); diff --git a/Source/Engine/Content/Storage/ContentStorageManager.h b/Source/Engine/Content/Storage/ContentStorageManager.h index c615632e9..84a6dc07e 100644 --- a/Source/Engine/Content/Storage/ContentStorageManager.h +++ b/Source/Engine/Content/Storage/ContentStorageManager.h @@ -75,6 +75,9 @@ public: /// static void EnsureUnlocked(); + // Formats path into valid format used by the storage system (normalized and absolute). + static void FormatPath(String& path); + public: /// /// Determines whether the specified path can be a binary asset file (based on it's extension). From 6cf7d49a10985ec6039e97213ab15e5c61c3e27b Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Thu, 23 Nov 2023 17:54:50 +0100 Subject: [PATCH 12/15] Fix crash during asset loading due to potential threading issue when 2 threads load the same asset #1930 --- Source/Engine/Content/Content.cpp | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/Source/Engine/Content/Content.cpp b/Source/Engine/Content/Content.cpp index 910e16d66..6970fc6d7 100644 --- a/Source/Engine/Content/Content.cpp +++ b/Source/Engine/Content/Content.cpp @@ -54,8 +54,7 @@ namespace // Assets CriticalSection AssetsLocker; Dictionary Assets(2048); - CriticalSection LoadCallAssetsLocker; - Array LoadCallAssets(64); + Array LoadCallAssets(PLATFORM_THREADS_LIMIT); CriticalSection LoadedAssetsToInvokeLocker; Array LoadedAssetsToInvoke(64); Array ToUnload; @@ -924,29 +923,29 @@ Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) } // Check if that asset is during loading - LoadCallAssetsLocker.Lock(); + AssetsLocker.Lock(); if (LoadCallAssets.Contains(id)) { - LoadCallAssetsLocker.Unlock(); + AssetsLocker.Unlock(); - // Wait for load end - // TODO: dont use active waiting and prevent deadlocks if running on a main thread - //while (!Engine::ShouldExit()) + // Wait for loading end by other thread while (true) { - LoadCallAssetsLocker.Lock(); - const bool contains = LoadCallAssets.Contains(id); - LoadCallAssetsLocker.Unlock(); - if (!contains) - return GetAsset(id); Platform::Sleep(1); + result = nullptr; + AssetsLocker.Lock(); + if (!LoadCallAssets.Contains(id)) + Assets.TryGet(id, result); + AssetsLocker.Unlock(); + if (result) + return result; } } else { // Mark asset as loading LoadCallAssets.Add(id); - LoadCallAssetsLocker.Unlock(); + AssetsLocker.Unlock(); } // Load asset @@ -954,9 +953,9 @@ Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) result = load(id, type, assetInfo); // End loading - LoadCallAssetsLocker.Lock(); + AssetsLocker.Lock(); LoadCallAssets.Remove(id); - LoadCallAssetsLocker.Unlock(); + AssetsLocker.Unlock(); return result; } From 359d4ba8efe5efd0e1cb7fd17f61094df988c63c Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Fri, 24 Nov 2023 10:25:21 +0100 Subject: [PATCH 13/15] More refactoring to 6cf7d49a10985ec6039e97213ab15e5c61c3e27b for better assets loading #1930 --- Source/Engine/Content/Content.cpp | 67 ++++++++++++++----------------- Source/Engine/Content/Content.h | 1 - 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/Source/Engine/Content/Content.cpp b/Source/Engine/Content/Content.cpp index 6970fc6d7..47a465bf0 100644 --- a/Source/Engine/Content/Content.cpp +++ b/Source/Engine/Content/Content.cpp @@ -910,9 +910,13 @@ Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) return nullptr; // Check if asset has been already loaded - Asset* result = GetAsset(id); + Asset* result = nullptr; + AssetsLocker.Lock(); + Assets.TryGet(id, result); if (result) { + AssetsLocker.Unlock(); + // Validate type if (IsAssetTypeIdInvalid(type, result->GetTypeHandle()) && !result->Is(type)) { @@ -923,57 +927,39 @@ Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) } // Check if that asset is during loading - AssetsLocker.Lock(); if (LoadCallAssets.Contains(id)) { AssetsLocker.Unlock(); // Wait for loading end by other thread - while (true) + bool contains = true; + while (contains) { Platform::Sleep(1); - result = nullptr; AssetsLocker.Lock(); - if (!LoadCallAssets.Contains(id)) - Assets.TryGet(id, result); + contains = LoadCallAssets.Contains(id); AssetsLocker.Unlock(); - if (result) - return result; } - } - else - { - // Mark asset as loading - LoadCallAssets.Add(id); - AssetsLocker.Unlock(); + Assets.TryGet(id, result); + return result; } - // Load asset - AssetInfo assetInfo; - result = load(id, type, assetInfo); - - // End loading - AssetsLocker.Lock(); - LoadCallAssets.Remove(id); + // Mark asset as loading and release lock so other threads can load other assets + LoadCallAssets.Add(id); AssetsLocker.Unlock(); - return result; -} - -Asset* Content::load(const Guid& id, const ScriptingTypeHandle& type, AssetInfo& assetInfo) -{ // Get cached asset info (from registry) + AssetInfo assetInfo; if (!GetAssetInfo(id, assetInfo)) { LOG(Warning, "Invalid or missing asset ({0}, {1}).", id, type.ToString()); - return nullptr; + goto LOAD_FAILED; } - #if ASSETS_LOADING_EXTRA_VERIFICATION if (!FileSystem::FileExists(assetInfo.Path)) { LOG(Error, "Cannot find file '{0}'", assetInfo.Path); - return nullptr; + goto LOAD_FAILED; } #endif @@ -982,28 +968,27 @@ Asset* Content::load(const Guid& id, const ScriptingTypeHandle& type, AssetInfo& if (factory == nullptr) { LOG(Error, "Cannot find asset factory. Info: {0}", assetInfo.ToString()); - return nullptr; + goto LOAD_FAILED; } // Create asset object - auto result = factory->New(assetInfo); + result = factory->New(assetInfo); if (result == nullptr) { LOG(Error, "Cannot create asset object. Info: {0}", assetInfo.ToString()); - return nullptr; + goto LOAD_FAILED; } - + ASSERT(result->GetID() == id); #if ASSETS_LOADING_EXTRA_VERIFICATION if (IsAssetTypeIdInvalid(type, result->GetTypeHandle()) && !result->Is(type)) { - LOG(Error, "Different loaded asset type! Asset: '{0}'. Expected type: {1}", assetInfo.ToString(), type.ToString()); + LOG(Warning, "Different loaded asset type! Asset: '{0}'. Expected type: {1}", assetInfo.ToString(), type.ToString()); result->DeleteObject(); - return nullptr; + goto LOAD_FAILED; } #endif // Register asset - ASSERT(result->GetID() == id); AssetsLocker.Lock(); #if ASSETS_LOADING_EXTRA_VERIFICATION ASSERT(!Assets.ContainsKey(id)); @@ -1011,12 +996,20 @@ Asset* Content::load(const Guid& id, const ScriptingTypeHandle& type, AssetInfo& Assets.Add(id, result); // Start asset loading - // TODO: refactor this to create asset loading task-chain before AssetsLocker.Lock() to allow better parallelization result->startLoading(); + // Remove from the loading queue and release lock + LoadCallAssets.Remove(id); AssetsLocker.Unlock(); return result; + +LOAD_FAILED: + // Remove from loading queue + AssetsLocker.Lock(); + LoadCallAssets.Remove(id); + AssetsLocker.Unlock(); + return nullptr; } #if ENABLE_ASSETS_DISCOVERY diff --git a/Source/Engine/Content/Content.h b/Source/Engine/Content/Content.h index 6393ce48b..cce57194c 100644 --- a/Source/Engine/Content/Content.h +++ b/Source/Engine/Content/Content.h @@ -366,7 +366,6 @@ private: static void onAssetLoaded(Asset* asset); static void onAssetUnload(Asset* asset); static void onAssetChangeId(Asset* asset, const Guid& oldId, const Guid& newId); - static Asset* load(const Guid& id, const ScriptingTypeHandle& type, AssetInfo& assetInfo); private: static void deleteFileSafety(const StringView& path, const Guid& id); From aa1b779463cc297d822aa81581703dfaea4ab0d7 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Fri, 24 Nov 2023 10:25:46 +0100 Subject: [PATCH 14/15] Fix `Win32CriticalSection` to use spin count of `4000` instead of just `100` #1930 --- Source/Engine/Platform/Win32/Win32CriticalSection.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Engine/Platform/Win32/Win32CriticalSection.h b/Source/Engine/Platform/Win32/Win32CriticalSection.h index 95d85efac..97221626c 100644 --- a/Source/Engine/Platform/Win32/Win32CriticalSection.h +++ b/Source/Engine/Platform/Win32/Win32CriticalSection.h @@ -28,7 +28,7 @@ public: /// Win32CriticalSection() { - Windows::InitializeCriticalSectionEx(&_criticalSection, 100, 0x01000000); + Windows::InitializeCriticalSectionEx(&_criticalSection, 4000, 0x01000000); } /// From f61f35b259757b425f2c4b0817d42d2ad88bbabf Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Fri, 24 Nov 2023 10:53:07 +0100 Subject: [PATCH 15/15] Fix compilation regression --- Source/Engine/Content/Content.cpp | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Source/Engine/Content/Content.cpp b/Source/Engine/Content/Content.cpp index 47a465bf0..982ae599f 100644 --- a/Source/Engine/Content/Content.cpp +++ b/Source/Engine/Content/Content.cpp @@ -948,18 +948,20 @@ Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) LoadCallAssets.Add(id); AssetsLocker.Unlock(); +#define LOAD_FAILED() AssetsLocker.Lock(); LoadCallAssets.Remove(id); AssetsLocker.Unlock(); return nullptr + // Get cached asset info (from registry) AssetInfo assetInfo; if (!GetAssetInfo(id, assetInfo)) { LOG(Warning, "Invalid or missing asset ({0}, {1}).", id, type.ToString()); - goto LOAD_FAILED; + LOAD_FAILED(); } #if ASSETS_LOADING_EXTRA_VERIFICATION if (!FileSystem::FileExists(assetInfo.Path)) { LOG(Error, "Cannot find file '{0}'", assetInfo.Path); - goto LOAD_FAILED; + LOAD_FAILED(); } #endif @@ -968,7 +970,7 @@ Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) if (factory == nullptr) { LOG(Error, "Cannot find asset factory. Info: {0}", assetInfo.ToString()); - goto LOAD_FAILED; + LOAD_FAILED(); } // Create asset object @@ -976,7 +978,7 @@ Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) if (result == nullptr) { LOG(Error, "Cannot create asset object. Info: {0}", assetInfo.ToString()); - goto LOAD_FAILED; + LOAD_FAILED(); } ASSERT(result->GetID() == id); #if ASSETS_LOADING_EXTRA_VERIFICATION @@ -984,7 +986,7 @@ Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) { LOG(Warning, "Different loaded asset type! Asset: '{0}'. Expected type: {1}", assetInfo.ToString(), type.ToString()); result->DeleteObject(); - goto LOAD_FAILED; + LOAD_FAILED(); } #endif @@ -1002,14 +1004,9 @@ Asset* Content::LoadAsync(const Guid& id, const ScriptingTypeHandle& type) LoadCallAssets.Remove(id); AssetsLocker.Unlock(); - return result; +#undef LOAD_FAILED -LOAD_FAILED: - // Remove from loading queue - AssetsLocker.Lock(); - LoadCallAssets.Remove(id); - AssetsLocker.Unlock(); - return nullptr; + return result; } #if ENABLE_ASSETS_DISCOVERY