Fix crash when closing handles to asset file while any asset streaming task is active for asset from that file

Source/Engine/Audio/AudioClip.cpp

@@ -277,6 +277,11 @@ bool AudioClip::ExtractDataRaw(Array<byte>& resultData, AudioDataInfo& resultDat
     return true;
 }
 
+void AudioClip::CancelStreaming()
+{
+    CancelStreamingTasks();
+}
+
 int32 AudioClip::GetMaxResidency() const
 {
     return _totalChunks;
@@ -338,6 +343,15 @@ Task* AudioClip::CreateStreamingTask(int32 residency)
     return result;
 }
 
+void AudioClip::CancelStreamingTasks()
+{
+    if (_streamingTask)
+    {
+        _streamingTask->Cancel();
+        ASSERT_LOW_LAYER(_streamingTask == nullptr);
+    }
+}
+
 bool AudioClip::init(AssetInitData& initData)
 {
     // Validate

Source/Engine/Audio/AudioClip.h

@@ -200,6 +200,9 @@ public:
 
 public:
 
+    // [BinaryAsset]
+    void CancelStreaming() override;
+    
     // [StreamableResource]
     int32 GetMaxResidency() const override;
     int32 GetCurrentResidency() const override;
@@ -207,6 +210,7 @@ public:
     bool CanBeUpdated() const override;
     Task* UpdateAllocation(int32 residency) override;
     Task* CreateStreamingTask(int32 residency) override;
+    void CancelStreamingTasks() override;
 
 protected:
 

Source/Engine/Content/Asset.cpp

@@ -481,6 +481,10 @@ void Asset::InitAsVirtual()
     _isLoaded = true;
 }
 
+void Asset::CancelStreaming()
+{
+}
+
 #if USE_EDITOR
 
 void Asset::GetReferences(Array<Guid>& output) const

Source/Engine/Content/Asset.h

@@ -154,6 +154,11 @@ public:
     /// </summary>
     virtual void InitAsVirtual();
 
+    /// <summary>
+    /// Cancels any asynchronous content streaming by this asset (eg. mesh data streaming into GPU memory). Will release any locks for asset storage container.
+    /// </summary>
+    virtual void CancelStreaming();
+
 #if USE_EDITOR
 
     /// <summary>

Source/Engine/Content/Assets/Model.cpp

@@ -769,6 +769,11 @@ void Model::InitAsVirtual()
     BinaryAsset::InitAsVirtual();
 }
 
+void Model::CancelStreaming()
+{
+    CancelStreamingTasks();
+}
+
 #if USE_EDITOR
 
 void Model::GetReferences(Array<Guid>& output) const
@@ -849,6 +854,15 @@ Task* Model::CreateStreamingTask(int32 residency)
     return result;
 }
 
+void Model::CancelStreamingTasks()
+{
+    if (_streamingTask)
+    {
+        _streamingTask->Cancel();
+        ASSERT_LOW_LAYER(_streamingTask == nullptr);
+    }
+}
+
 Asset::LoadResult Model::load()
 {
     // Get header chunk

Source/Engine/Content/Assets/Model.h

@@ -251,6 +251,7 @@ public:
     int32 GetLODsCount() const override;
     void GetMeshes(Array<MeshBase*>& meshes, int32 lodIndex = 0) override;
     void InitAsVirtual() override;
+    void CancelStreaming() override;
 #if USE_EDITOR
     void GetReferences(Array<Guid>& output) const override;
 #endif
@@ -262,6 +263,7 @@ public:
     bool CanBeUpdated() const override;
     Task* UpdateAllocation(int32 residency) override;
     Task* CreateStreamingTask(int32 residency) override;
+    void CancelStreamingTasks() override;
 
 protected:
     // [ModelBase]

Source/Engine/Content/Assets/SkinnedModel.cpp

@@ -760,6 +760,11 @@ void SkinnedModel::InitAsVirtual()
     BinaryAsset::InitAsVirtual();
 }
 
+void SkinnedModel::CancelStreaming()
+{
+    CancelStreamingTasks();
+}
+
 #if USE_EDITOR
 
 void SkinnedModel::GetReferences(Array<Guid>& output) const
@@ -840,6 +845,15 @@ Task* SkinnedModel::CreateStreamingTask(int32 residency)
     return result;
 }
 
+void SkinnedModel::CancelStreamingTasks()
+{
+    if (_streamingTask)
+    {
+        _streamingTask->Cancel();
+        ASSERT_LOW_LAYER(_streamingTask == nullptr);
+    }
+}
+
 Asset::LoadResult SkinnedModel::load()
 {
     // Get header chunk

Source/Engine/Content/Assets/SkinnedModel.h

@@ -273,6 +273,7 @@ public:
     int32 GetLODsCount() const override;
     void GetMeshes(Array<MeshBase*>& meshes, int32 lodIndex = 0) override;
     void InitAsVirtual() override;
+    void CancelStreaming() override;
 #if USE_EDITOR
     void GetReferences(Array<Guid>& output) const override;
 #endif
@@ -284,6 +285,7 @@ public:
     bool CanBeUpdated() const override;
     Task* UpdateAllocation(int32 residency) override;
     Task* CreateStreamingTask(int32 residency) override;
+    void CancelStreamingTasks() override;
 
 protected:
     // [ModelBase]

Source/Engine/Content/Storage/FlaxStorage.cpp

@@ -9,6 +9,8 @@
 #include "Engine/Platform/File.h"
 #include "Engine/Profiler/ProfilerCPU.h"
 #include "Engine/Serialization/FileWriteStream.h"
+#include "Engine/Content/Asset.h"
+#include "Engine/Content/Content.h"
 #include "Engine/Threading/Threading.h"
 #if USE_EDITOR
 #include "Engine/Serialization/JsonWriter.h"
@@ -1289,9 +1291,24 @@ void FlaxStorage::CloseFileHandles()
     // In those situations all the async tasks using this storage should be cancelled externally
 
     // Ensure that no one is using this resource
-    int32 waitTime = 500;
+    int32 waitTime = 10;
     while (Platform::AtomicRead(&_chunksLock) != 0 && waitTime-- > 0)
         Platform::Sleep(10);
+    if (Platform::AtomicRead(&_chunksLock) != 0)
+    {
+        // File can be locked by some streaming tasks (eg. AudioClip::StreamingTask or StreamModelLODTask)
+        for (int32 i = 0; i < GetEntriesCount(); i++)
+        {
+            Entry e;
+            GetEntry(i, e);
+            Asset* asset = Content::GetAsset(e.ID);
+            if (asset)
+            {
+                LOG(Info, "Canceling streaming for asset {0}", asset->ToString());
+                asset->CancelStreaming();
+            }
+        }
+    }
     ASSERT(_chunksLock == 0);
 
     _file.DeleteAll();

Source/Engine/Graphics/Textures/StreamingTexture.cpp

@@ -45,7 +45,6 @@ StreamingTexture::StreamingTexture(ITextureOwner* parent, const String& name)
     : StreamableResource(StreamingGroups::Instance()->Textures())
     , _owner(parent)
     , _texture(nullptr)
-    , _streamingTasksCount(0)
     , _isBlockCompressed(false)
 {
     ASSERT(_owner != nullptr);
@@ -62,7 +61,7 @@ StreamingTexture::~StreamingTexture()
 {
     UnloadTexture();
     SAFE_DELETE(_texture);
-    ASSERT(_streamingTasksCount == 0);
+    ASSERT(_streamingTasks.Count() == 0);
 }
 
 Float2 StreamingTexture::Size() const
@@ -138,7 +137,7 @@ void StreamingTexture::UnloadTexture()
     _texture->ReleaseGPU();
     _header.MipLevels = 0;
 
-    ASSERT(_streamingTasksCount == 0);
+    ASSERT(_streamingTasks.Count() == 0);
 }
 
 uint64 StreamingTexture::GetTotalMemoryUsage() const
@@ -173,7 +172,12 @@ bool StreamingTexture::CanBeUpdated() const
     // - is not initialized
     // - mip data uploading job running
     // - resize texture job running
-    return IsInitialized() && Platform::AtomicRead(&_streamingTasksCount) == 0;
+    if (IsInitialized())
+    {
+        ScopeLock lock(_owner->GetOwnerLocker());
+        return _streamingTasks.Count() == 0;
+    }
+    return false;
 }
 
 class StreamTextureResizeTask : public GPUTask
@@ -189,7 +193,7 @@ public:
         , _streamingTexture(texture)
         , _newTexture(newTexture)
     {
-        Platform::InterlockedIncrement(&_streamingTexture->_streamingTasksCount);
+        _streamingTexture->_streamingTasks.Add(this);
     }
 
     ~StreamTextureResizeTask()
@@ -222,7 +226,11 @@ protected:
 
     void OnEnd() override
     {
-        Platform::InterlockedDecrement(&_streamingTexture->_streamingTasksCount);
+        if (_streamingTexture)
+        {
+            ScopeLock lock(_streamingTexture->GetOwner()->GetOwnerLocker());
+            _streamingTexture->_streamingTasks.Remove(this);
+        }
 
         // Base
         GPUTask::OnEnd();
@@ -318,7 +326,7 @@ public:
         , _streamingTexture(texture)
         , _dataLock(_streamingTexture->GetOwner()->LockData())
     {
-        Platform::InterlockedIncrement(&_streamingTexture->_streamingTasksCount);
+        _streamingTexture->_streamingTasks.Add(this);
         _texture.OnUnload.Bind<StreamTextureMipTask, &StreamTextureMipTask::onResourceUnload2>(this);
     }
 
@@ -328,7 +336,8 @@ private:
         // Unlink texture
         if (_streamingTexture)
         {
-            Platform::InterlockedDecrement(&_streamingTexture->_streamingTasksCount);
+            ScopeLock lock(_streamingTexture->GetOwner()->GetOwnerLocker());
+            _streamingTexture->_streamingTasks.Remove(this);
             _streamingTexture = nullptr;
         }
     }
@@ -379,7 +388,8 @@ protected:
         _dataLock.Release();
         if (_streamingTexture)
         {
-            Platform::InterlockedDecrement(&_streamingTexture->_streamingTasksCount);
+            ScopeLock lock(_streamingTexture->GetOwner()->GetOwnerLocker());
+            _streamingTexture->_streamingTasks.Remove(this);
             _streamingTexture = nullptr;
         }
 
@@ -456,3 +466,11 @@ Task* StreamingTexture::CreateStreamingTask(int32 residency)
 
     return result;
 }
+
+void StreamingTexture::CancelStreamingTasks()
+{
+    ScopeLock lock(_owner->GetOwnerLocker());
+    auto tasks = _streamingTasks;
+    for (auto task : tasks)
+        task->Cancel();
+}

Source/Engine/Graphics/Textures/StreamingTexture.h

@@ -19,9 +19,9 @@ protected:
     ITextureOwner* _owner;
     GPUTexture* _texture;
     TextureHeader _header;
-    volatile mutable int64 _streamingTasksCount;
     int32 _minMipCountBlockCompressed;
     bool _isBlockCompressed;
+    Array<Task*, FixedAllocation<16>> _streamingTasks;
 
 public:
     StreamingTexture(ITextureOwner* owner, const String& name);
@@ -173,4 +173,5 @@ public:
     bool CanBeUpdated() const override;
     Task* UpdateAllocation(int32 residency) override;
     Task* CreateStreamingTask(int32 residency) override;
+    void CancelStreamingTasks() override;
 };

Source/Engine/Graphics/Textures/TextureBase.cpp

@@ -636,6 +636,11 @@ bool TextureBase::Init(void* ptr)
     return Init(initData);
 }
 
+void TextureBase::CancelStreaming()
+{
+    _texture.CancelStreamingTasks();
+}
+
 int32 TextureBase::CalculateChunkIndex(int32 mipIndex) const
 {
     // Mips are in 0-13 chunks

Source/Engine/Graphics/Textures/TextureBase.h

@@ -218,6 +218,9 @@ private:
     API_FUNCTION(NoProxy) bool Init(void* ptr);
 
 public:
+    // [BinaryAsset]
+    void CancelStreaming() override;
+
     // [ITextureOwner]
     CriticalSection& GetOwnerLocker() const override;
     Task* RequestMipDataAsync(int32 mipIndex) override;

Source/Engine/Streaming/StreamableResource.h

@@ -101,6 +101,11 @@ public:
     /// <returns>Async task or tasks that update resource residency level. Must be preceded with UpdateAllocation call.</returns>
     virtual Task* CreateStreamingTask(int32 residency) = 0;
 
+    /// <summary>
+    /// Cancels any streaming task (or tasks sequence) started for this resource.
+    /// </summary>
+    virtual void CancelStreamingTasks() = 0;
+
 public:
 
     struct StreamingCache