Fix deadlock in asset thumbnails rendering queue when texture streaming fails

#2057

Source/Editor/Content/Thumbnails/ThumbnailRequest.cs

@@ -35,6 +35,11 @@ namespace FlaxEditor.Content.Thumbnails
             /// The finalized state.
             /// </summary>
             Disposed,
+
+            /// <summary>
+            /// The request has failed (eg. asset cannot be loaded).
+            /// </summary>
+            Failed,
         };
 
         /// <summary>
@@ -78,6 +83,14 @@ namespace FlaxEditor.Content.Thumbnails
             Proxy = proxy;
         }
 
+        internal void Update()
+        {
+            if (State == States.Prepared && (!Asset || Asset.LastLoadFailed))
+            {
+                State = States.Failed;
+            }
+        }
+
         /// <summary>
         /// Prepares this request.
         /// </summary>
@@ -85,11 +98,8 @@ namespace FlaxEditor.Content.Thumbnails
         {
             if (State != States.Created)
                 throw new InvalidOperationException();
-
-            // Prepare
             Asset = FlaxEngine.Content.LoadAsync(Item.Path);
             Proxy.OnThumbnailDrawPrepare(this);
-
             State = States.Prepared;
         }
 
@@ -101,9 +111,7 @@ namespace FlaxEditor.Content.Thumbnails
         {
             if (State != States.Prepared)
                 throw new InvalidOperationException();
-
             Item.Thumbnail = icon;
-
             State = States.Rendered;
         }
 

Source/Editor/Content/Thumbnails/ThumbnailsModule.cs

@@ -120,6 +120,8 @@ namespace FlaxEditor.Content.Thumbnails
 
         internal static bool HasMinimumQuality(TextureBase asset)
         {
+            if (asset.HasStreamingError)
+                return true; // Don't block thumbnails queue when texture fails to stream in (eg. unsupported format)
             var mipLevels = asset.MipLevels;
             var minMipLevels = Mathf.Min(mipLevels, 7);
             return asset.IsLoaded && asset.ResidentMipLevels >= Mathf.Max(minMipLevels, (int)(mipLevels * MinimumRequiredResourcesQuality));
@@ -499,6 +501,7 @@ namespace FlaxEditor.Content.Thumbnails
                         var request = _requests[i];
                         try
                         {
+                            request.Update();
                             if (request.IsReady)
                             {
                                 isAnyReady = true;
@@ -507,6 +510,10 @@ namespace FlaxEditor.Content.Thumbnails
                             {
                                 request.Prepare();
                             }
+                            else if (request.State == ThumbnailRequest.States.Failed)
+                            {
+                                _requests.RemoveAt(i--);
+                            }
                         }
                         catch (Exception ex)
                         {

Source/Engine/Graphics/Textures/StreamingTexture.cpp

@@ -296,6 +296,7 @@ Task* StreamingTexture::UpdateAllocation(int32 residency)
         // Setup texture
         if (texture->Init(desc))
         {
+            Streaming.Error = true;
             LOG(Error, "Cannot allocate texture {0}.", ToString());
         }
         if (allocatedResidency != 0)

Source/Engine/Graphics/Textures/TextureBase.cpp

@@ -223,6 +223,11 @@ void TextureBase::SetTextureGroup(int32 textureGroup)
     }
 }
 
+bool TextureBase::HasStreamingError() const
+{
+    return _texture.Streaming.Error;
+}
+
 BytesContainer TextureBase::GetMipData(int32 mipIndex, int32& rowPitch, int32& slicePitch)
 {
     BytesContainer result;

Source/Engine/Graphics/Textures/TextureBase.h

@@ -148,6 +148,11 @@ public:
     /// </summary>
     API_PROPERTY() void SetTextureGroup(int32 textureGroup);
 
+    /// <summary>
+    /// Returns true if texture streaming failed (eg. pixel format is unsupported or texture data cannot be uploaded to GPU due to memory limit).
+    /// </summary>
+    API_PROPERTY() bool HasStreamingError() const;
+
 public:
     /// <summary>
     /// Gets the mip data.

Source/Engine/Streaming/StreamableResource.h

@@ -111,8 +111,9 @@ public:
     struct StreamingCache
     {
         int64 LastUpdate = 0;
-        int32 TargetResidency = 0;
         int64 TargetResidencyChange = 0;
+        int32 TargetResidency = 0;
+        bool Error = false;
         SamplesBuffer<float, 5> QualitySamples;
     };
 
@@ -131,7 +132,8 @@ public:
     /// <summary>
     /// Stops the streaming (eg. on streaming fail).
     /// </summary>
-    void ResetStreaming();
+    /// <param name="error">True if streaming failed.</param>
+    void ResetStreaming(bool error = true);
 
 protected:
 

Source/Engine/Streaming/Streaming.cpp

@@ -84,8 +84,9 @@ void StreamableResource::RequestStreamingUpdate()
     Streaming.LastUpdate = 0;
 }
 
-void StreamableResource::ResetStreaming()
+void StreamableResource::ResetStreaming(bool error)
 {
+    Streaming.Error = error;
     Streaming.TargetResidency = 0;
     Streaming.LastUpdate = DateTime::MaxValue().Ticks;
 }