From ffe510560241a2f085c5dfb20f5a0590129c4c28 Mon Sep 17 00:00:00 2001
From: Wojtek Figat <wojtek@figat.pl>
Date: Fri, 10 May 2024 13:12:07 +0200
Subject: [PATCH] Fixes for audio playback in videos

---
 Source/Engine/Video/MF/VideoBackendMF.cpp | 23 +++++++++++++--
 Source/Engine/Video/Types.h               |  4 +++
 Source/Engine/Video/Video.cpp             | 34 +++++++++++++++++++++--
 3 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/Source/Engine/Video/MF/VideoBackendMF.cpp b/Source/Engine/Video/MF/VideoBackendMF.cpp
index 207f9b72c..08929fe5d 100644
--- a/Source/Engine/Video/MF/VideoBackendMF.cpp
+++ b/Source/Engine/Video/MF/VideoBackendMF.cpp
@@ -330,7 +330,11 @@ namespace MF
             if (sample)
                 sample->Release();
             if (isGoodSample)
+            {
                 goodSamplesLeft--;
+                if (!sample)
+                    return true; // Got good sample but without data so seek
+            }
 
             if (flags & MF_SOURCE_READERF_ENDOFSTREAM)
             {
@@ -392,6 +396,7 @@ namespace MF
         }
 
         // Update current position
+        bool canSeek = true;
     SEEK_START:
         if (playerMF.Seek)
         {
@@ -418,7 +423,12 @@ namespace MF
         {
             // Failed to pick a valid sample so try again with seeking
             playerMF.Seek = 1;
-            goto SEEK_START;
+            if (canSeek)
+            {
+                // Prevent deadlock on sync
+                canSeek = false;
+                goto SEEK_START;
+            }
         }
         if (player.AudioInfo.BitDepth != 0)
             ReadStream(player, playerMF, MF_SOURCE_READER_FIRST_AUDIO_STREAM, dt);
@@ -497,6 +507,7 @@ void VideoBackendMF::Player_Play(VideoBackendPlayer& player)
     PROFILE_CPU();
     auto& playerMF = player.GetBackendState<VideoPlayerMF>();
     playerMF.Playing = 1;
+    player.PlayAudio();
 }
 
 void VideoBackendMF::Player_Pause(VideoBackendPlayer& player)
@@ -504,6 +515,7 @@ void VideoBackendMF::Player_Pause(VideoBackendPlayer& player)
     PROFILE_CPU();
     auto& playerMF = player.GetBackendState<VideoPlayerMF>();
     playerMF.Playing = 0;
+    player.PauseAudio();
 }
 
 void VideoBackendMF::Player_Stop(VideoBackendPlayer& player)
@@ -514,14 +526,19 @@ void VideoBackendMF::Player_Stop(VideoBackendPlayer& player)
     playerMF.Playing = 0;
     playerMF.FirstFrame = 1;
     playerMF.Seek = 1;
+    player.StopAudio();
 }
 
 void VideoBackendMF::Player_Seek(VideoBackendPlayer& player, TimeSpan time)
 {
     PROFILE_CPU();
     auto& playerMF = player.GetBackendState<VideoPlayerMF>();
-    playerMF.Time = time;
-    playerMF.Seek = 1;
+    if (playerMF.Time != time)
+    {
+        playerMF.Time = time;
+        playerMF.Seek = 1;
+        player.StopAudio();
+    }
 }
 
 TimeSpan VideoBackendMF::Player_GetTime(const VideoBackendPlayer& player)
diff --git a/Source/Engine/Video/Types.h b/Source/Engine/Video/Types.h
index 20b065493..bbf920b5c 100644
--- a/Source/Engine/Video/Types.h
+++ b/Source/Engine/Video/Types.h
@@ -35,6 +35,7 @@ struct VideoBackendPlayer
     int32 VideoFrameWidth, VideoFrameHeight;
     PixelFormat Format;
     float FrameRate;
+    uint8 IsAudioPlayPending : 1;
     TimeSpan Duration;
     TimeSpan VideoFrameTime, VideoFrameDuration;
     TimeSpan AudioBufferTime, AudioBufferDuration;
@@ -67,6 +68,9 @@ struct VideoBackendPlayer
     }
 
     void Created(const VideoBackendPlayerInfo& info);
+    void PlayAudio();
+    void PauseAudio();
+    void StopAudio();
     void InitVideoFrame();
     void UpdateVideoFrame(Span<byte> data, TimeSpan time, TimeSpan duration);
     void UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeSpan duration);
diff --git a/Source/Engine/Video/Video.cpp b/Source/Engine/Video/Video.cpp
index 2da3d2b4d..705517e20 100644
--- a/Source/Engine/Video/Video.cpp
+++ b/Source/Engine/Video/Video.cpp
@@ -214,6 +214,33 @@ void VideoBackendPlayer::Created(const VideoBackendPlayerInfo& info)
 #endif
 }
 
+void VideoBackendPlayer::PlayAudio()
+{
+    if (AudioSource)
+    {
+        IsAudioPlayPending = 0;
+        AudioBackend::Source::Play(AudioSource);
+    }
+}
+
+void VideoBackendPlayer::PauseAudio()
+{
+    if (AudioSource)
+    {
+        IsAudioPlayPending = 0;
+        AudioBackend::Source::Pause(AudioSource);
+    }
+}
+
+void VideoBackendPlayer::StopAudio()
+{
+    if (AudioSource)
+    {
+        AudioBackend::Source::Stop(AudioSource);
+        IsAudioPlayPending = 1;
+    }
+}
+
 void VideoBackendPlayer::InitVideoFrame()
 {
     if (!GPUDevice::Instance)
@@ -282,12 +309,12 @@ void VideoBackendPlayer::UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeS
         return;
 
     // Setup audio source
-    bool newSource = AudioSource == 0;
-    if (newSource)
+    if (AudioSource == 0)
     {
         // TODO: spatial video player
         // TODO: video player volume/pan control
         AudioSource = AudioBackend::Source::Add(AudioInfo, Vector3::Zero, Quaternion::Identity, 1.0f, 1.0f, 0.0f, false, false, 1.0f, 1000.0f, 1.0f);
+        IsAudioPlayPending = 1;
     }
     else
     {
@@ -320,8 +347,9 @@ void VideoBackendPlayer::UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeS
 
     // Append audio buffer
     AudioBackend::Source::QueueBuffer(AudioSource, bufferId);
-    if (newSource)
+    if (IsAudioPlayPending)
     {
+        IsAudioPlayPending = 0;
         AudioBackend::Source::Play(AudioSource);
     }
 }