Add audio playback support to video player

Source/Engine/Video/MF/VideoBackendMF.cpp

@@ -162,22 +162,58 @@ namespace
 
     bool ReadStream(VideoBackendPlayer& player, VideoPlayerMF& playerMF, DWORD streamIndex, TimeSpan dt)
     {
+        PROFILE_CPU_NAMED("ReadStream");
+        ZoneText(player.DebugUrl, player.DebugUrlLen);
         const bool isVideo = streamIndex == MF_SOURCE_READER_FIRST_VIDEO_STREAM;
         const bool isAudio = streamIndex == MF_SOURCE_READER_FIRST_AUDIO_STREAM;
-        const TimeSpan lastFrameTime = isVideo ? player.VideoFrameTime : player.AudioBufferTime;
+        int32 goodSamples = 1;
-        const TimeSpan lastFrameDuration = isVideo ? player.VideoFrameDuration : player.AudioBufferDuration;
+        TimeSpan validTimeRangeStart(0), validTimeRangeEnd(0);
-
+        if (isAudio)
-        // Check if the current frame is valid (eg. when playing 24fps video at 60fps)
-        if (lastFrameDuration.Ticks > 0 &&
-            Math::IsInRange(playerMF.Time, lastFrameTime, lastFrameTime + lastFrameDuration))
         {
-            return false;
+            constexpr int32 AudioFramesQueue = 10; // How many frames to read into the audio buffers queue in advance (to improve audio playback smoothness)
+            if (player.AudioBufferDuration.Ticks == 0)
+            {
+                // Read more samples for audio to enqueue multiple audio buffers for smoother playback
+                goodSamples = AudioFramesQueue;
+            }
+            else
+            {
+                // Skip reading if the last sample was already over this range (we've got enough in a queue)
+                validTimeRangeStart = player.AudioBufferTime - player.AudioBufferDuration * AudioFramesQueue;
+                validTimeRangeEnd = validTimeRangeStart + player.AudioBufferDuration;
+                if (Math::IsInRange(playerMF.Time, validTimeRangeStart, validTimeRangeEnd))
+                {
+                    return false;
+                }
+
+                // Allow to read future samples within queue range
+                validTimeRangeStart = player.AudioBufferTime - player.AudioBufferDuration;
+                validTimeRangeEnd = player.AudioBufferTime + player.AudioBufferDuration * AudioFramesQueue;
+
+                // Read more samples to keep queue at capacity
+                TimeSpan targetQueueEnd = playerMF.Time + player.AudioBufferDuration * AudioFramesQueue;
+                TimeSpan activeBufferEnd = player.AudioBufferTime + player.AudioBufferDuration;
+                TimeSpan missingQueueDuration = targetQueueEnd - activeBufferEnd;
+                goodSamples = (int32)Math::DivideAndRoundUp(missingQueueDuration.Ticks, player.AudioBufferDuration.Ticks);
+                if (goodSamples < 1)
+                    goodSamples = 1;
+            }
+        }
+        else if (isVideo)
+        {
+            // Check if the current frame is valid (eg. when playing 24fps video at 60fps)
+            if (player.VideoFrameDuration.Ticks > 0 &&
+                Math::IsInRange(playerMF.Time, player.VideoFrameTime, player.VideoFrameTime + player.VideoFrameDuration))
+            {
+                return false;
+            }
         }
 
         // Read samples until frame is matching the current time
         int32 samplesLeft = 500;
+        int32 goodSamplesLeft = goodSamples;
         HRESULT hr;
-        for (; samplesLeft > 0; samplesLeft--)
+        for (; samplesLeft > 0 && goodSamplesLeft > 0; samplesLeft--)
         {
             // Read sample
             DWORD flags = 0;
@@ -199,7 +235,11 @@ namespace
                 franeDuration.Ticks = sampleDuration;
             }
             //const int32 framesToTime = (playerMF.Time.Ticks - frameTime.Ticks) / franeDuration.Ticks;
-            const bool isGoodSample = Math::IsInRange(playerMF.Time, frameTime, frameTime + franeDuration);
+            bool isGoodSample = goodSamples != goodSamplesLeft; // If we've reached good frame, then use following frames too
+            if (validTimeRangeStart.Ticks != 0)
+                isGoodSample |= Math::IsInRange(frameTime, validTimeRangeStart, validTimeRangeEnd); // Ensure frame hits the valid range
+            else
+                isGoodSample |= Math::IsInRange(playerMF.Time, frameTime, frameTime + franeDuration); // Ensure current time hits this frame range
 
             // Process sample
             if (sample && isGoodSample)
@@ -288,6 +328,8 @@ namespace
             }
             if (sample)
                 sample->Release();
+            if (isGoodSample)
+                goodSamplesLeft--;
 
             if (flags & MF_SOURCE_READERF_ENDOFSTREAM)
             {
@@ -299,10 +341,6 @@ namespace
                 // Format/metadata might have changed so update the stream
                 Configure(player, playerMF, streamIndex);
             }
-
-            // End loop if got good sample or need to seek back
-            if (isGoodSample)
-                break;
         }
 
         // True if run out of samples and failed to get frame for the current time
@@ -354,6 +392,7 @@ bool VideoBackendMF::Player_Create(const VideoBackendPlayerInfo& info, VideoBack
     player.Backend = this;
     playerMF.Loop = info.Loop;
     playerMF.FirstFrame = 1;
+    player.Created(info);
     Players.Add(&player);
 
     return false;
@@ -493,6 +532,9 @@ void VideoBackendMF::Base_Update()
     SEEK_START:
         if (playerMF.Seek)
         {
+            // Reset cached frames timings
+            player.VideoFrameDuration = player.AudioBufferDuration = TimeSpan::Zero();
+
             seeks++;
             playerMF.Seek = 0;
             PROPVARIANT var;

Source/Engine/Video/Types.h

@@ -26,6 +26,11 @@ struct VideoBackendPlayer
     VideoBackend* Backend;
     GPUTexture* Frame;
     GPUBuffer* FrameUpload;
+    class GPUUploadVideoFrameTask* UploadVideoFrameTask;
+#ifdef TRACY_ENABLE
+    Char* DebugUrl;
+    int32 DebugUrlLen;
+#endif
     int32 Width, Height, AvgVideoBitRate, FramesCount;
     int32 VideoFrameWidth, VideoFrameHeight;
     PixelFormat Format;
@@ -35,9 +40,9 @@ struct VideoBackendPlayer
     TimeSpan AudioBufferTime, AudioBufferDuration;
     AudioDataInfo AudioInfo;
     BytesContainer VideoFrameMemory;
-    uint32 AudioBuffer;
     uint32 AudioSource;
-    class GPUUploadVideoFrameTask* UploadVideoFrameTask;
+    uint32 NextAudioBuffer;
+    uint32 AudioBuffers[30];
     uintptr BackendState[8];
 
     VideoBackendPlayer()
@@ -61,6 +66,7 @@ struct VideoBackendPlayer
         return *(const T*)BackendState;
     }
 
+    void Created(const VideoBackendPlayerInfo& info);
     void InitVideoFrame();
     void UpdateVideoFrame(Span<byte> data, TimeSpan time, TimeSpan duration);
     void UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeSpan duration);

Source/Engine/Video/Video.cpp

@@ -4,6 +4,7 @@
 #include "VideoBackend.h"
 #include "Engine/Audio/AudioBackend.h"
 #include "Engine/Core/Log.h"
+#include "Engine/Core/Math/Quaternion.h"
 #include "Engine/Profiler/ProfilerCPU.h"
 #include "Engine/Engine/EngineService.h"
 #include "Engine/Graphics/GPUDevice.h"
@@ -50,6 +51,8 @@ protected:
         GPUTexture* frame = _player->Frame;
         if (!frame->IsAllocated())
             return Result::MissingResources;
+        PROFILE_CPU();
+        ZoneText(_player->DebugUrl, _player->DebugUrlLen);
 
         if (PixelFormatExtensions::IsVideo(_player->Format))
         {
@@ -178,6 +181,15 @@ bool Video::CreatePlayerBackend(const VideoBackendPlayerInfo& info, VideoBackend
     return true;
 }
 
+void VideoBackendPlayer::Created(const VideoBackendPlayerInfo& info)
+{
+#ifdef TRACY_ENABLE
+    DebugUrlLen = info.Url.Length();
+    DebugUrl = (Char*)Allocator::Allocate(DebugUrlLen * sizeof(Char) + 2);
+    Platform::MemoryCopy(DebugUrl, *info.Url, DebugUrlLen * 2 + 2);
+#endif
+}
+
 void VideoBackendPlayer::InitVideoFrame()
 {
     if (!GPUDevice::Instance)
@@ -189,6 +201,7 @@ void VideoBackendPlayer::InitVideoFrame()
 void VideoBackendPlayer::UpdateVideoFrame(Span<byte> data, TimeSpan time, TimeSpan duration)
 {
     PROFILE_CPU();
+    ZoneText(DebugUrl, DebugUrlLen);
     VideoFrameTime = time;
     VideoFrameDuration = duration;
     if (!GPUDevice::Instance || GPUDevice::Instance->GetRendererType() == RendererType::Null)
@@ -238,32 +251,79 @@ void VideoBackendPlayer::UpdateVideoFrame(Span<byte> data, TimeSpan time, TimeSp
 void VideoBackendPlayer::UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeSpan duration)
 {
     PROFILE_CPU();
+    ZoneText(DebugUrl, DebugUrlLen);
     AudioBufferTime = time;
     AudioBufferDuration = duration;
-    auto start = time.GetTotalMilliseconds();
-    auto dur = duration.GetTotalMilliseconds();
-    auto end = (time + duration).GetTotalMilliseconds();
     if (!AudioBackend::Instance)
         return;
 
+    // Setup audio source
+    bool newSource = AudioSource == 0;
+    if (newSource)
+    {
+        // TODO: spatial video player
+        // TODO: video player volume/pan control
+        AudioSource = AudioBackend::Source::Add(AudioInfo, Vector3::Zero, Quaternion::Identity, 1.0f, 1.0f, 0.0f, false, false, 1.0f, 1000.0f, 1.0f);
+    }
+    else
+    {
+        // Get the processed buffers count
+        int32 numProcessedBuffers = 0;
+        AudioBackend::Source::GetProcessedBuffersCount(AudioSource, numProcessedBuffers);
+        if (numProcessedBuffers > 0)
+        {
+            // Unbind processed buffers from the source
+            AudioBackend::Source::DequeueProcessedBuffers(AudioSource);
+        }
+    }
+
+    // Get audio buffer
+    uint32 bufferId = AudioBuffers[NextAudioBuffer];
+    if (bufferId == 0)
+    {
+        bufferId = AudioBackend::Buffer::Create();
+        AudioBuffers[NextAudioBuffer] = bufferId;
+    }
+    NextAudioBuffer = (NextAudioBuffer + 1) % ARRAY_COUNT(AudioBuffers);
+
     // Update audio buffer
-    if (!AudioBuffer)
-        AudioBuffer = AudioBackend::Buffer::Create();
     AudioDataInfo dataInfo = AudioInfo;
     const uint32 samplesPerSecond = dataInfo.SampleRate * dataInfo.NumChannels;
     const uint32 maxSamplesInData = (uint32)data.Length() * 8 / dataInfo.BitDepth;
     const uint32 maxSamplesInDuration = (uint32)Math::CeilToInt(samplesPerSecond * duration.GetTotalSeconds());
     dataInfo.NumSamples = Math::Min(maxSamplesInData, maxSamplesInDuration);
-    AudioBackend::Buffer::Write(AudioBuffer, data.Get(), dataInfo);
+    AudioBackend::Buffer::Write(bufferId, data.Get(), dataInfo);
+
+    // Append audio buffer
+    AudioBackend::Source::QueueBuffer(AudioSource, bufferId);
+    if (newSource)
+    {
+        AudioBackend::Source::Play(AudioSource);
+    }
 }
 
 void VideoBackendPlayer::ReleaseResources()
 {
-    if (AudioBuffer)
+    if (AudioSource)
-        AudioBackend::Buffer::Delete(AudioBuffer);
+    {
+        AudioBackend::Source::Stop(AudioSource);
+        AudioBackend::Source::Remove(AudioSource);
+        AudioSource = 0;
+    }
+    for (uint32& bufferId : AudioBuffers)
+    {
+        if (bufferId)
+        {
+            AudioBackend::Buffer::Delete(bufferId);
+            bufferId = 0;
+        }
+    }
     if (UploadVideoFrameTask)
         UploadVideoFrameTask->Cancel();
     VideoFrameMemory.Release();
     SAFE_DELETE_GPU_RESOURCE(Frame);
     SAFE_DELETE_GPU_RESOURCE(FrameUpload);
+#ifdef TRACY_ENABLE
+    Allocator::Free(DebugUrl);
+#endif
 }