From deb2319190e1b1b857c2037212b455171e27bd23 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Wed, 8 May 2024 10:19:08 +0200 Subject: [PATCH] Add audio playback support to video player --- Source/Engine/Video/MF/VideoBackendMF.cpp | 68 ++++++++++++++++---- Source/Engine/Video/Types.h | 10 ++- Source/Engine/Video/Video.cpp | 76 ++++++++++++++++++++--- 3 files changed, 131 insertions(+), 23 deletions(-) diff --git a/Source/Engine/Video/MF/VideoBackendMF.cpp b/Source/Engine/Video/MF/VideoBackendMF.cpp index 6396c7379..6f0225041 100644 --- a/Source/Engine/Video/MF/VideoBackendMF.cpp +++ b/Source/Engine/Video/MF/VideoBackendMF.cpp @@ -162,22 +162,58 @@ namespace bool ReadStream(VideoBackendPlayer& player, VideoPlayerMF& playerMF, DWORD streamIndex, TimeSpan dt) { + PROFILE_CPU_NAMED("ReadStream"); + ZoneText(player.DebugUrl, player.DebugUrlLen); const bool isVideo = streamIndex == MF_SOURCE_READER_FIRST_VIDEO_STREAM; const bool isAudio = streamIndex == MF_SOURCE_READER_FIRST_AUDIO_STREAM; - const TimeSpan lastFrameTime = isVideo ? player.VideoFrameTime : player.AudioBufferTime; - const TimeSpan lastFrameDuration = isVideo ? player.VideoFrameDuration : player.AudioBufferDuration; - - // Check if the current frame is valid (eg. when playing 24fps video at 60fps) - if (lastFrameDuration.Ticks > 0 && - Math::IsInRange(playerMF.Time, lastFrameTime, lastFrameTime + lastFrameDuration)) + int32 goodSamples = 1; + TimeSpan validTimeRangeStart(0), validTimeRangeEnd(0); + if (isAudio) { - return false; + constexpr int32 AudioFramesQueue = 10; // How many frames to read into the audio buffers queue in advance (to improve audio playback smoothness) + if (player.AudioBufferDuration.Ticks == 0) + { + // Read more samples for audio to enqueue multiple audio buffers for smoother playback + goodSamples = AudioFramesQueue; + } + else + { + // Skip reading if the last sample was already over this range (we've got enough in a queue) + validTimeRangeStart = player.AudioBufferTime - player.AudioBufferDuration * AudioFramesQueue; + validTimeRangeEnd = validTimeRangeStart + player.AudioBufferDuration; + if (Math::IsInRange(playerMF.Time, validTimeRangeStart, validTimeRangeEnd)) + { + return false; + } + + // Allow to read future samples within queue range + validTimeRangeStart = player.AudioBufferTime - player.AudioBufferDuration; + validTimeRangeEnd = player.AudioBufferTime + player.AudioBufferDuration * AudioFramesQueue; + + // Read more samples to keep queue at capacity + TimeSpan targetQueueEnd = playerMF.Time + player.AudioBufferDuration * AudioFramesQueue; + TimeSpan activeBufferEnd = player.AudioBufferTime + player.AudioBufferDuration; + TimeSpan missingQueueDuration = targetQueueEnd - activeBufferEnd; + goodSamples = (int32)Math::DivideAndRoundUp(missingQueueDuration.Ticks, player.AudioBufferDuration.Ticks); + if (goodSamples < 1) + goodSamples = 1; + } + } + else if (isVideo) + { + // Check if the current frame is valid (eg. when playing 24fps video at 60fps) + if (player.VideoFrameDuration.Ticks > 0 && + Math::IsInRange(playerMF.Time, player.VideoFrameTime, player.VideoFrameTime + player.VideoFrameDuration)) + { + return false; + } } // Read samples until frame is matching the current time int32 samplesLeft = 500; + int32 goodSamplesLeft = goodSamples; HRESULT hr; - for (; samplesLeft > 0; samplesLeft--) + for (; samplesLeft > 0 && goodSamplesLeft > 0; samplesLeft--) { // Read sample DWORD flags = 0; @@ -199,7 +235,11 @@ namespace franeDuration.Ticks = sampleDuration; } //const int32 framesToTime = (playerMF.Time.Ticks - frameTime.Ticks) / franeDuration.Ticks; - const bool isGoodSample = Math::IsInRange(playerMF.Time, frameTime, frameTime + franeDuration); + bool isGoodSample = goodSamples != goodSamplesLeft; // If we've reached good frame, then use following frames too + if (validTimeRangeStart.Ticks != 0) + isGoodSample |= Math::IsInRange(frameTime, validTimeRangeStart, validTimeRangeEnd); // Ensure frame hits the valid range + else + isGoodSample |= Math::IsInRange(playerMF.Time, frameTime, frameTime + franeDuration); // Ensure current time hits this frame range // Process sample if (sample && isGoodSample) @@ -288,6 +328,8 @@ namespace } if (sample) sample->Release(); + if (isGoodSample) + goodSamplesLeft--; if (flags & MF_SOURCE_READERF_ENDOFSTREAM) { @@ -299,10 +341,6 @@ namespace // Format/metadata might have changed so update the stream Configure(player, playerMF, streamIndex); } - - // End loop if got good sample or need to seek back - if (isGoodSample) - break; } // True if run out of samples and failed to get frame for the current time @@ -354,6 +392,7 @@ bool VideoBackendMF::Player_Create(const VideoBackendPlayerInfo& info, VideoBack player.Backend = this; playerMF.Loop = info.Loop; playerMF.FirstFrame = 1; + player.Created(info); Players.Add(&player); return false; @@ -493,6 +532,9 @@ void VideoBackendMF::Base_Update() SEEK_START: if (playerMF.Seek) { + // Reset cached frames timings + player.VideoFrameDuration = player.AudioBufferDuration = TimeSpan::Zero(); + seeks++; playerMF.Seek = 0; PROPVARIANT var; diff --git a/Source/Engine/Video/Types.h b/Source/Engine/Video/Types.h index 40ccb8e0f..20b065493 100644 --- a/Source/Engine/Video/Types.h +++ b/Source/Engine/Video/Types.h @@ -26,6 +26,11 @@ struct VideoBackendPlayer VideoBackend* Backend; GPUTexture* Frame; GPUBuffer* FrameUpload; + class GPUUploadVideoFrameTask* UploadVideoFrameTask; +#ifdef TRACY_ENABLE + Char* DebugUrl; + int32 DebugUrlLen; +#endif int32 Width, Height, AvgVideoBitRate, FramesCount; int32 VideoFrameWidth, VideoFrameHeight; PixelFormat Format; @@ -35,9 +40,9 @@ struct VideoBackendPlayer TimeSpan AudioBufferTime, AudioBufferDuration; AudioDataInfo AudioInfo; BytesContainer VideoFrameMemory; - uint32 AudioBuffer; uint32 AudioSource; - class GPUUploadVideoFrameTask* UploadVideoFrameTask; + uint32 NextAudioBuffer; + uint32 AudioBuffers[30]; uintptr BackendState[8]; VideoBackendPlayer() @@ -61,6 +66,7 @@ struct VideoBackendPlayer return *(const T*)BackendState; } + void Created(const VideoBackendPlayerInfo& info); void InitVideoFrame(); void UpdateVideoFrame(Span data, TimeSpan time, TimeSpan duration); void UpdateAudioBuffer(Span data, TimeSpan time, TimeSpan duration); diff --git a/Source/Engine/Video/Video.cpp b/Source/Engine/Video/Video.cpp index 814713c3b..185b96739 100644 --- a/Source/Engine/Video/Video.cpp +++ b/Source/Engine/Video/Video.cpp @@ -4,6 +4,7 @@ #include "VideoBackend.h" #include "Engine/Audio/AudioBackend.h" #include "Engine/Core/Log.h" +#include "Engine/Core/Math/Quaternion.h" #include "Engine/Profiler/ProfilerCPU.h" #include "Engine/Engine/EngineService.h" #include "Engine/Graphics/GPUDevice.h" @@ -50,6 +51,8 @@ protected: GPUTexture* frame = _player->Frame; if (!frame->IsAllocated()) return Result::MissingResources; + PROFILE_CPU(); + ZoneText(_player->DebugUrl, _player->DebugUrlLen); if (PixelFormatExtensions::IsVideo(_player->Format)) { @@ -178,6 +181,15 @@ bool Video::CreatePlayerBackend(const VideoBackendPlayerInfo& info, VideoBackend return true; } +void VideoBackendPlayer::Created(const VideoBackendPlayerInfo& info) +{ +#ifdef TRACY_ENABLE + DebugUrlLen = info.Url.Length(); + DebugUrl = (Char*)Allocator::Allocate(DebugUrlLen * sizeof(Char) + 2); + Platform::MemoryCopy(DebugUrl, *info.Url, DebugUrlLen * 2 + 2); +#endif +} + void VideoBackendPlayer::InitVideoFrame() { if (!GPUDevice::Instance) @@ -189,6 +201,7 @@ void VideoBackendPlayer::InitVideoFrame() void VideoBackendPlayer::UpdateVideoFrame(Span data, TimeSpan time, TimeSpan duration) { PROFILE_CPU(); + ZoneText(DebugUrl, DebugUrlLen); VideoFrameTime = time; VideoFrameDuration = duration; if (!GPUDevice::Instance || GPUDevice::Instance->GetRendererType() == RendererType::Null) @@ -238,32 +251,79 @@ void VideoBackendPlayer::UpdateVideoFrame(Span data, TimeSpan time, TimeSp void VideoBackendPlayer::UpdateAudioBuffer(Span data, TimeSpan time, TimeSpan duration) { PROFILE_CPU(); + ZoneText(DebugUrl, DebugUrlLen); AudioBufferTime = time; AudioBufferDuration = duration; - auto start = time.GetTotalMilliseconds(); - auto dur = duration.GetTotalMilliseconds(); - auto end = (time + duration).GetTotalMilliseconds(); if (!AudioBackend::Instance) return; + // Setup audio source + bool newSource = AudioSource == 0; + if (newSource) + { + // TODO: spatial video player + // TODO: video player volume/pan control + AudioSource = AudioBackend::Source::Add(AudioInfo, Vector3::Zero, Quaternion::Identity, 1.0f, 1.0f, 0.0f, false, false, 1.0f, 1000.0f, 1.0f); + } + else + { + // Get the processed buffers count + int32 numProcessedBuffers = 0; + AudioBackend::Source::GetProcessedBuffersCount(AudioSource, numProcessedBuffers); + if (numProcessedBuffers > 0) + { + // Unbind processed buffers from the source + AudioBackend::Source::DequeueProcessedBuffers(AudioSource); + } + } + + // Get audio buffer + uint32 bufferId = AudioBuffers[NextAudioBuffer]; + if (bufferId == 0) + { + bufferId = AudioBackend::Buffer::Create(); + AudioBuffers[NextAudioBuffer] = bufferId; + } + NextAudioBuffer = (NextAudioBuffer + 1) % ARRAY_COUNT(AudioBuffers); + // Update audio buffer - if (!AudioBuffer) - AudioBuffer = AudioBackend::Buffer::Create(); AudioDataInfo dataInfo = AudioInfo; const uint32 samplesPerSecond = dataInfo.SampleRate * dataInfo.NumChannels; const uint32 maxSamplesInData = (uint32)data.Length() * 8 / dataInfo.BitDepth; const uint32 maxSamplesInDuration = (uint32)Math::CeilToInt(samplesPerSecond * duration.GetTotalSeconds()); dataInfo.NumSamples = Math::Min(maxSamplesInData, maxSamplesInDuration); - AudioBackend::Buffer::Write(AudioBuffer, data.Get(), dataInfo); + AudioBackend::Buffer::Write(bufferId, data.Get(), dataInfo); + + // Append audio buffer + AudioBackend::Source::QueueBuffer(AudioSource, bufferId); + if (newSource) + { + AudioBackend::Source::Play(AudioSource); + } } void VideoBackendPlayer::ReleaseResources() { - if (AudioBuffer) - AudioBackend::Buffer::Delete(AudioBuffer); + if (AudioSource) + { + AudioBackend::Source::Stop(AudioSource); + AudioBackend::Source::Remove(AudioSource); + AudioSource = 0; + } + for (uint32& bufferId : AudioBuffers) + { + if (bufferId) + { + AudioBackend::Buffer::Delete(bufferId); + bufferId = 0; + } + } if (UploadVideoFrameTask) UploadVideoFrameTask->Cancel(); VideoFrameMemory.Release(); SAFE_DELETE_GPU_RESOURCE(Frame); SAFE_DELETE_GPU_RESOURCE(FrameUpload); +#ifdef TRACY_ENABLE + Allocator::Free(DebugUrl); +#endif }