GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add audio playback support to video player

Browse Source
This commit is contained in:
Wojtek Figat
2024-05-08 10:19:08 +02:00
parent 4b8970f674
commit deb2319190
3 changed files with 131 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
Source/Engine/Video/MF/VideoBackendMF.cpp
View File

@@ -162,22 +162,58 @@ namespace
bool ReadStream(VideoBackendPlayer& player, VideoPlayerMF& playerMF, DWORD streamIndex, TimeSpan dt)
{
PROFILE_CPU_NAMED("ReadStream");
ZoneText(player.DebugUrl, player.DebugUrlLen);
const bool isVideo = streamIndex == MF_SOURCE_READER_FIRST_VIDEO_STREAM;
const bool isAudio = streamIndex == MF_SOURCE_READER_FIRST_AUDIO_STREAM;
const TimeSpan lastFrameTime = isVideo ? player.VideoFrameTime : player.AudioBufferTime;
const TimeSpan lastFrameDuration = isVideo ? player.VideoFrameDuration : player.AudioBufferDuration;
// Check if the current frame is valid (eg. when playing 24fps video at 60fps)
if (lastFrameDuration.Ticks > 0 &&
Math::IsInRange(playerMF.Time, lastFrameTime, lastFrameTime + lastFrameDuration))
int32 goodSamples = 1;
TimeSpan validTimeRangeStart(0), validTimeRangeEnd(0);
if (isAudio)
{
return false;
constexpr int32 AudioFramesQueue = 10; // How many frames to read into the audio buffers queue in advance (to improve audio playback smoothness)
if (player.AudioBufferDuration.Ticks == 0)
{
// Read more samples for audio to enqueue multiple audio buffers for smoother playback
goodSamples = AudioFramesQueue;
}
else
{
// Skip reading if the last sample was already over this range (we've got enough in a queue)
validTimeRangeStart = player.AudioBufferTime - player.AudioBufferDuration * AudioFramesQueue;
validTimeRangeEnd = validTimeRangeStart + player.AudioBufferDuration;
if (Math::IsInRange(playerMF.Time, validTimeRangeStart, validTimeRangeEnd))
{
return false;
}
// Allow to read future samples within queue range
validTimeRangeStart = player.AudioBufferTime - player.AudioBufferDuration;
validTimeRangeEnd = player.AudioBufferTime + player.AudioBufferDuration * AudioFramesQueue;
// Read more samples to keep queue at capacity
TimeSpan targetQueueEnd = playerMF.Time + player.AudioBufferDuration * AudioFramesQueue;
TimeSpan activeBufferEnd = player.AudioBufferTime + player.AudioBufferDuration;
TimeSpan missingQueueDuration = targetQueueEnd - activeBufferEnd;
goodSamples = (int32)Math::DivideAndRoundUp(missingQueueDuration.Ticks, player.AudioBufferDuration.Ticks);
if (goodSamples < 1)
goodSamples = 1;
}
}
else if (isVideo)
{
// Check if the current frame is valid (eg. when playing 24fps video at 60fps)
if (player.VideoFrameDuration.Ticks > 0 &&
Math::IsInRange(playerMF.Time, player.VideoFrameTime, player.VideoFrameTime + player.VideoFrameDuration))
{
return false;
}
}
// Read samples until frame is matching the current time
int32 samplesLeft = 500;
int32 goodSamplesLeft = goodSamples;
HRESULT hr;
for (; samplesLeft > 0; samplesLeft--)
for (; samplesLeft > 0 && goodSamplesLeft > 0; samplesLeft--)
{
// Read sample
DWORD flags = 0;
@@ -199,7 +235,11 @@ namespace
franeDuration.Ticks = sampleDuration;
}
//const int32 framesToTime = (playerMF.Time.Ticks - frameTime.Ticks) / franeDuration.Ticks;
const bool isGoodSample = Math::IsInRange(playerMF.Time, frameTime, frameTime + franeDuration);
bool isGoodSample = goodSamples != goodSamplesLeft; // If we've reached good frame, then use following frames too
if (validTimeRangeStart.Ticks != 0)
isGoodSample |= Math::IsInRange(frameTime, validTimeRangeStart, validTimeRangeEnd); // Ensure frame hits the valid range
else
isGoodSample |= Math::IsInRange(playerMF.Time, frameTime, frameTime + franeDuration); // Ensure current time hits this frame range
// Process sample
if (sample && isGoodSample)
@@ -288,6 +328,8 @@ namespace
}
if (sample)
sample->Release();
if (isGoodSample)
goodSamplesLeft--;
if (flags & MF_SOURCE_READERF_ENDOFSTREAM)
{
@@ -299,10 +341,6 @@ namespace
// Format/metadata might have changed so update the stream
Configure(player, playerMF, streamIndex);
}
// End loop if got good sample or need to seek back
if (isGoodSample)
break;
}
// True if run out of samples and failed to get frame for the current time
@@ -354,6 +392,7 @@ bool VideoBackendMF::Player_Create(const VideoBackendPlayerInfo& info, VideoBack
player.Backend = this;
playerMF.Loop = info.Loop;
playerMF.FirstFrame = 1;
player.Created(info);
Players.Add(&player);
return false;
@@ -493,6 +532,9 @@ void VideoBackendMF::Base_Update()
SEEK_START:
if (playerMF.Seek)
{
// Reset cached frames timings
player.VideoFrameDuration = player.AudioBufferDuration = TimeSpan::Zero();
seeks++;
playerMF.Seek = 0;
PROPVARIANT var;

10
Source/Engine/Video/Types.h
View File

@@ -26,6 +26,11 @@ struct VideoBackendPlayer
VideoBackend* Backend;
GPUTexture* Frame;
GPUBuffer* FrameUpload;
class GPUUploadVideoFrameTask* UploadVideoFrameTask;
#ifdef TRACY_ENABLE
Char* DebugUrl;
int32 DebugUrlLen;
#endif
int32 Width, Height, AvgVideoBitRate, FramesCount;
int32 VideoFrameWidth, VideoFrameHeight;
PixelFormat Format;
@@ -35,9 +40,9 @@ struct VideoBackendPlayer
TimeSpan AudioBufferTime, AudioBufferDuration;
AudioDataInfo AudioInfo;
BytesContainer VideoFrameMemory;
uint32 AudioBuffer;
uint32 AudioSource;
class GPUUploadVideoFrameTask* UploadVideoFrameTask;
uint32 NextAudioBuffer;
uint32 AudioBuffers[30];
uintptr BackendState[8];
VideoBackendPlayer()
@@ -61,6 +66,7 @@ struct VideoBackendPlayer
return *(const T*)BackendState;
}
void Created(const VideoBackendPlayerInfo& info);
void InitVideoFrame();
void UpdateVideoFrame(Span<byte> data, TimeSpan time, TimeSpan duration);
void UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeSpan duration);

76
Source/Engine/Video/Video.cpp
View File

@@ -4,6 +4,7 @@
#include "VideoBackend.h"
#include "Engine/Audio/AudioBackend.h"
#include "Engine/Core/Log.h"
#include "Engine/Core/Math/Quaternion.h"
#include "Engine/Profiler/ProfilerCPU.h"
#include "Engine/Engine/EngineService.h"
#include "Engine/Graphics/GPUDevice.h"
@@ -50,6 +51,8 @@ protected:
GPUTexture* frame = _player->Frame;
if (!frame->IsAllocated())
return Result::MissingResources;
PROFILE_CPU();
ZoneText(_player->DebugUrl, _player->DebugUrlLen);
if (PixelFormatExtensions::IsVideo(_player->Format))
{
@@ -178,6 +181,15 @@ bool Video::CreatePlayerBackend(const VideoBackendPlayerInfo& info, VideoBackend
return true;
}
void VideoBackendPlayer::Created(const VideoBackendPlayerInfo& info)
{
#ifdef TRACY_ENABLE
DebugUrlLen = info.Url.Length();
DebugUrl = (Char*)Allocator::Allocate(DebugUrlLen * sizeof(Char) + 2);
Platform::MemoryCopy(DebugUrl, *info.Url, DebugUrlLen * 2 + 2);
#endif
}
void VideoBackendPlayer::InitVideoFrame()
{
if (!GPUDevice::Instance)
@@ -189,6 +201,7 @@ void VideoBackendPlayer::InitVideoFrame()
void VideoBackendPlayer::UpdateVideoFrame(Span<byte> data, TimeSpan time, TimeSpan duration)
{
PROFILE_CPU();
ZoneText(DebugUrl, DebugUrlLen);
VideoFrameTime = time;
VideoFrameDuration = duration;
if (!GPUDevice::Instance || GPUDevice::Instance->GetRendererType() == RendererType::Null)
@@ -238,32 +251,79 @@ void VideoBackendPlayer::UpdateVideoFrame(Span<byte> data, TimeSpan time, TimeSp
void VideoBackendPlayer::UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeSpan duration)
{
PROFILE_CPU();
ZoneText(DebugUrl, DebugUrlLen);
AudioBufferTime = time;
AudioBufferDuration = duration;
auto start = time.GetTotalMilliseconds();
auto dur = duration.GetTotalMilliseconds();
auto end = (time + duration).GetTotalMilliseconds();
if (!AudioBackend::Instance)
return;
// Setup audio source
bool newSource = AudioSource == 0;
if (newSource)
{
// TODO: spatial video player
// TODO: video player volume/pan control
AudioSource = AudioBackend::Source::Add(AudioInfo, Vector3::Zero, Quaternion::Identity, 1.0f, 1.0f, 0.0f, false, false, 1.0f, 1000.0f, 1.0f);
}
else
{
// Get the processed buffers count
int32 numProcessedBuffers = 0;
AudioBackend::Source::GetProcessedBuffersCount(AudioSource, numProcessedBuffers);
if (numProcessedBuffers > 0)
{
// Unbind processed buffers from the source
AudioBackend::Source::DequeueProcessedBuffers(AudioSource);
}
}
// Get audio buffer
uint32 bufferId = AudioBuffers[NextAudioBuffer];
if (bufferId == 0)
{
bufferId = AudioBackend::Buffer::Create();
AudioBuffers[NextAudioBuffer] = bufferId;
}
NextAudioBuffer = (NextAudioBuffer + 1) % ARRAY_COUNT(AudioBuffers);
// Update audio buffer
if (!AudioBuffer)
AudioBuffer = AudioBackend::Buffer::Create();
AudioDataInfo dataInfo = AudioInfo;
const uint32 samplesPerSecond = dataInfo.SampleRate * dataInfo.NumChannels;
const uint32 maxSamplesInData = (uint32)data.Length() * 8 / dataInfo.BitDepth;
const uint32 maxSamplesInDuration = (uint32)Math::CeilToInt(samplesPerSecond * duration.GetTotalSeconds());
dataInfo.NumSamples = Math::Min(maxSamplesInData, maxSamplesInDuration);
AudioBackend::Buffer::Write(AudioBuffer, data.Get(), dataInfo);
AudioBackend::Buffer::Write(bufferId, data.Get(), dataInfo);
// Append audio buffer
AudioBackend::Source::QueueBuffer(AudioSource, bufferId);
if (newSource)
{
AudioBackend::Source::Play(AudioSource);
}
}
void VideoBackendPlayer::ReleaseResources()
{
if (AudioBuffer)
AudioBackend::Buffer::Delete(AudioBuffer);
if (AudioSource)
{
AudioBackend::Source::Stop(AudioSource);
AudioBackend::Source::Remove(AudioSource);
AudioSource = 0;
}
for (uint32& bufferId : AudioBuffers)
{
if (bufferId)
{
AudioBackend::Buffer::Delete(bufferId);
bufferId = 0;
}
}
if (UploadVideoFrameTask)
UploadVideoFrameTask->Cancel();
VideoFrameMemory.Release();
SAFE_DELETE_GPU_RESOURCE(Frame);
SAFE_DELETE_GPU_RESOURCE(FrameUpload);
#ifdef TRACY_ENABLE
Allocator::Free(DebugUrl);
#endif
}