From 6b31d51e318e3d67b9c4ef3d638519ea23d6c448 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Fri, 10 May 2024 13:54:52 +0200 Subject: [PATCH] Add volume, pan and spatial audio options for video playback --- Source/Engine/Audio/AudioSource.h | 7 +-- Source/Engine/Video/MF/VideoBackendMF.cpp | 5 +- Source/Engine/Video/Types.h | 8 +++ Source/Engine/Video/Video.cpp | 30 ++++++++- Source/Engine/Video/VideoBackend.h | 6 ++ Source/Engine/Video/VideoPlayer.cpp | 50 +++++++++++++++ Source/Engine/Video/VideoPlayer.h | 76 ++++++++++++++++++++++- 7 files changed, 171 insertions(+), 11 deletions(-) diff --git a/Source/Engine/Audio/AudioSource.h b/Source/Engine/Audio/AudioSource.h index 682c07563..b90bb4a73 100644 --- a/Source/Engine/Audio/AudioSource.h +++ b/Source/Engine/Audio/AudioSource.h @@ -5,7 +5,6 @@ #include "Engine/Level/Actor.h" #include "Engine/Content/AssetReference.h" #include "AudioClip.h" -#include "Config.h" /// /// Represents a source for emitting audio. Audio can be played spatially (gun shot), or normally (music). Each audio source must have an AudioClip to play - back, and it can also have a position in the case of spatial (3D) audio. @@ -141,7 +140,7 @@ public: API_PROPERTY() void SetIsLooping(bool value); /// - /// Determines whether the audio clip should auto play on level start. + /// Determines whether the audio clip should autoplay on level start. /// API_PROPERTY(Attributes="EditorOrder(50), DefaultValue(false), EditorDisplay(\"Audio Source\", \"Play On Start\")") FORCE_INLINE bool GetPlayOnStart() const @@ -159,7 +158,7 @@ public: } /// - /// Determines whether the audio clip should auto play on game start. + /// Determines whether the audio clip should autoplay on game start. /// API_PROPERTY() void SetPlayOnStart(bool value); @@ -211,7 +210,7 @@ public: API_PROPERTY() void SetDopplerFactor(float value); /// - /// If checked, source can play spatial 3d audio (when audio clip supports it), otherwise will always play as 2d sound. At 0, no distance attenuation ever occurs. + /// If checked, source can play spatial 3d audio (when audio clip supports it), otherwise will always play as 2d sound. /// API_PROPERTY(Attributes="EditorOrder(80), DefaultValue(true), EditorDisplay(\"Audio Source\")") FORCE_INLINE bool GetAllowSpatialization() const diff --git a/Source/Engine/Video/MF/VideoBackendMF.cpp b/Source/Engine/Video/MF/VideoBackendMF.cpp index 264022893..1aa623502 100644 --- a/Source/Engine/Video/MF/VideoBackendMF.cpp +++ b/Source/Engine/Video/MF/VideoBackendMF.cpp @@ -432,6 +432,8 @@ namespace MF } if (player.AudioInfo.BitDepth != 0) ReadStream(player, playerMF, MF_SOURCE_READER_FIRST_AUDIO_STREAM, dt); + + player.Tick(); } } @@ -499,7 +501,8 @@ void VideoBackendMF::Player_UpdateInfo(VideoBackendPlayer& player, const VideoBa { PROFILE_CPU(); auto& playerMF = player.GetBackendState(); - playerMF.Loop = true; + playerMF.Loop = info.Loop; + player.Updated(info); } void VideoBackendMF::Player_Play(VideoBackendPlayer& player) diff --git a/Source/Engine/Video/Types.h b/Source/Engine/Video/Types.h index bbf920b5c..5113b00d8 100644 --- a/Source/Engine/Video/Types.h +++ b/Source/Engine/Video/Types.h @@ -27,6 +27,7 @@ struct VideoBackendPlayer GPUTexture* Frame; GPUBuffer* FrameUpload; class GPUUploadVideoFrameTask* UploadVideoFrameTask; + const Transform* Transform; #ifdef TRACY_ENABLE Char* DebugUrl; int32 DebugUrlLen; @@ -35,6 +36,11 @@ struct VideoBackendPlayer int32 VideoFrameWidth, VideoFrameHeight; PixelFormat Format; float FrameRate; + float AudioVolume; + float AudioPan; + float AudioMinDistance; + float AudioAttenuation; + uint8 IsAudioSpatial : 1; uint8 IsAudioPlayPending : 1; TimeSpan Duration; TimeSpan VideoFrameTime, VideoFrameDuration; @@ -68,11 +74,13 @@ struct VideoBackendPlayer } void Created(const VideoBackendPlayerInfo& info); + void Updated(const VideoBackendPlayerInfo& info); void PlayAudio(); void PauseAudio(); void StopAudio(); void InitVideoFrame(); void UpdateVideoFrame(Span data, TimeSpan time, TimeSpan duration); void UpdateAudioBuffer(Span data, TimeSpan time, TimeSpan duration); + void Tick(); void ReleaseResources(); }; diff --git a/Source/Engine/Video/Video.cpp b/Source/Engine/Video/Video.cpp index 5b14d6005..7af4caedf 100644 --- a/Source/Engine/Video/Video.cpp +++ b/Source/Engine/Video/Video.cpp @@ -5,6 +5,7 @@ #include "Engine/Audio/AudioBackend.h" #include "Engine/Core/Log.h" #include "Engine/Core/Math/Quaternion.h" +#include "Engine/Core/Math/Transform.h" #include "Engine/Profiler/ProfilerCPU.h" #include "Engine/Engine/Engine.h" #include "Engine/Engine/EngineService.h" @@ -212,6 +213,23 @@ void VideoBackendPlayer::Created(const VideoBackendPlayerInfo& info) DebugUrl = (Char*)Allocator::Allocate(DebugUrlLen * sizeof(Char) + 2); Platform::MemoryCopy(DebugUrl, *info.Url, DebugUrlLen * 2 + 2); #endif + Updated(info); +} + +void VideoBackendPlayer::Updated(const VideoBackendPlayerInfo& info) +{ + IsAudioSpatial = info.Spatial; + AudioVolume = info.Volume; + AudioPan = info.Pan; + AudioMinDistance = info.MinDistance; + AudioAttenuation = info.Attenuation; + Transform = info.Transform; + if (AudioSource) + { + AudioBackend::Source::VolumeChanged(AudioSource, AudioVolume); + AudioBackend::Source::PanChanged(AudioSource, AudioPan); + AudioBackend::Source::SpatialSetupChanged(AudioSource, IsAudioSpatial, AudioAttenuation, AudioMinDistance, 1.0f); + } } void VideoBackendPlayer::PlayAudio() @@ -311,9 +329,7 @@ void VideoBackendPlayer::UpdateAudioBuffer(Span data, TimeSpan time, TimeS // Setup audio source if (AudioSource == 0) { - // TODO: spatial video player - // TODO: video player volume/pan control - AudioSource = AudioBackend::Source::Add(AudioInfo, Vector3::Zero, Quaternion::Identity, 1.0f, 1.0f, 0.0f, false, false, 1.0f, 1000.0f, 1.0f); + AudioSource = AudioBackend::Source::Add(AudioInfo, Vector3::Zero, Quaternion::Identity, AudioVolume, 1.0f, AudioPan, false, IsAudioSpatial, AudioAttenuation, AudioMinDistance, 1.0f); IsAudioPlayPending = 1; } else @@ -354,6 +370,14 @@ void VideoBackendPlayer::UpdateAudioBuffer(Span data, TimeSpan time, TimeS } } +void VideoBackendPlayer::Tick() +{ + if (AudioSource && IsAudioSpatial && Transform) + { + AudioBackend::Source::TransformChanged(AudioSource, Transform->Translation, Transform->Orientation); + } +} + void VideoBackendPlayer::ReleaseResources() { if (AudioSource) diff --git a/Source/Engine/Video/VideoBackend.h b/Source/Engine/Video/VideoBackend.h index 9829483b6..008268c55 100644 --- a/Source/Engine/Video/VideoBackend.h +++ b/Source/Engine/Video/VideoBackend.h @@ -13,6 +13,12 @@ struct VideoBackendPlayerInfo { StringView Url; bool Loop; + bool Spatial; + float Volume; + float Pan; + float MinDistance; + float Attenuation; + const Transform* Transform; }; /// diff --git a/Source/Engine/Video/VideoPlayer.cpp b/Source/Engine/Video/VideoPlayer.cpp index baf8e617e..5bdcad7bf 100644 --- a/Source/Engine/Video/VideoPlayer.cpp +++ b/Source/Engine/Video/VideoPlayer.cpp @@ -31,6 +31,50 @@ void VideoPlayer::SetIsLooping(bool value) UpdateInfo(); } +void VideoPlayer::SetIsAudioSpatial(bool value) +{ + if (_isSpatial == value) + return; + _isSpatial = value; + UpdateInfo(); +} + +void VideoPlayer::SetAudioVolume(float value) +{ + value = Math::Saturate(value); + if (Math::NearEqual(_volume, value)) + return; + _volume = value; + UpdateInfo(); +} + +void VideoPlayer::SetAudioPan(float value) +{ + value = Math::Clamp(value, -1.0f, 1.0f); + if (Math::NearEqual(_pan, value)) + return; + _pan = value; + UpdateInfo(); +} + +void VideoPlayer::SetAudioMinDistance(float value) +{ + value = Math::Max(0.0f, value); + if (Math::NearEqual(_minDistance, value)) + return; + _minDistance = value; + UpdateInfo(); +} + +void VideoPlayer::SetAudioAttenuation(float value) +{ + value = Math::Max(0.0f, value); + if (Math::NearEqual(_attenuation, value)) + return; + _attenuation = value; + UpdateInfo(); +} + void VideoPlayer::Play() { auto state = _state; @@ -127,6 +171,12 @@ void VideoPlayer::GetInfo(VideoBackendPlayerInfo& info) const { info.Url = Url; info.Loop = _loop; + info.Spatial = _isSpatial; + info.Volume = _volume; + info.Pan = _pan; + info.MinDistance = _minDistance; + info.Attenuation = _attenuation; + info.Transform = &_transform; } void VideoPlayer::UpdateInfo() diff --git a/Source/Engine/Video/VideoPlayer.h b/Source/Engine/Video/VideoPlayer.h index 75416e36b..24ee2ebce 100644 --- a/Source/Engine/Video/VideoPlayer.h +++ b/Source/Engine/Video/VideoPlayer.h @@ -11,7 +11,6 @@ /// Video playback utility. Video content can be presented in UI (via VideoBrush), used in materials (via texture parameter bind) or used manually in shaders. /// API_CLASS(Attributes="ActorContextMenu(\"New/Visuals/Video Player\"), ActorToolbox(\"Visuals\")") - class FLAXENGINE_API VideoPlayer : public Actor { DECLARE_SCENE_OBJECT(VideoPlayer); @@ -42,7 +41,8 @@ public: private: VideoBackendPlayer _player; States _state = States::Stopped; - bool _loop = false; + bool _loop = false, _isSpatial = false; + float _volume = 1.0f, _pan = 0.0f, _minDistance = 1000.0f, _attenuation = 1.0f; public: ~VideoPlayer(); @@ -68,7 +68,7 @@ public: API_PROPERTY() void SetIsLooping(bool value); /// - /// Determines whether the video clip should auto play on level start. + /// Determines whether the video clip should autoplay on level start. /// API_FIELD(Attributes="EditorOrder(30), DefaultValue(false), EditorDisplay(\"Video Player\", \"Play On Start\")") bool PlayOnStart = false; @@ -79,6 +79,76 @@ public: API_FIELD(Attributes = "EditorOrder(35), DefaultValue(0.0f), Limit(0, float.MaxValue, 0.01f), EditorDisplay(\"Video Player\"), VisibleIf(nameof(PlayOnStart))") float StartTime = 0.0f; + /// + /// If checked, video player us using spatialization to play 3d audio, otherwise will always play as 2d sound. + /// + API_PROPERTY(Attributes="EditorOrder(50), DefaultValue(false), EditorDisplay(\"Video Player\")") + FORCE_INLINE bool GetIsAudioSpatial() const + { + return _isSpatial; + } + + /// + /// If checked, source can play spatial 3d audio (when audio clip supports it), otherwise will always play as 2d sound. At 0, no distance attenuation ever occurs. + /// + API_PROPERTY() void SetIsAudioSpatial(bool value); + + /// + /// Gets the volume of the audio played from this video, in [0, 1] range. + /// + API_PROPERTY(Attributes="EditorOrder(100), DefaultValue(1.0f), Limit(0, 1, 0.01f), EditorDisplay(\"Video Player\")") + FORCE_INLINE float GetAudioVolume() const + { + return _volume; + } + + /// + /// Sets the volume of the audio played from this video, in [0, 1] range. + /// + API_PROPERTY() void SetAudioVolume(float value); + + /// + /// Gets the stereo pan of the played audio (-1 is left speaker, 1 is right speaker, 0 is balanced). The default is 1. Used by non-spatial audio only. + /// + API_PROPERTY(Attributes="EditorOrder(110), DefaultValue(0.0f), Limit(-1.0f, 1.0f), EditorDisplay(\"Video Player\"), VisibleIf(nameof(IsAudioSpatial), true)") + FORCE_INLINE float GetAudioPan() const + { + return _pan; + } + + /// + /// Sets the stereo pan of the played audio (-1 is left speaker, 1 is right speaker, 0 is balanced). The default is 0. Used by non-spatial audio only. + /// + API_PROPERTY() void SetAudioPan(float value); + + /// + /// Gets the minimum distance at which audio attenuation starts. When the listener is closer to the video player than this value, audio is heard at full volume. Once farther away the audio starts attenuating. + /// + API_PROPERTY(Attributes="EditorOrder(120), DefaultValue(1000.0f), Limit(0, float.MaxValue, 0.1f), EditorDisplay(\"Video Player\"), VisibleIf(nameof(IsAudioSpatial))") + FORCE_INLINE float GetAudioMinDistance() const + { + return _minDistance; + } + + /// + /// Sets the minimum distance at which audio attenuation starts. When the listener is closer to the video player than this value, audio is heard at full volume. Once farther away the audio starts attenuating. + /// + API_PROPERTY() void SetAudioMinDistance(float value); + + /// + /// Gets the attenuation that controls how quickly does audio volume drop off as the listener moves further from the video player. + /// + API_PROPERTY(Attributes="EditorOrder(130), DefaultValue(1.0f), Limit(0, float.MaxValue, 0.1f), EditorDisplay(\"Video Player\"), VisibleIf(nameof(IsAudioSpatial))") + FORCE_INLINE float GetAudioAttenuation() const + { + return _attenuation; + } + + /// + /// Sets the attenuation that controls how quickly does audio volume drop off as the listener moves further from the video player. At 0, no distance attenuation ever occurs. + /// + API_PROPERTY() void SetAudioAttenuation(float value); + public: /// /// Starts playing the currently assigned video Url.