GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add volume, pan and spatial audio options for video playback

Browse Source
This commit is contained in:
Wojtek Figat
2024-05-10 13:54:52 +02:00
parent f0d143ecaa
commit 6b31d51e31
7 changed files with 171 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Source/Engine/Audio/AudioSource.h
View File

@@ -5,7 +5,6 @@
#include "Engine/Level/Actor.h"
#include "Engine/Content/AssetReference.h"
#include "AudioClip.h"
#include "Config.h"
/// <summary>
/// Represents a source for emitting audio. Audio can be played spatially (gun shot), or normally (music). Each audio source must have an AudioClip to play - back, and it can also have a position in the case of spatial (3D) audio.
@@ -141,7 +140,7 @@ public:
API_PROPERTY() void SetIsLooping(bool value);
/// <summary>
/// Determines whether the audio clip should auto play on level start.
/// Determines whether the audio clip should autoplay on level start.
/// </summary>
API_PROPERTY(Attributes="EditorOrder(50), DefaultValue(false), EditorDisplay(\"Audio Source\", \"Play On Start\")")
FORCE_INLINE bool GetPlayOnStart() const
@@ -159,7 +158,7 @@ public:
}
/// <summary>
/// Determines whether the audio clip should auto play on game start.
/// Determines whether the audio clip should autoplay on game start.
/// </summary>
API_PROPERTY() void SetPlayOnStart(bool value);
@@ -211,7 +210,7 @@ public:
API_PROPERTY() void SetDopplerFactor(float value);
/// <summary>
/// If checked, source can play spatial 3d audio (when audio clip supports it), otherwise will always play as 2d sound. At 0, no distance attenuation ever occurs.
/// If checked, source can play spatial 3d audio (when audio clip supports it), otherwise will always play as 2d sound.
/// </summary>
API_PROPERTY(Attributes="EditorOrder(80), DefaultValue(true), EditorDisplay(\"Audio Source\")")
FORCE_INLINE bool GetAllowSpatialization() const

5
Source/Engine/Video/MF/VideoBackendMF.cpp
View File

@@ -432,6 +432,8 @@ namespace MF
}
if (player.AudioInfo.BitDepth != 0)
ReadStream(player, playerMF, MF_SOURCE_READER_FIRST_AUDIO_STREAM, dt);
player.Tick();
}
}
@@ -499,7 +501,8 @@ void VideoBackendMF::Player_UpdateInfo(VideoBackendPlayer& player, const VideoBa
{
PROFILE_CPU();
auto& playerMF = player.GetBackendState<VideoPlayerMF>();
playerMF.Loop = true;
playerMF.Loop = info.Loop;
player.Updated(info);
}
void VideoBackendMF::Player_Play(VideoBackendPlayer& player)

8
Source/Engine/Video/Types.h
View File

@@ -27,6 +27,7 @@ struct VideoBackendPlayer
GPUTexture* Frame;
GPUBuffer* FrameUpload;
class GPUUploadVideoFrameTask* UploadVideoFrameTask;
const Transform* Transform;
#ifdef TRACY_ENABLE
Char* DebugUrl;
int32 DebugUrlLen;
@@ -35,6 +36,11 @@ struct VideoBackendPlayer
int32 VideoFrameWidth, VideoFrameHeight;
PixelFormat Format;
float FrameRate;
float AudioVolume;
float AudioPan;
float AudioMinDistance;
float AudioAttenuation;
uint8 IsAudioSpatial : 1;
uint8 IsAudioPlayPending : 1;
TimeSpan Duration;
TimeSpan VideoFrameTime, VideoFrameDuration;
@@ -68,11 +74,13 @@ struct VideoBackendPlayer
}
void Created(const VideoBackendPlayerInfo& info);
void Updated(const VideoBackendPlayerInfo& info);
void PlayAudio();
void PauseAudio();
void StopAudio();
void InitVideoFrame();
void UpdateVideoFrame(Span<byte> data, TimeSpan time, TimeSpan duration);
void UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeSpan duration);
void Tick();
void ReleaseResources();
};

30
Source/Engine/Video/Video.cpp
View File

@@ -5,6 +5,7 @@
#include "Engine/Audio/AudioBackend.h"
#include "Engine/Core/Log.h"
#include "Engine/Core/Math/Quaternion.h"
#include "Engine/Core/Math/Transform.h"
#include "Engine/Profiler/ProfilerCPU.h"
#include "Engine/Engine/Engine.h"
#include "Engine/Engine/EngineService.h"
@@ -212,6 +213,23 @@ void VideoBackendPlayer::Created(const VideoBackendPlayerInfo& info)
DebugUrl = (Char*)Allocator::Allocate(DebugUrlLen * sizeof(Char) + 2);
Platform::MemoryCopy(DebugUrl, *info.Url, DebugUrlLen * 2 + 2);
#endif
Updated(info);
}
void VideoBackendPlayer::Updated(const VideoBackendPlayerInfo& info)
{
IsAudioSpatial = info.Spatial;
AudioVolume = info.Volume;
AudioPan = info.Pan;
AudioMinDistance = info.MinDistance;
AudioAttenuation = info.Attenuation;
Transform = info.Transform;
if (AudioSource)
{
AudioBackend::Source::VolumeChanged(AudioSource, AudioVolume);
AudioBackend::Source::PanChanged(AudioSource, AudioPan);
AudioBackend::Source::SpatialSetupChanged(AudioSource, IsAudioSpatial, AudioAttenuation, AudioMinDistance, 1.0f);
}
}
void VideoBackendPlayer::PlayAudio()
@@ -311,9 +329,7 @@ void VideoBackendPlayer::UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeS
// Setup audio source
if (AudioSource == 0)
{
// TODO: spatial video player
// TODO: video player volume/pan control
AudioSource = AudioBackend::Source::Add(AudioInfo, Vector3::Zero, Quaternion::Identity, 1.0f, 1.0f, 0.0f, false, false, 1.0f, 1000.0f, 1.0f);
AudioSource = AudioBackend::Source::Add(AudioInfo, Vector3::Zero, Quaternion::Identity, AudioVolume, 1.0f, AudioPan, false, IsAudioSpatial, AudioAttenuation, AudioMinDistance, 1.0f);
IsAudioPlayPending = 1;
}
else
@@ -354,6 +370,14 @@ void VideoBackendPlayer::UpdateAudioBuffer(Span<byte> data, TimeSpan time, TimeS
}
}
void VideoBackendPlayer::Tick()
{
if (AudioSource && IsAudioSpatial && Transform)
{
AudioBackend::Source::TransformChanged(AudioSource, Transform->Translation, Transform->Orientation);
}
}
void VideoBackendPlayer::ReleaseResources()
{
if (AudioSource)

6
Source/Engine/Video/VideoBackend.h
View File

@@ -13,6 +13,12 @@ struct VideoBackendPlayerInfo
{
StringView Url;
bool Loop;
bool Spatial;
float Volume;
float Pan;
float MinDistance;
float Attenuation;
const Transform* Transform;
};
/// <summary>

50
Source/Engine/Video/VideoPlayer.cpp
View File

@@ -31,6 +31,50 @@ void VideoPlayer::SetIsLooping(bool value)
UpdateInfo();
}
void VideoPlayer::SetIsAudioSpatial(bool value)
{
if (_isSpatial == value)
return;
_isSpatial = value;
UpdateInfo();
}
void VideoPlayer::SetAudioVolume(float value)
{
value = Math::Saturate(value);
if (Math::NearEqual(_volume, value))
return;
_volume = value;
UpdateInfo();
}
void VideoPlayer::SetAudioPan(float value)
{
value = Math::Clamp(value, -1.0f, 1.0f);
if (Math::NearEqual(_pan, value))
return;
_pan = value;
UpdateInfo();
}
void VideoPlayer::SetAudioMinDistance(float value)
{
value = Math::Max(0.0f, value);
if (Math::NearEqual(_minDistance, value))
return;
_minDistance = value;
UpdateInfo();
}
void VideoPlayer::SetAudioAttenuation(float value)
{
value = Math::Max(0.0f, value);
if (Math::NearEqual(_attenuation, value))
return;
_attenuation = value;
UpdateInfo();
}
void VideoPlayer::Play()
{
auto state = _state;
@@ -127,6 +171,12 @@ void VideoPlayer::GetInfo(VideoBackendPlayerInfo& info) const
{
info.Url = Url;
info.Loop = _loop;
info.Spatial = _isSpatial;
info.Volume = _volume;
info.Pan = _pan;
info.MinDistance = _minDistance;
info.Attenuation = _attenuation;
info.Transform = &_transform;
}
void VideoPlayer::UpdateInfo()

76
Source/Engine/Video/VideoPlayer.h
View File

@@ -11,7 +11,6 @@
/// Video playback utility. Video content can be presented in UI (via VideoBrush), used in materials (via texture parameter bind) or used manually in shaders.
/// </summary>
API_CLASS(Attributes="ActorContextMenu(\"New/Visuals/Video Player\"), ActorToolbox(\"Visuals\")")
class FLAXENGINE_API VideoPlayer : public Actor
{
DECLARE_SCENE_OBJECT(VideoPlayer);
@@ -42,7 +41,8 @@ public:
private:
VideoBackendPlayer _player;
States _state = States::Stopped;
bool _loop = false;
bool _loop = false, _isSpatial = false;
float _volume = 1.0f, _pan = 0.0f, _minDistance = 1000.0f, _attenuation = 1.0f;
public:
~VideoPlayer();
@@ -68,7 +68,7 @@ public:
API_PROPERTY() void SetIsLooping(bool value);
/// <summary>
/// Determines whether the video clip should auto play on level start.
/// Determines whether the video clip should autoplay on level start.
/// </summary>
API_FIELD(Attributes="EditorOrder(30), DefaultValue(false), EditorDisplay(\"Video Player\", \"Play On Start\")")
bool PlayOnStart = false;
@@ -79,6 +79,76 @@ public:
API_FIELD(Attributes = "EditorOrder(35), DefaultValue(0.0f), Limit(0, float.MaxValue, 0.01f), EditorDisplay(\"Video Player\"), VisibleIf(nameof(PlayOnStart))")
float StartTime = 0.0f;
/// <summary>
/// If checked, video player us using spatialization to play 3d audio, otherwise will always play as 2d sound.
/// </summary>
API_PROPERTY(Attributes="EditorOrder(50), DefaultValue(false), EditorDisplay(\"Video Player\")")
FORCE_INLINE bool GetIsAudioSpatial() const
{
return _isSpatial;
}
/// <summary>
/// If checked, source can play spatial 3d audio (when audio clip supports it), otherwise will always play as 2d sound. At 0, no distance attenuation ever occurs.
/// </summary>
API_PROPERTY() void SetIsAudioSpatial(bool value);
/// <summary>
/// Gets the volume of the audio played from this video, in [0, 1] range.
/// </summary>
API_PROPERTY(Attributes="EditorOrder(100), DefaultValue(1.0f), Limit(0, 1, 0.01f), EditorDisplay(\"Video Player\")")
FORCE_INLINE float GetAudioVolume() const
{
return _volume;
}
/// <summary>
/// Sets the volume of the audio played from this video, in [0, 1] range.
/// </summary>
API_PROPERTY() void SetAudioVolume(float value);
/// <summary>
/// Gets the stereo pan of the played audio (-1 is left speaker, 1 is right speaker, 0 is balanced). The default is 1. Used by non-spatial audio only.
/// </summary>
API_PROPERTY(Attributes="EditorOrder(110), DefaultValue(0.0f), Limit(-1.0f, 1.0f), EditorDisplay(\"Video Player\"), VisibleIf(nameof(IsAudioSpatial), true)")
FORCE_INLINE float GetAudioPan() const
{
return _pan;
}
/// <summary>
/// Sets the stereo pan of the played audio (-1 is left speaker, 1 is right speaker, 0 is balanced). The default is 0. Used by non-spatial audio only.
/// </summary>
API_PROPERTY() void SetAudioPan(float value);
/// <summary>
/// Gets the minimum distance at which audio attenuation starts. When the listener is closer to the video player than this value, audio is heard at full volume. Once farther away the audio starts attenuating.
/// </summary>
API_PROPERTY(Attributes="EditorOrder(120), DefaultValue(1000.0f), Limit(0, float.MaxValue, 0.1f), EditorDisplay(\"Video Player\"), VisibleIf(nameof(IsAudioSpatial))")
FORCE_INLINE float GetAudioMinDistance() const
{
return _minDistance;
}
/// <summary>
/// Sets the minimum distance at which audio attenuation starts. When the listener is closer to the video player than this value, audio is heard at full volume. Once farther away the audio starts attenuating.
/// </summary>
API_PROPERTY() void SetAudioMinDistance(float value);
/// <summary>
/// Gets the attenuation that controls how quickly does audio volume drop off as the listener moves further from the video player.
/// </summary>
API_PROPERTY(Attributes="EditorOrder(130), DefaultValue(1.0f), Limit(0, float.MaxValue, 0.1f), EditorDisplay(\"Video Player\"), VisibleIf(nameof(IsAudioSpatial))")
FORCE_INLINE float GetAudioAttenuation() const
{
return _attenuation;
}
/// <summary>
/// Sets the attenuation that controls how quickly does audio volume drop off as the listener moves further from the video player. At 0, no distance attenuation ever occurs.
/// </summary>
API_PROPERTY() void SetAudioAttenuation(float value);
public:
/// <summary>
/// Starts playing the currently assigned video Url.