GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Refactor 3D audio implementation in XAudio2 backend to match OpenAL

Browse Source 
#1612
This commit is contained in:
Wojtek Figat
2024-02-29 01:41:40 +01:00
parent 53bd576ade
commit 4df56cb506
2 changed files with 96 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
Source/Engine/Audio/AudioBackendTools.h
View File

@@ -37,30 +37,39 @@ public:
Quaternion Orientation; Quaternion Orientation;
}; };
enum class Channels enum Channels
{ {
FrontLeft = 0, FrontLeft = 0,
FrontRight = 1, FrontRight = 1,
Center = 2, FontCenter = 2,
BackLeft = 3, BackLeft = 3,
BackRight = 4, BackRight = 4,
SideLeft = 5, SideLeft = 5,
SideRight = 6, SideRight = 6,
MAX MaxChannels
}; };
struct SoundMix struct SoundMix
{ {
float Pitch; float Pitch;
float Volume; float Volume;
float Channels[(int32)Channels::MAX]; float Channels[MaxChannels];
void VolumeIntoChannels()
{
for (float& c : Channels)
c *= Volume;
Volume = 1.0f;
}
}; };
static SoundMix CalculateSoundMix(const Settings& settings, const Listener& listener, const Source& source, int32 channelCount = 2) static SoundMix CalculateSoundMix(const Settings& settings, const Listener& listener, const Source& source, int32 channelCount = 2)
{ {
ASSERT_LOW_LAYER(channelCount <= MaxChannels);
SoundMix mix; SoundMix mix;
mix.Pitch = source.Pitch; mix.Pitch = source.Pitch;
mix.Volume = source.Volume * settings.Volume; mix.Volume = source.Volume * settings.Volume;
Platform::MemoryClear(mix.Channels, sizeof(mix.Channels));
if (source.Is3D) if (source.Is3D)
{ {
const Transform listenerTransform(listener.Position, listener.Orientation); const Transform listenerTransform(listener.Position, listener.Orientation);
@@ -78,7 +87,7 @@ public:
// Calculate panning // Calculate panning
// Ramy Sadek and Chris Kyriakakis, 2004, "A Novel Multichannel Panning Method for Standard and Arbitrary Loudspeaker Configurations" // Ramy Sadek and Chris Kyriakakis, 2004, "A Novel Multichannel Panning Method for Standard and Arbitrary Loudspeaker Configurations"
// [https://www.researchgate.net/publication/235080603_A_Novel_Multichannel_Panning_Method_for_Standard_and_Arbitrary_Loudspeaker_Configurations] // [https://www.researchgate.net/publication/235080603_A_Novel_Multichannel_Panning_Method_for_Standard_and_Arbitrary_Loudspeaker_Configurations]
static const Float3 ChannelDirections[(int32)Channels::MAX] = static const Float3 ChannelDirections[MaxChannels] =
{ {
Float3(-1.0, 0.0, -1.0).GetNormalized(), Float3(-1.0, 0.0, -1.0).GetNormalized(),
Float3(1.0, 0.0, -1.0).GetNormalized(), Float3(1.0, 0.0, -1.0).GetNormalized(),
@@ -117,11 +126,46 @@ public:
} }
else else
{ {
mix.Channels[(int32)Channels::FrontLeft] = Math::Min(1.0f - source.Pan, 1.0f); const float panLeft = Math::Min(1.0f - source.Pan, 1.0f);
mix.Channels[(int32)Channels::FrontRight] = Math::Min(1.0f + source.Pan, 1.0f); const float panRight = Math::Min(1.0f + source.Pan, 1.0f);
for (int32 i = 2; i < channelCount; i++) switch (channelCount)
mix.Channels[i] = 0.0f; {
case 1:
mix.Channels[0] = 1.0f;
break;
case 2:
default: // TODO: handle other channel configuration (eg. 7.1 or 5.1)
mix.Channels[FrontLeft] = panLeft;
mix.Channels[FrontRight] = panRight;
break;
}
} }
return mix; return mix;
} }
static void MapChannels(int32 sourceChannels, int32 outputChannels, float channels[MaxChannels], float* outputMatrix)
{
Platform::MemoryClear(outputMatrix, sizeof(float) * sourceChannels * outputChannels);
switch (outputChannels)
{
case 1:
outputMatrix[0] = channels[FrontLeft];
break;
case 2:
default: // TODO: implement multi-channel support (eg. 5.1, 7.1)
if (sourceChannels == 1)
{
outputMatrix[0] = channels[FrontLeft];
outputMatrix[1] = channels[FrontRight];
}
else if (sourceChannels == 2)
{
outputMatrix[0] = channels[FrontLeft];
outputMatrix[1] = 0.0f;
outputMatrix[2] = 0.0f;
outputMatrix[3] = channels[FrontRight];
}
break;
}
}
}; };

162
Source/Engine/Audio/XAudio2/AudioBackendXAudio2.cpp
View File

@@ -3,7 +3,7 @@
#if AUDIO_API_XAUDIO2 #if AUDIO_API_XAUDIO2
#include "AudioBackendXAudio2.h" #include "AudioBackendXAudio2.h"
#include "Engine/Audio/AudioSettings.h" #include "Engine/Audio/AudioBackendTools.h"
#include "Engine/Core/Collections/Array.h" #include "Engine/Core/Collections/Array.h"
#include "Engine/Core/Collections/ChunkedArray.h" #include "Engine/Core/Collections/ChunkedArray.h"
#include "Engine/Core/Log.h" #include "Engine/Core/Log.h"
@@ -22,10 +22,11 @@
// Documentation: https://docs.microsoft.com/en-us/windows/desktop/xaudio2/xaudio2-apis-portal // Documentation: https://docs.microsoft.com/en-us/windows/desktop/xaudio2/xaudio2-apis-portal
#include <xaudio2.h> #include <xaudio2.h>
//#include <xaudio2fx.h> //#include <xaudio2fx.h>
#include <x3daudio.h> //#include <x3daudio.h>
// TODO: implement multi-channel support (eg. 5.1, 7.1)
#define MAX_INPUT_CHANNELS 2 #define MAX_INPUT_CHANNELS 2
#define MAX_OUTPUT_CHANNELS 8 #define MAX_OUTPUT_CHANNELS 2
#define MAX_CHANNELS_MATRIX_SIZE (MAX_INPUT_CHANNELS*MAX_OUTPUT_CHANNELS) #define MAX_CHANNELS_MATRIX_SIZE (MAX_INPUT_CHANNELS*MAX_OUTPUT_CHANNELS)
#if ENABLE_ASSERTION #if ENABLE_ASSERTION
#define XAUDIO2_CHECK_ERROR(method) \ #define XAUDIO2_CHECK_ERROR(method) \
@@ -36,18 +37,12 @@
#else #else
#define XAUDIO2_CHECK_ERROR(method) #define XAUDIO2_CHECK_ERROR(method)
#endif #endif
#define FLAX_COORD_SCALE 0.01f // units are meters
#define FLAX_DST_TO_XAUDIO(x) x * FLAX_COORD_SCALE
#define FLAX_POS_TO_XAUDIO(vec) X3DAUDIO_VECTOR(vec.X * FLAX_COORD_SCALE, vec.Y * FLAX_COORD_SCALE, vec.Z * FLAX_COORD_SCALE)
#define FLAX_VEL_TO_XAUDIO(vec) X3DAUDIO_VECTOR(vec.X * (FLAX_COORD_SCALE*FLAX_COORD_SCALE), vec.Y * (FLAX_COORD_SCALE*FLAX_COORD_SCALE), vec.Z * (FLAX_COORD_SCALE*FLAX_COORD_SCALE))
#define FLAX_VEC_TO_XAUDIO(vec) (*((X3DAUDIO_VECTOR*)&vec))
namespace XAudio2 namespace XAudio2
{ {
struct Listener struct Listener : AudioBackendTools::Listener
{ {
AudioListener* AudioListener; AudioListener* AudioListener;
X3DAUDIO_LISTENER Data;
Listener() Listener()
{ {
@@ -57,7 +52,6 @@ namespace XAudio2
void Init() void Init()
{ {
AudioListener = nullptr; AudioListener = nullptr;
Data.pCone = nullptr;
} }
bool IsFree() const bool IsFree() const
@@ -67,21 +61,13 @@ namespace XAudio2
void UpdateTransform() void UpdateTransform()
{ {
const Vector3& position = AudioListener->GetPosition(); Position = AudioListener->GetPosition();
const Quaternion& orientation = AudioListener->GetOrientation(); Orientation = AudioListener->GetOrientation();
const Vector3 front = orientation * Vector3::Forward;
const Vector3 top = orientation * Vector3::Up;
Data.OrientFront = FLAX_VEC_TO_XAUDIO(front);
Data.OrientTop = FLAX_VEC_TO_XAUDIO(top);
Data.Position = FLAX_POS_TO_XAUDIO(position);
} }
void UpdateVelocity() void UpdateVelocity()
{ {
const Vector3& velocity = AudioListener->GetVelocity(); Velocity = AudioListener->GetVelocity();
Data.Velocity = FLAX_VEL_TO_XAUDIO(velocity);
} }
}; };
@@ -123,23 +109,18 @@ namespace XAudio2
void PeekSamples(); void PeekSamples();
}; };
struct Source struct Source : AudioBackendTools::Source
{ {
IXAudio2SourceVoice* Voice; IXAudio2SourceVoice* Voice;
X3DAUDIO_EMITTER Data;
WAVEFORMATEX Format; WAVEFORMATEX Format;
XAUDIO2_SEND_DESCRIPTOR Destination; XAUDIO2_SEND_DESCRIPTOR Destination;
float Pitch;
float Pan;
float StartTimeForQueueBuffer; float StartTimeForQueueBuffer;
float LastBufferStartTime; float LastBufferStartTime;
float DopplerFactor;
uint64 LastBufferStartSamplesPlayed; uint64 LastBufferStartSamplesPlayed;
int32 BuffersProcessed; int32 BuffersProcessed;
int32 Channels;
bool IsDirty; bool IsDirty;
bool Is3D;
bool IsPlaying; bool IsPlaying;
bool IsForceMono3D;
VoiceCallback Callback; VoiceCallback Callback;
Source() Source()
@@ -150,8 +131,6 @@ namespace XAudio2
void Init() void Init()
{ {
Voice = nullptr; Voice = nullptr;
Platform::MemoryClear(&Data, sizeof(Data));
Data.CurveDistanceScaler = 1.0f;
Destination.Flags = 0; Destination.Flags = 0;
Destination.pOutputVoice = nullptr; Destination.pOutputVoice = nullptr;
Pitch = 1.0f; Pitch = 1.0f;
@@ -172,21 +151,13 @@ namespace XAudio2
void UpdateTransform(const AudioSource* source) void UpdateTransform(const AudioSource* source)
{ {
const Vector3& position = source->GetPosition(); Position = source->GetPosition();
const Quaternion& orientation = source->GetOrientation(); Orientation = source->GetOrientation();
const Vector3 front = orientation * Vector3::Forward;
const Vector3 top = orientation * Vector3::Up;
Data.OrientFront = FLAX_VEC_TO_XAUDIO(front);
Data.OrientTop = FLAX_VEC_TO_XAUDIO(top);
Data.Position = FLAX_POS_TO_XAUDIO(position);
} }
void UpdateVelocity(const AudioSource* source) void UpdateVelocity(const AudioSource* source)
{ {
const Vector3& velocity = source->GetVelocity(); Velocity = source->GetVelocity();
Data.Velocity = FLAX_VEL_TO_XAUDIO(velocity);
} }
}; };
@@ -214,11 +185,9 @@ namespace XAudio2
IXAudio2* Instance = nullptr; IXAudio2* Instance = nullptr;
IXAudio2MasteringVoice* MasteringVoice = nullptr; IXAudio2MasteringVoice* MasteringVoice = nullptr;
X3DAUDIO_HANDLE X3DInstance; int32 Channels;
DWORD ChannelMask;
UINT32 SampleRate;
UINT32 Channels;
bool ForceDirty = true; bool ForceDirty = true;
AudioBackendTools::Settings Settings;
Listener Listeners[AUDIO_MAX_LISTENERS]; Listener Listeners[AUDIO_MAX_LISTENERS];
CriticalSection Locker; CriticalSection Locker;
ChunkedArray<Source, 32> Sources; ChunkedArray<Source, 32> Sources;
@@ -408,26 +377,25 @@ void AudioBackendXAudio2::Source_OnAdd(AudioSource* source)
if (FAILED(hr)) if (FAILED(hr))
return; return;
sourceID++; // 0 is invalid ID so shift them
source->SourceIDs.Add(sourceID);
// Prepare source state // Prepare source state
aSource->Callback.Source = source; aSource->Callback.Source = source;
aSource->IsDirty = true; aSource->IsDirty = true;
aSource->Data.ChannelCount = format.nChannels;
aSource->Data.InnerRadius = FLAX_DST_TO_XAUDIO(source->GetMinDistance());
aSource->Is3D = source->Is3D(); aSource->Is3D = source->Is3D();
aSource->IsForceMono3D = header.Is3D && header.Info.NumChannels > 1;
aSource->Pitch = source->GetPitch(); aSource->Pitch = source->GetPitch();
aSource->Pan = source->GetPan(); aSource->Pan = source->GetPan();
aSource->DopplerFactor = source->GetDopplerFactor(); aSource->DopplerFactor = source->GetDopplerFactor();
aSource->Volume = source->GetVolume();
aSource->MinDistance = source->GetMinDistance();
aSource->Attenuation = source->GetAttenuation();
aSource->Channels = format.nChannels;
aSource->UpdateTransform(source); aSource->UpdateTransform(source);
aSource->UpdateVelocity(source); aSource->UpdateVelocity(source);
hr = aSource->Voice->SetVolume(source->GetVolume()); hr = aSource->Voice->SetVolume(source->GetVolume());
XAUDIO2_CHECK_ERROR(SetVolume); XAUDIO2_CHECK_ERROR(SetVolume);
// 0 is invalid ID so shift them
sourceID++;
source->SourceIDs.Add(sourceID);
source->Restore(); source->Restore();
} }
@@ -462,6 +430,7 @@ void AudioBackendXAudio2::Source_VolumeChanged(AudioSource* source)
auto aSource = XAudio2::GetSource(source); auto aSource = XAudio2::GetSource(source);
if (aSource && aSource->Voice) if (aSource && aSource->Voice)
{ {
aSource->Volume = source->GetVolume();
const HRESULT hr = aSource->Voice->SetVolume(source->GetVolume()); const HRESULT hr = aSource->Voice->SetVolume(source->GetVolume());
XAUDIO2_CHECK_ERROR(SetVolume); XAUDIO2_CHECK_ERROR(SetVolume);
} }
@@ -546,17 +515,10 @@ void AudioBackendXAudio2::Source_SpatialSetupChanged(AudioSource* source)
auto aSource = XAudio2::GetSource(source); auto aSource = XAudio2::GetSource(source);
if (aSource) if (aSource)
{ {
// TODO: implement attenuation settings for 3d audio
auto clip = source->Clip.Get();
if (clip && clip->IsLoaded())
{
const auto& header = clip->AudioHeader;
aSource->Data.ChannelCount = source->Is3D() ? 1 : header.Info.NumChannels; // 3d audio is always mono (AudioClip auto-converts before buffer write)
aSource->IsForceMono3D = header.Is3D && header.Info.NumChannels > 1;
}
aSource->Is3D = source->Is3D(); aSource->Is3D = source->Is3D();
aSource->MinDistance = source->GetMinDistance();
aSource->Attenuation = source->GetAttenuation();
aSource->DopplerFactor = source->GetDopplerFactor(); aSource->DopplerFactor = source->GetDopplerFactor();
aSource->Data.InnerRadius = FLAX_DST_TO_XAUDIO(source->GetMinDistance());
aSource->IsDirty = true; aSource->IsDirty = true;
} }
} }
@@ -655,7 +617,7 @@ float AudioBackendXAudio2::Source_GetCurrentBufferTime(const AudioSource* source
aSource->Voice->GetState(&state); aSource->Voice->GetState(&state);
const uint32 numChannels = clipInfo.NumChannels; const uint32 numChannels = clipInfo.NumChannels;
const uint32 totalSamples = clipInfo.NumSamples / numChannels; const uint32 totalSamples = clipInfo.NumSamples / numChannels;
const uint32 sampleRate = clipInfo.SampleRate;// / clipInfo.NumChannels; const uint32 sampleRate = clipInfo.SampleRate; // / clipInfo.NumChannels;
state.SamplesPlayed -= aSource->LastBufferStartSamplesPlayed % totalSamples; // Offset by the last buffer start to get time relative to its begin state.SamplesPlayed -= aSource->LastBufferStartSamplesPlayed % totalSamples; // Offset by the last buffer start to get time relative to its begin
time = aSource->LastBufferStartTime + (state.SamplesPlayed % totalSamples) / static_cast<float>(Math::Max(1U, sampleRate)); time = aSource->LastBufferStartTime + (state.SamplesPlayed % totalSamples) / static_cast<float>(Math::Max(1U, sampleRate));
} }
@@ -770,6 +732,8 @@ void AudioBackendXAudio2::Buffer_Delete(uint32 bufferId)
void AudioBackendXAudio2::Buffer_Write(uint32 bufferId, byte* samples, const AudioDataInfo& info) void AudioBackendXAudio2::Buffer_Write(uint32 bufferId, byte* samples, const AudioDataInfo& info)
{ {
CHECK(info.NumChannels <= MAX_INPUT_CHANNELS);
XAudio2::Locker.Lock(); XAudio2::Locker.Lock();
XAudio2::Buffer* aBuffer = XAudio2::Buffers[bufferId - 1]; XAudio2::Buffer* aBuffer = XAudio2::Buffers[bufferId - 1];
XAudio2::Locker.Unlock(); XAudio2::Locker.Unlock();
@@ -796,6 +760,7 @@ void AudioBackendXAudio2::Base_OnActiveDeviceChanged()
void AudioBackendXAudio2::Base_SetDopplerFactor(float value) void AudioBackendXAudio2::Base_SetDopplerFactor(float value)
{ {
XAudio2::Settings.DopplerFactor = value;
XAudio2::MarkAllDirty(); XAudio2::MarkAllDirty();
} }
@@ -803,6 +768,7 @@ void AudioBackendXAudio2::Base_SetVolume(float value)
{ {
if (XAudio2::MasteringVoice) if (XAudio2::MasteringVoice)
{ {
XAudio2::Settings.Volume = 1.0f; // Volume is applied via MasteringVoice
const HRESULT hr = XAudio2::MasteringVoice->SetVolume(value); const HRESULT hr = XAudio2::MasteringVoice->SetVolume(value);
XAUDIO2_CHECK_ERROR(SetVolume); XAUDIO2_CHECK_ERROR(SetVolume);
} }
@@ -830,7 +796,8 @@ bool AudioBackendXAudio2::Base_Init()
} }
XAUDIO2_VOICE_DETAILS details; XAUDIO2_VOICE_DETAILS details;
XAudio2::MasteringVoice->GetVoiceDetails(&details); XAudio2::MasteringVoice->GetVoiceDetails(&details);
XAudio2::SampleRate = details.InputSampleRate; #if 0
// TODO: implement multi-channel support (eg. 5.1, 7.1)
XAudio2::Channels = details.InputChannels; XAudio2::Channels = details.InputChannels;
hr = XAudio2::MasteringVoice->GetChannelMask(&XAudio2::ChannelMask); hr = XAudio2::MasteringVoice->GetChannelMask(&XAudio2::ChannelMask);
if (FAILED(hr)) if (FAILED(hr))
@@ -838,19 +805,10 @@ bool AudioBackendXAudio2::Base_Init()
LOG(Error, "Failed to get XAudio2 mastering voice channel mask. Error: 0x{0:x}", hr); LOG(Error, "Failed to get XAudio2 mastering voice channel mask. Error: 0x{0:x}", hr);
return true; return true;
} }
#else
// Initialize spatial audio subsystem XAudio2::Channels = 2;
DWORD dwChannelMask; #endif
XAudio2::MasteringVoice->GetChannelMask(&dwChannelMask); LOG(Info, "XAudio2: {0} channels at {1} kHz", XAudio2::Channels, details.InputSampleRate / 1000.0f);
hr = X3DAudioInitialize(dwChannelMask, X3DAUDIO_SPEED_OF_SOUND, XAudio2::X3DInstance);
if (FAILED(hr))
{
LOG(Error, "Failed to initalize XAudio2 3D support. Error: 0x{0:x}", hr);
return true;
}
// Info
LOG(Info, "XAudio2: {0} channels at {1} kHz (channel mask {2})", XAudio2::Channels, XAudio2::SampleRate / 1000.0f, XAudio2::ChannelMask);
// Dummy device // Dummy device
devices.Resize(1); devices.Resize(1);
@@ -864,53 +822,19 @@ void AudioBackendXAudio2::Base_Update()
{ {
// Update dirty voices // Update dirty voices
const auto listener = XAudio2::GetListener(); const auto listener = XAudio2::GetListener();
const float dopplerFactor = AudioSettings::Get()->DopplerFactor; float outputMatrix[MAX_CHANNELS_MATRIX_SIZE];
float matrixCoefficients[MAX_CHANNELS_MATRIX_SIZE];
X3DAUDIO_DSP_SETTINGS dsp = { 0 };
dsp.DstChannelCount = XAudio2::Channels;
dsp.pMatrixCoefficients = matrixCoefficients;
for (int32 i = 0; i < XAudio2::Sources.Count(); i++) for (int32 i = 0; i < XAudio2::Sources.Count(); i++)
{ {
auto& source = XAudio2::Sources[i]; auto& source = XAudio2::Sources[i];
if (source.IsFree() || !(source.IsDirty || XAudio2::ForceDirty)) if (source.IsFree() || !(source.IsDirty || XAudio2::ForceDirty))
continue; continue;
dsp.SrcChannelCount = source.Data.ChannelCount; auto mix = AudioBackendTools::CalculateSoundMix(XAudio2::Settings, *listener, source, XAudio2::Channels);
if (source.Is3D && listener) mix.VolumeIntoChannels();
{ AudioBackendTools::MapChannels(source.Channels, XAudio2::Channels, mix.Channels, outputMatrix);
// 3D sound
X3DAudioCalculate(XAudio2::X3DInstance, &listener->Data, &source.Data, X3DAUDIO_CALCULATE_MATRIX | X3DAUDIO_CALCULATE_DOPPLER, &dsp); source.Voice->SetFrequencyRatio(mix.Pitch);
} source.Voice->SetOutputMatrix(XAudio2::MasteringVoice, source.Channels, XAudio2::Channels, outputMatrix);
else
{
// 2D sound
dsp.DopplerFactor = 1.0f;
Platform::MemoryClear(dsp.pMatrixCoefficients, sizeof(matrixCoefficients));
dsp.pMatrixCoefficients[0] = 1.0f;
if (source.Format.nChannels == 1)
{
dsp.pMatrixCoefficients[1] = 1.0f;
}
else
{
dsp.pMatrixCoefficients[3] = 1.0f;
}
const float panLeft = Math::Min(1.0f - source.Pan, 1.0f);
const float panRight = Math::Min(1.0f + source.Pan, 1.0f);
if (source.Format.nChannels >= 2)
{
dsp.pMatrixCoefficients[0] *= panLeft;
dsp.pMatrixCoefficients[3] *= panRight;
}
}
if (source.IsForceMono3D)
{
// Hack to fix playback speed for 3D clip that has auto-converted stereo to mono at runtime
dsp.DopplerFactor *= 0.5f;
}
const float frequencyRatio = dopplerFactor * source.Pitch * dsp.DopplerFactor * source.DopplerFactor;
source.Voice->SetFrequencyRatio(frequencyRatio);
source.Voice->SetOutputMatrix(XAudio2::MasteringVoice, dsp.SrcChannelCount, dsp.DstChannelCount, dsp.pMatrixCoefficients);
source.IsDirty = false; source.IsDirty = false;
} }