GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor 3D audio implementation in XAudio2 backend to match OpenAL

Browse Source 
#1612
This commit is contained in:
Wojtek Figat
2024-02-29 01:41:40 +01:00
parent 53bd576ade
commit 4df56cb506
2 changed files with 96 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
Source/Engine/Audio/AudioBackendTools.h
View File

@@ -37,30 +37,39 @@ public:
Quaternion Orientation;
};
enum class Channels
enum Channels
{
FrontLeft = 0,
FrontRight = 1,
Center = 2,
FontCenter = 2,
BackLeft = 3,
BackRight = 4,
SideLeft = 5,
SideRight = 6,
MAX
MaxChannels
};
struct SoundMix
{
float Pitch;
float Volume;
float Channels[(int32)Channels::MAX];
float Channels[MaxChannels];
void VolumeIntoChannels()
{
for (float& c : Channels)
c *= Volume;
Volume = 1.0f;
}
};
static SoundMix CalculateSoundMix(const Settings& settings, const Listener& listener, const Source& source, int32 channelCount = 2)
{
ASSERT_LOW_LAYER(channelCount <= MaxChannels);
SoundMix mix;
mix.Pitch = source.Pitch;
mix.Volume = source.Volume * settings.Volume;
Platform::MemoryClear(mix.Channels, sizeof(mix.Channels));
if (source.Is3D)
{
const Transform listenerTransform(listener.Position, listener.Orientation);
@@ -78,7 +87,7 @@ public:
// Calculate panning
// Ramy Sadek and Chris Kyriakakis, 2004, "A Novel Multichannel Panning Method for Standard and Arbitrary Loudspeaker Configurations"
// [https://www.researchgate.net/publication/235080603_A_Novel_Multichannel_Panning_Method_for_Standard_and_Arbitrary_Loudspeaker_Configurations]
static const Float3 ChannelDirections[(int32)Channels::MAX] =
static const Float3 ChannelDirections[MaxChannels] =
{
Float3(-1.0, 0.0, -1.0).GetNormalized(),
Float3(1.0, 0.0, -1.0).GetNormalized(),
@@ -117,11 +126,46 @@ public:
}
else
{
mix.Channels[(int32)Channels::FrontLeft] = Math::Min(1.0f - source.Pan, 1.0f);
mix.Channels[(int32)Channels::FrontRight] = Math::Min(1.0f + source.Pan, 1.0f);
for (int32 i = 2; i < channelCount; i++)
mix.Channels[i] = 0.0f;
const float panLeft = Math::Min(1.0f - source.Pan, 1.0f);
const float panRight = Math::Min(1.0f + source.Pan, 1.0f);
switch (channelCount)
{
case 1:
mix.Channels[0] = 1.0f;
break;
case 2:
default: // TODO: handle other channel configuration (eg. 7.1 or 5.1)
mix.Channels[FrontLeft] = panLeft;
mix.Channels[FrontRight] = panRight;
break;
}
}
return mix;
}
static void MapChannels(int32 sourceChannels, int32 outputChannels, float channels[MaxChannels], float* outputMatrix)
{
Platform::MemoryClear(outputMatrix, sizeof(float) * sourceChannels * outputChannels);
switch (outputChannels)
{
case 1:
outputMatrix[0] = channels[FrontLeft];
break;
case 2:
default: // TODO: implement multi-channel support (eg. 5.1, 7.1)
if (sourceChannels == 1)
{
outputMatrix[0] = channels[FrontLeft];
outputMatrix[1] = channels[FrontRight];
}
else if (sourceChannels == 2)
{
outputMatrix[0] = channels[FrontLeft];
outputMatrix[1] = 0.0f;
outputMatrix[2] = 0.0f;
outputMatrix[3] = channels[FrontRight];
}
break;
}
}
};

162
Source/Engine/Audio/XAudio2/AudioBackendXAudio2.cpp
View File

@@ -3,7 +3,7 @@
#if AUDIO_API_XAUDIO2
#include "AudioBackendXAudio2.h"
#include "Engine/Audio/AudioSettings.h"
#include "Engine/Audio/AudioBackendTools.h"
#include "Engine/Core/Collections/Array.h"
#include "Engine/Core/Collections/ChunkedArray.h"
#include "Engine/Core/Log.h"
@@ -22,10 +22,11 @@
// Documentation: https://docs.microsoft.com/en-us/windows/desktop/xaudio2/xaudio2-apis-portal
#include <xaudio2.h>
//#include <xaudio2fx.h>
#include <x3daudio.h>
//#include <x3daudio.h>
// TODO: implement multi-channel support (eg. 5.1, 7.1)
#define MAX_INPUT_CHANNELS 2
#define MAX_OUTPUT_CHANNELS 8
#define MAX_OUTPUT_CHANNELS 2
#define MAX_CHANNELS_MATRIX_SIZE (MAX_INPUT_CHANNELS*MAX_OUTPUT_CHANNELS)
#if ENABLE_ASSERTION
#define XAUDIO2_CHECK_ERROR(method) \
@@ -36,18 +37,12 @@
#else
#define XAUDIO2_CHECK_ERROR(method)
#endif
#define FLAX_COORD_SCALE 0.01f // units are meters
#define FLAX_DST_TO_XAUDIO(x) x * FLAX_COORD_SCALE
#define FLAX_POS_TO_XAUDIO(vec) X3DAUDIO_VECTOR(vec.X * FLAX_COORD_SCALE, vec.Y * FLAX_COORD_SCALE, vec.Z * FLAX_COORD_SCALE)
#define FLAX_VEL_TO_XAUDIO(vec) X3DAUDIO_VECTOR(vec.X * (FLAX_COORD_SCALE*FLAX_COORD_SCALE), vec.Y * (FLAX_COORD_SCALE*FLAX_COORD_SCALE), vec.Z * (FLAX_COORD_SCALE*FLAX_COORD_SCALE))
#define FLAX_VEC_TO_XAUDIO(vec) (*((X3DAUDIO_VECTOR*)&vec))
namespace XAudio2
{
struct Listener
struct Listener : AudioBackendTools::Listener
{
AudioListener* AudioListener;
X3DAUDIO_LISTENER Data;
Listener()
{
@@ -57,7 +52,6 @@ namespace XAudio2
void Init()
{
AudioListener = nullptr;
Data.pCone = nullptr;
}
bool IsFree() const
@@ -67,21 +61,13 @@ namespace XAudio2
void UpdateTransform()
{
const Vector3& position = AudioListener->GetPosition();
const Quaternion& orientation = AudioListener->GetOrientation();
const Vector3 front = orientation * Vector3::Forward;
const Vector3 top = orientation * Vector3::Up;
Data.OrientFront = FLAX_VEC_TO_XAUDIO(front);
Data.OrientTop = FLAX_VEC_TO_XAUDIO(top);
Data.Position = FLAX_POS_TO_XAUDIO(position);
Position = AudioListener->GetPosition();
Orientation = AudioListener->GetOrientation();
}
void UpdateVelocity()
{
const Vector3& velocity = AudioListener->GetVelocity();
Data.Velocity = FLAX_VEL_TO_XAUDIO(velocity);
Velocity = AudioListener->GetVelocity();
}
};
@@ -123,23 +109,18 @@ namespace XAudio2
void PeekSamples();
};
struct Source
struct Source : AudioBackendTools::Source
{
IXAudio2SourceVoice* Voice;
X3DAUDIO_EMITTER Data;
WAVEFORMATEX Format;
XAUDIO2_SEND_DESCRIPTOR Destination;
float Pitch;
float Pan;
float StartTimeForQueueBuffer;
float LastBufferStartTime;
float DopplerFactor;
uint64 LastBufferStartSamplesPlayed;
int32 BuffersProcessed;
int32 Channels;
bool IsDirty;
bool Is3D;
bool IsPlaying;
bool IsForceMono3D;
VoiceCallback Callback;
Source()
@@ -150,8 +131,6 @@ namespace XAudio2
void Init()
{
Voice = nullptr;
Platform::MemoryClear(&Data, sizeof(Data));
Data.CurveDistanceScaler = 1.0f;
Destination.Flags = 0;
Destination.pOutputVoice = nullptr;
Pitch = 1.0f;
@@ -172,21 +151,13 @@ namespace XAudio2
void UpdateTransform(const AudioSource* source)
{
const Vector3& position = source->GetPosition();
const Quaternion& orientation = source->GetOrientation();
const Vector3 front = orientation * Vector3::Forward;
const Vector3 top = orientation * Vector3::Up;
Data.OrientFront = FLAX_VEC_TO_XAUDIO(front);
Data.OrientTop = FLAX_VEC_TO_XAUDIO(top);
Data.Position = FLAX_POS_TO_XAUDIO(position);
Position = source->GetPosition();
Orientation = source->GetOrientation();
}
void UpdateVelocity(const AudioSource* source)
{
const Vector3& velocity = source->GetVelocity();
Data.Velocity = FLAX_VEL_TO_XAUDIO(velocity);
Velocity = source->GetVelocity();
}
};
@@ -214,11 +185,9 @@ namespace XAudio2
IXAudio2* Instance = nullptr;
IXAudio2MasteringVoice* MasteringVoice = nullptr;
X3DAUDIO_HANDLE X3DInstance;
DWORD ChannelMask;
UINT32 SampleRate;
UINT32 Channels;
int32 Channels;
bool ForceDirty = true;
AudioBackendTools::Settings Settings;
Listener Listeners[AUDIO_MAX_LISTENERS];
CriticalSection Locker;
ChunkedArray<Source, 32> Sources;
@@ -408,26 +377,25 @@ void AudioBackendXAudio2::Source_OnAdd(AudioSource* source)
if (FAILED(hr))
return;
sourceID++; // 0 is invalid ID so shift them
source->SourceIDs.Add(sourceID);
// Prepare source state
aSource->Callback.Source = source;
aSource->IsDirty = true;
aSource->Data.ChannelCount = format.nChannels;
aSource->Data.InnerRadius = FLAX_DST_TO_XAUDIO(source->GetMinDistance());
aSource->Is3D = source->Is3D();
aSource->IsForceMono3D = header.Is3D && header.Info.NumChannels > 1;
aSource->Pitch = source->GetPitch();
aSource->Pan = source->GetPan();
aSource->DopplerFactor = source->GetDopplerFactor();
aSource->Volume = source->GetVolume();
aSource->MinDistance = source->GetMinDistance();
aSource->Attenuation = source->GetAttenuation();
aSource->Channels = format.nChannels;
aSource->UpdateTransform(source);
aSource->UpdateVelocity(source);
hr = aSource->Voice->SetVolume(source->GetVolume());
XAUDIO2_CHECK_ERROR(SetVolume);
// 0 is invalid ID so shift them
sourceID++;
source->SourceIDs.Add(sourceID);
source->Restore();
}
@@ -462,6 +430,7 @@ void AudioBackendXAudio2::Source_VolumeChanged(AudioSource* source)
auto aSource = XAudio2::GetSource(source);
if (aSource && aSource->Voice)
{
aSource->Volume = source->GetVolume();
const HRESULT hr = aSource->Voice->SetVolume(source->GetVolume());
XAUDIO2_CHECK_ERROR(SetVolume);
}
@@ -546,17 +515,10 @@ void AudioBackendXAudio2::Source_SpatialSetupChanged(AudioSource* source)
auto aSource = XAudio2::GetSource(source);
if (aSource)
{
// TODO: implement attenuation settings for 3d audio
auto clip = source->Clip.Get();
if (clip && clip->IsLoaded())
{
const auto& header = clip->AudioHeader;
aSource->Data.ChannelCount = source->Is3D() ? 1 : header.Info.NumChannels; // 3d audio is always mono (AudioClip auto-converts before buffer write)
aSource->IsForceMono3D = header.Is3D && header.Info.NumChannels > 1;
}
aSource->Is3D = source->Is3D();
aSource->MinDistance = source->GetMinDistance();
aSource->Attenuation = source->GetAttenuation();
aSource->DopplerFactor = source->GetDopplerFactor();
aSource->Data.InnerRadius = FLAX_DST_TO_XAUDIO(source->GetMinDistance());
aSource->IsDirty = true;
}
}
@@ -655,7 +617,7 @@ float AudioBackendXAudio2::Source_GetCurrentBufferTime(const AudioSource* source
aSource->Voice->GetState(&state);
const uint32 numChannels = clipInfo.NumChannels;
const uint32 totalSamples = clipInfo.NumSamples / numChannels;
const uint32 sampleRate = clipInfo.SampleRate;// / clipInfo.NumChannels;
const uint32 sampleRate = clipInfo.SampleRate; // / clipInfo.NumChannels;
state.SamplesPlayed -= aSource->LastBufferStartSamplesPlayed % totalSamples; // Offset by the last buffer start to get time relative to its begin
time = aSource->LastBufferStartTime + (state.SamplesPlayed % totalSamples) / static_cast<float>(Math::Max(1U, sampleRate));
}
@@ -770,6 +732,8 @@ void AudioBackendXAudio2::Buffer_Delete(uint32 bufferId)
void AudioBackendXAudio2::Buffer_Write(uint32 bufferId, byte* samples, const AudioDataInfo& info)
{
CHECK(info.NumChannels <= MAX_INPUT_CHANNELS);
XAudio2::Locker.Lock();
XAudio2::Buffer* aBuffer = XAudio2::Buffers[bufferId - 1];
XAudio2::Locker.Unlock();
@@ -796,6 +760,7 @@ void AudioBackendXAudio2::Base_OnActiveDeviceChanged()
void AudioBackendXAudio2::Base_SetDopplerFactor(float value)
{
XAudio2::Settings.DopplerFactor = value;
XAudio2::MarkAllDirty();
}
@@ -803,6 +768,7 @@ void AudioBackendXAudio2::Base_SetVolume(float value)
{
if (XAudio2::MasteringVoice)
{
XAudio2::Settings.Volume = 1.0f; // Volume is applied via MasteringVoice
const HRESULT hr = XAudio2::MasteringVoice->SetVolume(value);
XAUDIO2_CHECK_ERROR(SetVolume);
}
@@ -830,7 +796,8 @@ bool AudioBackendXAudio2::Base_Init()
}
XAUDIO2_VOICE_DETAILS details;
XAudio2::MasteringVoice->GetVoiceDetails(&details);
XAudio2::SampleRate = details.InputSampleRate;
#if 0
// TODO: implement multi-channel support (eg. 5.1, 7.1)
XAudio2::Channels = details.InputChannels;
hr = XAudio2::MasteringVoice->GetChannelMask(&XAudio2::ChannelMask);
if (FAILED(hr))
@@ -838,19 +805,10 @@ bool AudioBackendXAudio2::Base_Init()
LOG(Error, "Failed to get XAudio2 mastering voice channel mask. Error: 0x{0:x}", hr);
return true;
}
// Initialize spatial audio subsystem
DWORD dwChannelMask;
XAudio2::MasteringVoice->GetChannelMask(&dwChannelMask);
hr = X3DAudioInitialize(dwChannelMask, X3DAUDIO_SPEED_OF_SOUND, XAudio2::X3DInstance);
if (FAILED(hr))
{
LOG(Error, "Failed to initalize XAudio2 3D support. Error: 0x{0:x}", hr);
return true;
}
// Info
LOG(Info, "XAudio2: {0} channels at {1} kHz (channel mask {2})", XAudio2::Channels, XAudio2::SampleRate / 1000.0f, XAudio2::ChannelMask);
#else
XAudio2::Channels = 2;
#endif
LOG(Info, "XAudio2: {0} channels at {1} kHz", XAudio2::Channels, details.InputSampleRate / 1000.0f);
// Dummy device
devices.Resize(1);
@@ -864,53 +822,19 @@ void AudioBackendXAudio2::Base_Update()
{
// Update dirty voices
const auto listener = XAudio2::GetListener();
const float dopplerFactor = AudioSettings::Get()->DopplerFactor;
float matrixCoefficients[MAX_CHANNELS_MATRIX_SIZE];
X3DAUDIO_DSP_SETTINGS dsp = { 0 };
dsp.DstChannelCount = XAudio2::Channels;
dsp.pMatrixCoefficients = matrixCoefficients;
float outputMatrix[MAX_CHANNELS_MATRIX_SIZE];
for (int32 i = 0; i < XAudio2::Sources.Count(); i++)
{
auto& source = XAudio2::Sources[i];
if (source.IsFree() || !(source.IsDirty || XAudio2::ForceDirty))
continue;
dsp.SrcChannelCount = source.Data.ChannelCount;
if (source.Is3D && listener)
{
// 3D sound
X3DAudioCalculate(XAudio2::X3DInstance, &listener->Data, &source.Data, X3DAUDIO_CALCULATE_MATRIX | X3DAUDIO_CALCULATE_DOPPLER, &dsp);
}
else
{
// 2D sound
dsp.DopplerFactor = 1.0f;
Platform::MemoryClear(dsp.pMatrixCoefficients, sizeof(matrixCoefficients));
dsp.pMatrixCoefficients[0] = 1.0f;
if (source.Format.nChannels == 1)
{
dsp.pMatrixCoefficients[1] = 1.0f;
}
else
{
dsp.pMatrixCoefficients[3] = 1.0f;
}
const float panLeft = Math::Min(1.0f - source.Pan, 1.0f);
const float panRight = Math::Min(1.0f + source.Pan, 1.0f);
if (source.Format.nChannels >= 2)
{
dsp.pMatrixCoefficients[0] *= panLeft;
dsp.pMatrixCoefficients[3] *= panRight;
}
}
if (source.IsForceMono3D)
{
// Hack to fix playback speed for 3D clip that has auto-converted stereo to mono at runtime
dsp.DopplerFactor *= 0.5f;
}
const float frequencyRatio = dopplerFactor * source.Pitch * dsp.DopplerFactor * source.DopplerFactor;
source.Voice->SetFrequencyRatio(frequencyRatio);
source.Voice->SetOutputMatrix(XAudio2::MasteringVoice, dsp.SrcChannelCount, dsp.DstChannelCount, dsp.pMatrixCoefficients);
auto mix = AudioBackendTools::CalculateSoundMix(XAudio2::Settings, *listener, source, XAudio2::Channels);
mix.VolumeIntoChannels();
AudioBackendTools::MapChannels(source.Channels, XAudio2::Channels, mix.Channels, outputMatrix);
source.Voice->SetFrequencyRatio(mix.Pitch);
source.Voice->SetOutputMatrix(XAudio2::MasteringVoice, source.Channels, XAudio2::Channels, outputMatrix);
source.IsDirty = false;
}