GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add improved GPU crashes reporting

Browse Source
This commit is contained in:
Wojtek Figat
2025-01-24 22:56:52 +01:00
parent d4c72487cd
commit d501018feb
9 changed files with 132 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Source/Engine/Core/Math/Math.h
View File

@@ -604,7 +604,7 @@ namespace Math
template<typename T>
static T AlignUpWithMask(T value, T mask)
{
return (T)(value + mask & ~mask);
return (T)((value + mask) & ~mask);
}
template<typename T>
@@ -623,7 +623,7 @@ namespace Math
static T AlignUp(T value, T alignment)
{
T mask = alignment - 1;
return (T)(value + mask & ~mask);
return (T)((value + mask) & ~mask);
}
/// <summary>
@@ -648,7 +648,7 @@ namespace Math
template<typename T>
static bool IsAligned(T value, T alignment)
{
return 0 == (value & alignment - 1);
return 0 == (value & (alignment - 1));
}
template<typename T>

32
Source/Engine/Graphics/GPUDevice.cpp
View File

@@ -9,6 +9,7 @@
#include "RenderTools.h"
#include "Graphics.h"
#include "Shaders/GPUShader.h"
#include "Shaders/GPUVertexLayout.h"
#include "Async/DefaultGPUTasksExecutor.h"
#include "Async/GPUTasksManager.h"
#include "Engine/Core/Log.h"
@@ -26,8 +27,6 @@
#include "Engine/Renderer/RenderList.h"
#include "Engine/Scripting/Enums.h"
#include "Shaders/GPUVertexLayout.h"
GPUResourcePropertyBase::~GPUResourcePropertyBase()
{
const auto e = _resource;
@@ -311,6 +310,16 @@ struct GPUDevice::PrivateData
GPUDevice* GPUDevice::Instance = nullptr;
void GPUDevice::OnRequestingExit()
{
if (Engine::FatalError != FatalErrorType::GPUCrash &&
Engine::FatalError != FatalErrorType::GPUHang &&
Engine::FatalError != FatalErrorType::GPUOutOfMemory)
return;
// TODO: get and log actual GPU memory used by the engine (API-specific)
DumpResourcesToLog();
}
GPUDevice::GPUDevice(RendererType type, ShaderProfile profile)
: ScriptingObject(SpawnParams(Guid::New(), TypeInitializer))
, _state(DeviceState::Missing)
@@ -354,6 +363,7 @@ bool GPUDevice::Init()
LOG(Info, "Total graphics memory: {0}", Utilities::BytesToText(TotalGraphicsMemory));
if (!Limits.HasCompute)
LOG(Warning, "Compute Shaders are not supported");
Engine::RequestingExit.Bind<GPUDevice, &GPUDevice::OnRequestingExit>(this);
return false;
}
@@ -450,9 +460,23 @@ void GPUDevice::DumpResourcesToLog() const
output.AppendLine();
output.AppendLine();
const bool printTypes[(int32)GPUResourceType::MAX] =
{
true, // RenderTarget
true, // Texture
true, // CubeTexture
true, // VolumeTexture
true, // Buffer
true, // Shader
false, // PipelineState
false, // Descriptor
false, // Query
false, // Sampler
};
for (int32 typeIndex = 0; typeIndex < (int32)GPUResourceType::MAX; typeIndex++)
{
const auto type = static_cast<GPUResourceType>(typeIndex);
const auto printType = printTypes[typeIndex];
output.AppendFormat(TEXT("Group: {0}s"), ScriptingEnum::ToString(type));
output.AppendLine();
@@ -462,12 +486,12 @@ void GPUDevice::DumpResourcesToLog() const
for (int32 i = 0; i < _resources.Count(); i++)
{
const GPUResource* resource = _resources[i];
if (resource->GetResourceType() == type)
if (resource->GetResourceType() == type && resource->GetMemoryUsage() != 0)
{
count++;
memUsage += resource->GetMemoryUsage();
auto str = resource->ToString();
if (str.HasChars())
if (str.HasChars() && printType)
{
output.Append(TEXT('\t'));
output.Append(str);

2
Source/Engine/Graphics/GPUDevice.h
View File

@@ -97,6 +97,8 @@ protected:
Array<GPUResource*> _resources;
CriticalSection _resourcesLock;
void OnRequestingExit();
protected:
/// <summary>
/// Initializes a new instance of the <see cref="GPUDevice"/> class.

66
Source/Engine/GraphicsDevice/DirectX/RenderToolsDX.cpp
View File

@@ -130,7 +130,7 @@ DXGI_FORMAT RenderToolsDX::ToDxgiFormat(PixelFormat format)
return PixelFormatToDXGIFormat[(int32)format];
}
const Char* RenderToolsDX::GetFeatureLevelString(const D3D_FEATURE_LEVEL featureLevel)
const Char* RenderToolsDX::GetFeatureLevelString(D3D_FEATURE_LEVEL featureLevel)
{
switch (featureLevel)
{
@@ -159,11 +159,24 @@ const Char* RenderToolsDX::GetFeatureLevelString(const D3D_FEATURE_LEVEL feature
}
}
String RenderToolsDX::GetD3DErrorString(HRESULT errorCode)
uint32 RenderToolsDX::CountAdapterOutputs(IDXGIAdapter* adapter)
{
StringBuilder sb(256);
uint32 count = 0;
while (true)
{
IDXGIOutput* output;
HRESULT hr = adapter->EnumOutputs(count, &output);
if (FAILED(hr))
{
break;
}
count++;
}
return count;
}
// Switch error code
void FormatD3DErrorString(HRESULT errorCode, StringBuilder& sb, HRESULT& removedReason)
{
#define D3DERR(x) case x: sb.Append(TEXT(#x)); break
switch (errorCode)
{
@@ -184,6 +197,8 @@ String RenderToolsDX::GetD3DErrorString(HRESULT errorCode)
// DirectX 11
D3DERR(D3D11_ERROR_FILE_NOT_FOUND);
D3DERR(D3D11_ERROR_TOO_MANY_UNIQUE_STATE_OBJECTS);
D3DERR(D3D11_ERROR_TOO_MANY_UNIQUE_VIEW_OBJECTS);
D3DERR(D3D11_ERROR_DEFERRED_CONTEXT_MAP_WITHOUT_INITIAL_DISCARD);
// DirectX 12
//D3DERR(D3D12_ERROR_FILE_NOT_FOUND);
@@ -221,22 +236,20 @@ String RenderToolsDX::GetD3DErrorString(HRESULT errorCode)
#endif
default:
{
sb.AppendFormat(TEXT("0x{0:x}"), static_cast<unsigned int>(errorCode));
}
break;
}
#undef D3DERR
if (errorCode == DXGI_ERROR_DEVICE_REMOVED || errorCode == DXGI_ERROR_DEVICE_RESET || errorCode == DXGI_ERROR_DRIVER_INTERNAL_ERROR)
{
HRESULT reason = S_OK;
const RendererType rendererType = GPUDevice::Instance ? GPUDevice::Instance->GetRendererType() : RendererType::Unknown;
void* nativePtr = GPUDevice::Instance ? GPUDevice::Instance->GetNativePtr() : nullptr;
GPUDevice* device = GPUDevice::Instance;
const RendererType rendererType = device ? device->GetRendererType() : RendererType::Unknown;
void* nativePtr = device ? device->GetNativePtr() : nullptr;
#if GRAPHICS_API_DIRECTX12
if (rendererType == RendererType::DirectX12 && nativePtr)
{
reason = ((ID3D12Device*)nativePtr)->GetDeviceRemovedReason();
removedReason = ((ID3D12Device*)nativePtr)->GetDeviceRemovedReason();
}
#endif
#if GRAPHICS_API_DIRECTX11
@@ -244,11 +257,11 @@ String RenderToolsDX::GetD3DErrorString(HRESULT errorCode)
rendererType == RendererType::DirectX10_1 ||
rendererType == RendererType::DirectX10) && nativePtr)
{
reason = ((ID3D11Device*)nativePtr)->GetDeviceRemovedReason();
removedReason = ((ID3D11Device*)nativePtr)->GetDeviceRemovedReason();
}
#endif
const Char* reasonStr = nullptr;
switch (reason)
switch (removedReason)
{
case DXGI_ERROR_DEVICE_HUNG:
reasonStr = TEXT("HUNG");
@@ -269,8 +282,35 @@ String RenderToolsDX::GetD3DErrorString(HRESULT errorCode)
if (reasonStr != nullptr)
sb.AppendFormat(TEXT(", Device Removed Reason: {0}"), reasonStr);
}
}
return sb.ToString();
void RenderToolsDX::LogD3DResult(HRESULT result, const char* file, uint32 line, bool fatal)
{
ASSERT_LOW_LAYER(FAILED(result));
// Process error and format message
StringBuilder sb;
HRESULT removedReason = S_OK;
sb.Append(TEXT("DirectX error: "));
FormatD3DErrorString(result, sb, removedReason);
if (file)
sb.Append(TEXT(" at ")).Append(file).Append(':').Append(line);
const StringView msg(sb.ToStringView());
// Handle error
FatalErrorType errorType = FatalErrorType::None;
if (result == E_OUTOFMEMORY)
errorType = FatalErrorType::GPUOutOfMemory;
else if (removedReason != S_OK)
{
errorType = FatalErrorType::GPUCrash;
if (removedReason == DXGI_ERROR_DEVICE_HUNG)
errorType = FatalErrorType::GPUHang;
}
if (errorType != FatalErrorType::None)
Platform::Fatal(msg, nullptr, errorType);
else
Log::Logger::Write(fatal ? LogType::Fatal : LogType::Error, msg);
}
LPCSTR RenderToolsDX::GetVertexInputSemantic(VertexElement::Types type, UINT& semanticIndex)

95
Source/Engine/GraphicsDevice/DirectX/RenderToolsDX.h
View File

@@ -17,12 +17,6 @@
namespace RenderToolsDX
{
#if GRAPHICS_API_DIRECTX11
/// <summary>
/// Converts Flax GPUResourceUsage to DirectX 11 resource D3D11_USAGE enum type.
/// </summary>
/// <param name="usage">The Flax resource usage.</param>
/// <returns>The DirectX 11 resource usage.</returns>
inline D3D11_USAGE ToD3D11Usage(const GPUResourceUsage usage)
{
switch (usage)
@@ -37,11 +31,6 @@ namespace RenderToolsDX
}
}
/// <summary>
/// Gets the cpu access flags from the resource usage.
/// </summary>
/// <param name="usage">The Flax resource usage.</param>
/// <returns>The DirectX 11 resource CPU usage.</returns>
inline UINT GetDX11CpuAccessFlagsFromUsage(const GPUResourceUsage usage)
{
switch (usage)
@@ -56,82 +45,26 @@ namespace RenderToolsDX
return 0;
}
}
#endif
/// <summary>
/// Converts Flax Pixel Format to the DXGI Format.
/// </summary>
/// <param name="format">The Flax Pixel Format.</param>
/// <returns>The DXGI Format</returns>
extern DXGI_FORMAT ToDxgiFormat(PixelFormat format);
// Aligns location to the next multiple of align value.
template<typename T>
T Align(T location, T align)
{
ASSERT(!((0 == align) || (align & (align - 1))));
return ((location + (align - 1)) & ~(align - 1));
}
extern const Char* GetFeatureLevelString(const D3D_FEATURE_LEVEL featureLevel);
// Calculate a subresource index for a texture
FORCE_INLINE uint32 CalcSubresourceIndex(uint32 mipSlice, uint32 arraySlice, uint32 mipLevels)
{
return mipSlice + arraySlice * mipLevels;
}
inline uint32 CountAdapterOutputs(IDXGIAdapter* adapter)
{
uint32 count = 0;
while (true)
{
IDXGIOutput* output;
HRESULT hr = adapter->EnumOutputs(count, &output);
if (FAILED(hr))
{
break;
}
count++;
}
return count;
}
extern String GetD3DErrorString(HRESULT errorCode);
inline void ValidateD3DResult(HRESULT result, const char* file = "", uint32 line = 0)
{
ASSERT(FAILED(result));
const String& errorString = GetD3DErrorString(result);
LOG(Fatal, "DirectX error: {0} at {1}:{2}", errorString, String(file), line);
}
inline void LogD3DResult(HRESULT result, const char* file = "", uint32 line = 0)
{
ASSERT(FAILED(result));
const String& errorString = GetD3DErrorString(result);
LOG(Error, "DirectX error: {0} at {1}:{2}", errorString, String(file), line);
}
DXGI_FORMAT ToDxgiFormat(PixelFormat format);
const Char* GetFeatureLevelString(D3D_FEATURE_LEVEL featureLevel);
uint32 CountAdapterOutputs(IDXGIAdapter* adapter);
void LogD3DResult(HRESULT result, const char* file = nullptr, uint32 line = 0, bool fatal = false);
LPCSTR GetVertexInputSemantic(VertexElement::Types type, UINT& semanticIndex);
};
#if GPU_ENABLE_ASSERTION
// DirectX results validation
#define VALIDATE_DIRECTX_CALL(x) { HRESULT result = x; if (FAILED(result)) RenderToolsDX::ValidateD3DResult(result, __FILE__, __LINE__); }
#define VALIDATE_DIRECTX_CALL(x) { HRESULT result = x; if (FAILED(result)) RenderToolsDX::LogD3DResult(result, __FILE__, __LINE__, true); }
#define LOG_DIRECTX_RESULT(result) if (FAILED(result)) RenderToolsDX::LogD3DResult(result, __FILE__, __LINE__)
#define LOG_DIRECTX_RESULT_WITH_RETURN(result, returnValue) if (FAILED(result)) { RenderToolsDX::LogD3DResult(result, __FILE__, __LINE__); return returnValue; }
#else
#define VALIDATE_DIRECTX_CALL(x) x
#define LOG_DIRECTX_RESULT(result) if(FAILED(result)) RenderToolsDX::LogD3DResult(result)
#define LOG_DIRECTX_RESULT_WITH_RETURN(result, returnValue) if(FAILED(result)) { RenderToolsDX::LogD3DResult(result); return returnValue; }
#endif
#if GPU_ENABLE_DIAGNOSTICS || COMPILE_WITH_SHADER_COMPILER || GPU_ENABLE_RESOURCE_NAMING
#include "Engine/Utilities/StringConverter.h"
@@ -165,15 +98,15 @@ inline void SetDebugObjectName(IDXGIObject* resource, const char (&name)[NameLen
#if GRAPHICS_API_DIRECTX11
template<uint32 NameLength>
inline void SetDebugObjectName(ID3D10DeviceChild* resource, const char(&name)[NameLength])
inline void SetDebugObjectName(ID3D10DeviceChild* resource, const char (&name)[NameLength])
{
SetDebugObjectName(resource, name, NameLength - 1);
SetDebugObjectName(resource, name, NameLength - 1);
}
template<uint32 NameLength>
inline void SetDebugObjectName(ID3D11DeviceChild* resource, const char(&name)[NameLength])
inline void SetDebugObjectName(ID3D11DeviceChild* resource, const char (&name)[NameLength])
{
SetDebugObjectName(resource, name, NameLength - 1);
SetDebugObjectName(resource, name, NameLength - 1);
}
#endif
@@ -196,15 +129,15 @@ inline void SetDebugObjectName(ID3D12DeviceChild* resource, const char (&name)[N
#if GRAPHICS_API_DIRECTX11
template<uint32 NameLength>
inline void SetDebugObjectName(ID3D10Resource* resource, const char(&name)[NameLength])
inline void SetDebugObjectName(ID3D10Resource* resource, const char (&name)[NameLength])
{
resource->SetPrivateData(WKPDID_D3DDebugObjectName, NameLength - 1, name);
resource->SetPrivateData(WKPDID_D3DDebugObjectName, NameLength - 1, name);
}
template<uint32 NameLength>
inline void SetDebugObjectName(ID3D11Resource* resource, const char(&name)[NameLength])
inline void SetDebugObjectName(ID3D11Resource* resource, const char (&name)[NameLength])
{
resource->SetPrivateData(WKPDID_D3DDebugObjectName, NameLength - 1, name);
resource->SetPrivateData(WKPDID_D3DDebugObjectName, NameLength - 1, name);
}
#endif
@@ -241,7 +174,7 @@ inline void SetDebugObjectName(T* resource, const Char* data, UINT size)
#else
char* ansi = (char*)Allocator::Allocate(size + 1);
StringUtils::ConvertUTF162ANSI(data, ansi, size);
ansi[size] ='\0';
ansi[size] = '\0';
SetDebugObjectName(resource, ansi, size);
Allocator::Free(ansi);
#endif

45
Source/Engine/GraphicsDevice/Vulkan/RenderToolsVulkan.cpp
View File

@@ -180,7 +180,7 @@ void RenderToolsVulkan::SetObjectName(VkDevice device, uint64 objectHandle, VkOb
String RenderToolsVulkan::GetVkErrorString(VkResult result)
{
StringBuilder sb(256);
StringBuilder sb(64);
// Switch error code
switch (result)
@@ -228,33 +228,30 @@ String RenderToolsVulkan::GetVkErrorString(VkResult result)
return sb.ToString();
}
void RenderToolsVulkan::ValidateVkResult(VkResult result, const char* file, uint32 line)
void RenderToolsVulkan::LogVkResult(VkResult result, const char* file, uint32 line, bool fatal)
{
// Ensure result if invalid
ASSERT(result != VK_SUCCESS);
// Get error string
const String& errorString = GetVkErrorString(result);
// Process error and format message
StringBuilder sb;
sb.Append(TEXT("Vulkan error: "));
sb.Append(GetVkErrorString(result));
if (file)
sb.Append(TEXT(" at ")).Append(file).Append(':').Append(line);
const StringView msg(sb.ToStringView());
// Send error
LOG(Fatal, "Vulkan error: {0} at {1}:{2}", errorString, String(file), line);
}
void RenderToolsVulkan::LogVkResult(VkResult result, const char* file, uint32 line)
{
// Ensure result if invalid
ASSERT(result != VK_SUCCESS);
// Get error string
const String& errorString = GetVkErrorString(result);
// Send error
LOG(Error, "Vulkan error: {0} at {1}:{2}", errorString, String(file), line);
}
void RenderToolsVulkan::LogVkResult(VkResult result)
{
LogVkResult(result, "", 0);
// Handle error
FatalErrorType errorType = FatalErrorType::None;
if (result == VK_ERROR_OUT_OF_HOST_MEMORY || result == VK_ERROR_OUT_OF_DEVICE_MEMORY || result == VK_ERROR_OUT_OF_POOL_MEMORY)
errorType = FatalErrorType::GPUOutOfMemory;
else if (result == VK_TIMEOUT)
errorType = FatalErrorType::GPUHang;
else if (result == VK_ERROR_DEVICE_LOST || result == VK_ERROR_SURFACE_LOST_KHR || result == VK_ERROR_MEMORY_MAP_FAILED)
errorType = FatalErrorType::GPUCrash;
if (errorType != FatalErrorType::None)
Platform::Fatal(msg, nullptr, errorType);
else
Log::Logger::Write(fatal ? LogType::Fatal : LogType::Error, msg);
}
bool RenderToolsVulkan::HasExtension(const Array<const char*>& extensions, const char* name)

16
Source/Engine/GraphicsDevice/Vulkan/RenderToolsVulkan.h
View File

@@ -11,21 +11,13 @@
#if GRAPHICS_API_VULKAN
#if GPU_ENABLE_ASSERTION
// Vulkan results validation
#define VALIDATE_VULKAN_RESULT(x) { VkResult result = x; if (result != VK_SUCCESS) RenderToolsVulkan::ValidateVkResult(result, __FILE__, __LINE__); }
#define VALIDATE_VULKAN_RESULT(x) { VkResult result = x; if (result != VK_SUCCESS) RenderToolsVulkan::LogVkResult(result, __FILE__, __LINE__, true); }
#define LOG_VULKAN_RESULT(result) if (result != VK_SUCCESS) RenderToolsVulkan::LogVkResult(result, __FILE__, __LINE__)
#define LOG_VULKAN_RESULT_WITH_RETURN(result) if (result != VK_SUCCESS) { RenderToolsVulkan::LogVkResult(result, __FILE__, __LINE__); return true; }
#if GPU_ENABLE_ASSERTION
#define VK_SET_DEBUG_NAME(device, handle, type, name) RenderToolsVulkan::SetObjectName(device->Device, (uint64)handle, type, name)
#else
#define VALIDATE_VULKAN_RESULT(x) x
#define LOG_VULKAN_RESULT(result) if (result != VK_SUCCESS) RenderToolsVulkan::LogVkResult(result)
#define LOG_VULKAN_RESULT_WITH_RETURN(result) if (result != VK_SUCCESS) { RenderToolsVulkan::LogVkResult(result); return true; }
#define VK_SET_DEBUG_NAME(device, handle, type, name)
#endif
/// <summary>
@@ -46,9 +38,7 @@ public:
#endif
static String GetVkErrorString(VkResult result);
static void ValidateVkResult(VkResult result, const char* file, uint32 line);
static void LogVkResult(VkResult result, const char* file, uint32 line);
static void LogVkResult(VkResult result);
static void LogVkResult(VkResult result, const char* file = nullptr, uint32 line = 0, bool fatal = false);
static inline VkPipelineStageFlags GetBufferBarrierFlags(VkAccessFlags accessFlags)
{

4
Source/Engine/Platform/Base/PlatformBase.cpp
View File

@@ -348,8 +348,8 @@ void PlatformBase::Fatal(const StringView& msg, void* context, FatalErrorType er
LOG(Error, "Used Physical Memory: {0} ({1}%)", Utilities::BytesToText(memoryStats.UsedPhysicalMemory), (int32)(100 * memoryStats.UsedPhysicalMemory / memoryStats.TotalPhysicalMemory));
LOG(Error, "Used Virtual Memory: {0} ({1}%)", Utilities::BytesToText(memoryStats.UsedVirtualMemory), (int32)(100 * memoryStats.UsedVirtualMemory / memoryStats.TotalVirtualMemory));
const ProcessMemoryStats processMemoryStats = Platform::GetProcessMemoryStats();
LOG(Error, "Process Used Physical Memory: {0}", Utilities::BytesToText(processMemoryStats.UsedPhysicalMemory));
LOG(Error, "Process Used Virtual Memory: {0}", Utilities::BytesToText(processMemoryStats.UsedVirtualMemory));
LOG(Error, "Process Used Physical Memory: {0} ({1}%)", Utilities::BytesToText(processMemoryStats.UsedPhysicalMemory), (int32)(100 * processMemoryStats.UsedPhysicalMemory / memoryStats.TotalPhysicalMemory));
LOG(Error, "Process Used Virtual Memory: {0} ({1}%)", Utilities::BytesToText(processMemoryStats.UsedVirtualMemory), (int32)(100 * processMemoryStats.UsedVirtualMemory / memoryStats.TotalVirtualMemory));
}
}
if (Log::Logger::LogFilePath.HasChars())

6
Source/Engine/Platform/Base/PlatformBase.h
View File

@@ -140,6 +140,12 @@ API_ENUM() enum class FatalErrorType
Assertion,
// Program run out of memory to allocate.
OutOfMemory,
// The graphics device crashed, has been removed or restarted.
GPUCrash,
// The graphics device stopped responding (eg. incorrect rendering code or bug in driver).
GPUHang,
// The graphics device run out of video memory to allocate.
GPUOutOfMemory,
};
API_INJECT_CODE(cpp, "#include \"Engine/Platform/Platform.h\"");