GoaLitiuM/FlaxEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for decoding NV12 into RGB image

Browse Source
This commit is contained in:
Wojtek Figat
2024-05-15 11:15:19 +02:00
parent 9d2dc91920
commit 82bf4238df
6 changed files with 75 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
Content/Shaders/Quad.flax (Stored with Git LFS)
View File

Binary file not shown.

13
Source/Engine/Graphics/GPUDevice.cpp
View File

@@ -297,6 +297,7 @@ struct GPUDevice::PrivateData
GPUPipelineState* PS_CopyLinear = nullptr;
GPUPipelineState* PS_Clear = nullptr;
GPUPipelineState* PS_DecodeYUY2 = nullptr;
GPUPipelineState* PS_DecodeNV12 = nullptr;
GPUBuffer* FullscreenTriangleVB = nullptr;
AssetReference<Material> DefaultMaterial;
SoftAssetReference<Material> DefaultDeformableMaterial;
@@ -715,6 +716,18 @@ GPUPipelineState* GPUDevice::GetDecodeYUY2PS() const
return _res->PS_DecodeYUY2;
}
GPUPipelineState* GPUDevice::GetDecodeNV12PS() const
{
if (_res->PS_DecodeNV12 == nullptr)
{
auto psDesc = GPUPipelineState::Description::DefaultFullscreenTriangle;
psDesc.PS = QuadShader->GetPS("PS_DecodeNV12");
_res->PS_DecodeNV12 = const_cast<GPUDevice*>(this)->CreatePipelineState();
_res->PS_DecodeNV12->Init(psDesc);
}
return _res->PS_DecodeNV12;
}
GPUBuffer* GPUDevice::GetFullscreenTriangleVB() const
{
return _res->FullscreenTriangleVB;

5
Source/Engine/Graphics/GPUDevice.h
View File

@@ -275,6 +275,11 @@ public:
/// </summary>
GPUPipelineState* GetDecodeYUY2PS() const;
/// <summary>
/// Gets the shader pipeline state object for NV12 frame decoding to RGBA.
/// </summary>
GPUPipelineState* GetDecodeNV12PS() const;
/// <summary>
/// Gets the fullscreen-triangle vertex buffer.
/// </summary>

2
Source/Engine/Video/MF/VideoBackendMF.cpp
View File

@@ -253,7 +253,7 @@ namespace MF
IMF2DBuffer* buffer2D = nullptr;
BYTE* bufferData = nullptr;
LONG bufferStride = 0;
if (isVideo && sample->GetBufferByIndex(0, &buffer) == S_OK && buffer->QueryInterface(IID_PPV_ARGS(&buffer2D)) == S_OK)
if (isVideo && player.Format != PixelFormat::NV12 && sample->GetBufferByIndex(0, &buffer) == S_OK && buffer->QueryInterface(IID_PPV_ARGS(&buffer2D)) == S_OK)
{
LONG bufferPitch = 0;
hr = buffer2D->Lock2D(&bufferData, &bufferPitch);

15
Source/Engine/Video/Video.cpp
View File

@@ -91,8 +91,19 @@ protected:
context->GPU->SetViewportAndScissors((float)_player->Width, (float)_player->Height);
context->GPU->SetRenderTarget(frame->View());
context->GPU->BindSR(0, _player->FrameUpload->View());
ASSERT_LOW_LAYER(_player->Format == PixelFormat::YUY2);
context->GPU->SetState(GPUDevice::Instance->GetDecodeYUY2PS());
GPUPipelineState* pso;
switch (_player->Format)
{
case PixelFormat::YUY2:
pso = GPUDevice::Instance->GetDecodeYUY2PS();
break;
case PixelFormat::NV12:
pso = GPUDevice::Instance->GetDecodeNV12PS();
break;
default:
return Result::Failed;
}
context->GPU->SetState(pso);
context->GPU->DrawFullscreenTriangle();
}
else

51
Source/Shaders/Quad.shader
View File

@@ -83,12 +83,21 @@ float PS_DepthCopy(Quad_VS2PS input) : SV_Depth
#endif
float4 yuv2rgb(int y, int u, int v)
{
u -= 128;
v -= 128;
float r = y + 1.402 * v;
float g = y - 0.34414 * u - 0.71414 * v;
float b = y + 1.772 * u;
return float4(r, g, b, 256.0f) / 256.0f;
}
#ifdef _PS_DecodeYUY2
// Raw memory with texture of format YUY2 and size passed in Color.xy
Buffer<uint> SourceYUY2 : register(t0);
// Pixel Shader for copying depth buffer
META_PS(true, FEATURE_LEVEL_ES2)
float4 PS_DecodeYUY2(Quad_VS2PS input) : SV_Target
{
@@ -97,17 +106,39 @@ float4 PS_DecodeYUY2(Quad_VS2PS input) : SV_Target
uint data = SourceYUY2[p / 2];
// Unpack YUY components
uint v = (data & 0xff000000) >> 24;
uint y1 = (data & 0xff0000) >> 16;
uint u = (data & 0xff00) >> 8;
uint y0 = data & 0x000000FF;
uint y = p % 2 == 0 ? y0: y1;
int v = ((data & 0xff000000) >> 24);
int y1 = (data & 0xff0000) >> 16;
int u = ((data & 0xff00) >> 8);
int y0 = data & 0xff;
int y = p % 2 == 0 ? y0: y1;
// Convert yuv to rgb
float r = (y + 1.402 * (v - 128.0));
float g = (y - 0.344 * (u - 128.0) - 0.714 * (v - 128.0));
float b = (y + 1.772 * (u - 128.0));
return float4(r, g, b, 256.0f) / 256.0f;
return yuv2rgb(y, u, v);
}
#endif
#ifdef _PS_DecodeNV12
// Raw memory with texture of format NV12 and size passed in Color.xy
Buffer<uint> SourceNV12 : register(t0);
META_PS(true, FEATURE_LEVEL_ES2)
float4 PS_DecodeNV12(Quad_VS2PS input) : SV_Target
{
// Read NV12 pixel (Y plane of size w*h, followed by interleaved UV plane is of size w*h/2)
uint size = (uint)(Color.x * Color.y);
uint p = (uint)input.Position.y * (uint)Color.x + (uint)input.Position.x;
uint y = (SourceNV12[p / 4] >> ((p % 4) * 8)) & 0xff;
p = (uint)(input.Position.y * 0.5f) * (uint)Color.x + (uint)input.Position.x;
p = (p / 2) * 2;
uint u = (SourceNV12[size / 4 + p / 4] >> ((p % 4) * 8)) & 0xff;
p = (uint)(input.Position.y * 0.5f) * (uint)Color.x + (uint)input.Position.x;
p = (p / 2) * 2 + 1;
uint v = (SourceNV12[size / 4 + p / 4] >> ((p % 4) * 8)) & 0xff;
// Convert yuv to rgb
return yuv2rgb(y, u, v);
}
#endif