// Copyright (c) 2012-2024 Wojciech Figat. All rights reserved. #include "Video.h" #include "VideoBackend.h" #include "Engine/Core/Log.h" #include "Engine/Profiler/ProfilerCPU.h" #include "Engine/Engine/EngineService.h" #include "Engine/Graphics/GPUDevice.h" #include "Engine/Graphics/GPUBuffer.h" #include "Engine/Graphics/GPUResource.h" #include "Engine/Graphics/GPUPipelineState.h" #include "Engine/Graphics/PixelFormatExtensions.h" #include "Engine/Graphics/RenderTools.h" #include "Engine/Graphics/Async/GPUTask.h" #include "Engine/Graphics/Shaders/GPUShader.h" #include "Engine/Graphics/Textures/GPUTexture.h" #include "Engine/Scripting/Enums.h" #if VIDEO_API_MF #include "MF/VideoBackendMF.h" #endif /// /// Video frame upload task to the GPU. /// class GPUUploadVideoFrameTask : public GPUTask { private: VideoBackendPlayer* _player; public: GPUUploadVideoFrameTask(VideoBackendPlayer* player) : GPUTask(Type::UploadTexture, 0) , _player(player) { } public: // [GPUTask] bool HasReference(Object* resource) const override { return _player && _player->Frame == resource; } protected: // [GPUTask] Result run(GPUTasksContext* context) override { if (!_player || _player->VideoFrameMemory.IsInvalid()) return Result::MissingResources; GPUTexture* frame = _player->Frame; if (!frame->IsAllocated()) return Result::MissingResources; if (PixelFormatExtensions::IsVideo(_player->Format)) { // Allocate compressed frame uploading texture if (!_player->FrameUpload) _player->FrameUpload = GPUDevice::Instance->CreateBuffer(TEXT("VideoFrameUpload")); auto desc = GPUBufferDescription::Buffer(_player->VideoFrameMemory.Length(), GPUBufferFlags::ShaderResource, PixelFormat::R32_UInt, nullptr, 4, GPUResourceUsage::Dynamic); // TODO: add support for Transient textures (single frame data upload) if (_player->FrameUpload->GetDescription() != desc) { if (_player->FrameUpload->Init(desc)) return Result::Failed; } // Upload compressed texture data context->GPU->UpdateBuffer(_player->FrameUpload, _player->VideoFrameMemory.Get(), _player->VideoFrameMemory.Length()); // Decompress data into RGBA texture auto cb = GPUDevice::Instance->QuadShader->GetCB(0); QuadShaderData cbData; cbData.Color = Float4((float)_player->Width, (float)_player->Height, 0, 0); context->GPU->UpdateCB(cb, &cbData); context->GPU->BindCB(0, cb); context->GPU->SetViewportAndScissors((float)_player->Width, (float)_player->Height); context->GPU->SetRenderTarget(frame->View()); context->GPU->BindSR(0, _player->FrameUpload->View()); ASSERT_LOW_LAYER(_player->Format == PixelFormat::YUY2); context->GPU->SetState(GPUDevice::Instance->GetDecodeYUY2PS()); context->GPU->DrawFullscreenTriangle(); } else { // Raw texture data upload uint32 rowPitch, slicePitch; frame->ComputePitch(0, rowPitch, slicePitch); context->GPU->UpdateTexture(frame, 0, 0, _player->VideoFrameMemory.Get(), rowPitch, slicePitch); } // Frame has been updated _player->FramesCount++; return Result::Ok; } void OnEnd() override { // Unlink if (_player && _player->UploadVideoFrameTask == this) _player->UploadVideoFrameTask = nullptr; _player = nullptr; GPUTask::OnEnd(); } }; class VideoService : public EngineService { public: VideoService() : EngineService(TEXT("Video"), -40) { } VideoBackend* Backends[4] = {}; void InitBackend(int32 index, VideoBackend* backend) { LOG(Info, "Video initialization... (backend: {0})", backend->Base_Name()); if (backend->Base_Init()) { LOG(Warning, "Failed to initialize Video backend."); } Backends[index] = backend; } void Update() override; void Dispose() override; }; VideoService VideoServiceInstance; void VideoService::Update() { PROFILE_CPU_NAMED("Video.Update"); // Update backends for (VideoBackend*& backend : VideoServiceInstance.Backends) { if (backend) backend->Base_Update(); } } void VideoService::Dispose() { PROFILE_CPU_NAMED("Video.Dispose"); // Dispose backends for (VideoBackend*& backend : VideoServiceInstance.Backends) { if (backend) { delete backend; backend = nullptr; } } } bool Video::CreatePlayerBackend(const VideoBackendPlayerInfo& info, VideoBackendPlayer& player) { // Pick the first backend to support the player info int32 index = 0; VideoBackend* backend; #define TRY_USE_BACKEND(type) \ backend = VideoServiceInstance.Backends[index]; \ if (!backend) \ VideoServiceInstance.InitBackend(index, backend = new type()); \ if (!backend->Player_Create(info, player)) \ return false; #if VIDEO_API_MF TRY_USE_BACKEND(VideoBackendMF); #endif #undef TRY_USE_BACKEND LOG(Error, "Failed to setup Video playback backend for '{}'", info.Url); return true; } void VideoBackendPlayer::InitVideoFrame() { if (!GPUDevice::Instance) return; if (!Frame) Frame = GPUDevice::Instance->CreateTexture(TEXT("VideoFrame")); } void VideoBackendPlayer::UpdateVideoFrame(Span frame, TimeSpan time, TimeSpan duration) { PROFILE_CPU(); VideoFrameTime = time; VideoFrameDuration = duration; if (!GPUDevice::Instance || GPUDevice::Instance->GetRendererType() == RendererType::Null) return; // Ensure that sampled frame data matches the target texture size uint32 rowPitch, slicePitch; RenderTools::ComputePitch(Format, VideoFrameWidth, VideoFrameHeight, rowPitch, slicePitch); if (slicePitch != frame.Length()) { LOG(Warning, "Incorrect video frame stride {}, doesn't match stride {} of video {}x{} in format {}", frame.Length(), slicePitch, Width, Height, ScriptingEnum::ToString(Format)); return; } // Copy frame into buffer for video frames uploading if (VideoFrameMemory.Length() < (int32)slicePitch) { VideoFrameMemory.Allocate(slicePitch); if (VideoFrameMemory.IsInvalid()) { OUT_OF_MEMORY; return; } } Platform::MemoryCopy(VideoFrameMemory.Get(), frame.Get(), slicePitch); // Update output frame texture InitVideoFrame(); auto desc = GPUTextureDescription::New2D(Width, Height, PixelFormat::R8G8B8A8_UNorm, GPUTextureFlags::ShaderResource | GPUTextureFlags::RenderTarget); if (Frame->GetDescription() != desc) { if (Frame->Init(desc)) { LOG(Error, "Failed to allocate video frame texture"); return; } } // Start texture upload task (if not already - only one is needed to upload the latest frame) if (!UploadVideoFrameTask) { UploadVideoFrameTask = New(this); UploadVideoFrameTask->Start(); } } void VideoBackendPlayer::ReleaseResources() { if (UploadVideoFrameTask) UploadVideoFrameTask->Cancel(); VideoFrameMemory.Release(); SAFE_DELETE_GPU_RESOURCE(Frame); SAFE_DELETE_GPU_RESOURCE(FrameUpload); }