From 2af4e8fe1055f81a1c4658e9ab0df1278248a16d Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Wed, 22 May 2024 11:53:46 +0200 Subject: [PATCH] Add AV video backend for macOS and iOS --- Source/Engine/Video/AV/VideoBackendAV.cpp | 289 ++++++++++++++++++ Source/Engine/Video/AV/VideoBackendAV.h | 30 ++ .../Video/Android/VideoBackendAndroid.cpp | 2 +- Source/Engine/Video/MF/VideoBackendMF.cpp | 2 +- Source/Engine/Video/Video.Build.cs | 6 + Source/Engine/Video/Video.cpp | 20 +- .../Flax.Build/Platforms/Mac/MacToolchain.cs | 3 + .../Flax.Build/Platforms/iOS/iOSToolchain.cs | 3 + 8 files changed, 346 insertions(+), 9 deletions(-) create mode 100644 Source/Engine/Video/AV/VideoBackendAV.cpp create mode 100644 Source/Engine/Video/AV/VideoBackendAV.h diff --git a/Source/Engine/Video/AV/VideoBackendAV.cpp b/Source/Engine/Video/AV/VideoBackendAV.cpp new file mode 100644 index 000000000..f0c2a71b0 --- /dev/null +++ b/Source/Engine/Video/AV/VideoBackendAV.cpp @@ -0,0 +1,289 @@ +// Copyright (c) 2012-2024 Wojciech Figat. All rights reserved. + +#if VIDEO_API_AV + +#include "VideoBackendAV.h" +#include "Engine/Platform/Apple/AppleUtils.h" +#include "Engine/Profiler/ProfilerCPU.h" +#include "Engine/Threading/TaskGraph.h" +#include "Engine/Core/Log.h" +#include "Engine/Engine/Globals.h" +#include + +#define VIDEO_API_AV_ERROR(api, err) LOG(Warning, "[VideoBackendAV] {} failed with error 0x{:x}", TEXT(#api), (uint64)err) + +struct VideoPlayerAV +{ + AVPlayer* Player; + AVPlayerItemVideoOutput* Output; + int8 PendingPlay : 1; + int8 PendingPause : 1; + int8 PendingSeek : 1; + TimeSpan SeekTime; +}; + +namespace AV +{ + Array Players; + + TimeSpan ConvertTime(const CMTime& t) + { + return TimeSpan::FromSeconds(t.timescale != 0 ? (t.value / (double)t.timescale) : 0.0); + } + + CMTime ConvertTime(const TimeSpan& t) + { + return CMTime{(CMTimeValue)(100000.0 * t.GetTotalSeconds()), (CMTimeScale)100000, kCMTimeFlags_Valid, {}}; + } + + void UpdatePlayer(int32 index) + { + PROFILE_CPU(); + auto& player = *Players[index]; + ZoneText(player.DebugUrl, player.DebugUrlLen); + auto& playerAV = player.GetBackendState(); + + // Update format + AVPlayerItem* playerItem = [playerAV.Player currentItem]; + if (!playerItem) + return; + if (player.Width == 0) + { + CGSize size = [playerItem presentationSize]; + player.Width = player.VideoFrameWidth = size.width; + player.Height = player.VideoFrameHeight = size.height; + NSArray* tracks = [playerItem tracks]; + for (NSUInteger i = 0; i < [tracks count]; i++) + { + AVPlayerItemTrack* track = (AVPlayerItemTrack*)[tracks objectAtIndex:i]; + AVAssetTrack* assetTrack = track.assetTrack; + NSString* mediaType = assetTrack.mediaType; + if ([mediaType isEqualToString:AVMediaTypeVideo] && playerAV.Output == nullptr) + { + player.FrameRate = assetTrack.nominalFrameRate; + if (player.FrameRate <= 0.0f) + { + CMTime frameDuration = assetTrack.minFrameDuration; + if ((frameDuration.flags & kCMTimeFlags_Valid) != 0) + player.FrameRate = (float)frameDuration.timescale / (float)frameDuration.value; + else + player.FrameRate = 25; + } + CGSize frameSize = assetTrack.naturalSize; + player.Width = player.VideoFrameWidth = frameSize.width; + player.Height = player.VideoFrameHeight = frameSize.height; + + CMFormatDescriptionRef desc = (CMFormatDescriptionRef)[assetTrack.formatDescriptions objectAtIndex:0]; + CMVideoCodecType codec = CMFormatDescriptionGetMediaSubType(desc); + int32 pixelFormat = kCVPixelFormatType_32BGRA; // TODO: use packed vieo format + player.Format = PixelFormat::B8G8R8A8_UNorm; + + NSMutableDictionary* attributes = [NSMutableDictionary dictionary]; + [attributes setObject:[NSNumber numberWithInt: pixelFormat] forKey:(NSString*)kCVPixelBufferPixelFormatTypeKey]; + [attributes setObject:[NSNumber numberWithInteger:1] forKey:(NSString*)kCVPixelBufferBytesPerRowAlignmentKey]; + + playerAV.Output = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:attributes]; + [playerItem addOutput: playerAV.Output]; + } + else if ([mediaType isEqualToString:AVMediaTypeAudio]) + { + CMFormatDescriptionRef desc = (CMFormatDescriptionRef)[assetTrack.formatDescriptions objectAtIndex:0]; + const AudioStreamBasicDescription* audioDesc = CMAudioFormatDescriptionGetStreamBasicDescription(desc); + player.AudioInfo.SampleRate = audioDesc->mSampleRate; + player.AudioInfo.NumChannels = audioDesc->mChannelsPerFrame; + player.AudioInfo.BitDepth = audioDesc->mBitsPerChannel > 0 ? audioDesc->mBitsPerChannel : 16; + } + } + } + + // Wait for the video to be ready + //AVPlayerStatus status = [playerAV.Player status]; + //AVPlayerTimeControlStatus timeControlStatus = [playerAV.Player timeControlStatus]; + if (playerAV.Output == nullptr) + return; + + // Control playback + if (playerAV.PendingPlay) + { + playerAV.PendingPlay = 0; + [playerAV.Player play]; + } + else if (playerAV.PendingPause) + { + playerAV.PendingPause = 0; + [playerAV.Player pause]; + } + if (playerAV.PendingSeek) + { + playerAV.PendingSeek = 0; + [playerAV.Player seekToTime:AV::ConvertTime(playerAV.SeekTime)]; + //[playerAV.Player seekToTime:time toleranceBefore:time toleranceAfter:time]; + } + + // Check if there is a new video frame to process + CMTime currentTime = [playerAV.Player currentTime]; + if (playerAV.Output && [playerAV.Output hasNewPixelBufferForItemTime: currentTime]) + { + CVPixelBufferRef buffer = [playerAV.Output copyPixelBufferForItemTime:currentTime itemTimeForDisplay:nullptr]; + if (buffer) + { + const int32 bufferWidth = CVPixelBufferGetWidth(buffer); + const int32 bufferHeight = CVPixelBufferGetHeight(buffer); + const int32 bufferStride = CVPixelBufferGetBytesPerRow(buffer); + const int32 bufferSize = bufferStride * bufferHeight; + + // TODO: use Metal Texture Cache for faster GPU-based video processing + + if (CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly) == kCVReturnSuccess) + { + uint8* bufferData = (uint8*)CVPixelBufferGetBaseAddress(buffer); + player.UpdateVideoFrame(Span(bufferData, bufferSize), ConvertTime(currentTime), TimeSpan::FromSeconds(1.0f / player.FrameRate)); + CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + } + + CVPixelBufferRelease(buffer); + } + } + + player.Tick(); + } +} + +bool VideoBackendAV::Player_Create(const VideoBackendPlayerInfo& info, VideoBackendPlayer& player) +{ + PROFILE_CPU(); + player = VideoBackendPlayer(); + auto& playerAV = player.GetBackendState(); + + // Load media + NSURL* url; + if (info.Url.StartsWith(TEXT("http"), StringSearchCase::IgnoreCase)) + { + url = [NSURL URLWithString:(NSString*)AppleUtils::ToString(info.Url)]; + + } + else + { +#if PLATFORM_MAC + if (info.Url.StartsWith(TEXT("Content/"), StringSearchCase::CaseSensitive)) + url = [NSURL fileURLWithPath:(NSString*)AppleUtils::ToString(Globals::ProjectFolder / info.Url) isDirectory:NO]; + else + url = [NSURL fileURLWithPath:(NSString*)AppleUtils::ToString(info.Url) isDirectory:NO]; +#else + url = [NSURL fileURLWithPath:(NSString*)AppleUtils::ToString(StringUtils::GetFileName(info.Url)) isDirectory:NO]; +#endif + } + playerAV.Player = [AVPlayer playerWithURL:url]; + if (playerAV.Player == nullptr) + { + return true; + } + [playerAV.Player retain]; + + // Configure player + //[playerAV.Player addObserver:playerStatusObserver.get() forKeyPath:"status" options:NSKeyValueObservingOptionNew context:&player]; + playerAV.Player.actionAtItemEnd = info.Loop ? AVPlayerActionAtItemEndNone : AVPlayerActionAtItemEndPause; + [playerAV.Player setVolume: info.Volume]; + + // Setup player data + player.Backend = this; + player.Created(info); + AV::Players.Add(&player); + + return false; +} + +void VideoBackendAV::Player_Destroy(VideoBackendPlayer& player) +{ + PROFILE_CPU(); + player.ReleaseResources(); + auto& playerAV = player.GetBackendState(); + if (playerAV.PendingPause) + [playerAV.Player pause]; + if (playerAV.Output) + [playerAV.Output release]; + [playerAV.Player release]; + AV::Players.Remove(&player); + player = VideoBackendPlayer(); +} + +void VideoBackendAV::Player_UpdateInfo(VideoBackendPlayer& player, const VideoBackendPlayerInfo& info) +{ + PROFILE_CPU(); + auto& playerAV = player.GetBackendState(); + playerAV.Player.actionAtItemEnd = info.Loop ? AVPlayerActionAtItemEndNone : AVPlayerActionAtItemEndPause; + // TODO: spatial audio + // TODO: audio pan + [playerAV.Player setVolume: info.Volume]; + player.Updated(info); +} + +void VideoBackendAV::Player_Play(VideoBackendPlayer& player) +{ + PROFILE_CPU(); + auto& playerAV = player.GetBackendState(); + playerAV.PendingPlay = true; + playerAV.PendingPause = false; + player.PlayAudio(); +} + +void VideoBackendAV::Player_Pause(VideoBackendPlayer& player) +{ + PROFILE_CPU(); + auto& playerAV = player.GetBackendState(); + playerAV.PendingPlay = false; + playerAV.PendingPause = true; + player.PauseAudio(); +} + +void VideoBackendAV::Player_Stop(VideoBackendPlayer& player) +{ + PROFILE_CPU(); + auto& playerAV = player.GetBackendState(); + playerAV.PendingPlay = false; + playerAV.PendingPause = true; + playerAV.PendingSeek = true; + playerAV.SeekTime = TimeSpan::Zero(); + player.StopAudio(); +} + +void VideoBackendAV::Player_Seek(VideoBackendPlayer& player, TimeSpan time) +{ + PROFILE_CPU(); + auto& playerAV = player.GetBackendState(); + playerAV.PendingSeek = true; + playerAV.SeekTime = time; +} + +TimeSpan VideoBackendAV::Player_GetTime(const VideoBackendPlayer& player) +{ + PROFILE_CPU(); + auto& playerAV = player.GetBackendState(); + if (playerAV.PendingSeek) + return playerAV.SeekTime; + return AV::ConvertTime([playerAV.Player currentTime]); +} + +const Char* VideoBackendAV::Base_Name() +{ + return TEXT("AVFoundation"); +} + +bool VideoBackendAV::Base_Init() +{ + return false; +} + +void VideoBackendAV::Base_Update(TaskGraph* graph) +{ + // Schedule work to update all videos in async + Function job; + job.Bind(AV::UpdatePlayer); + graph->DispatchJob(job, AV::Players.Count()); +} + +void VideoBackendAV::Base_Dispose() +{ +} + +#endif diff --git a/Source/Engine/Video/AV/VideoBackendAV.h b/Source/Engine/Video/AV/VideoBackendAV.h new file mode 100644 index 000000000..85f7456e4 --- /dev/null +++ b/Source/Engine/Video/AV/VideoBackendAV.h @@ -0,0 +1,30 @@ +// Copyright (c) 2012-2024 Wojciech Figat. All rights reserved. + +#pragma once + +#if VIDEO_API_AV + +#include "../VideoBackend.h" + +/// +/// The AVFoundation video backend. +/// +class VideoBackendAV : public VideoBackend +{ +public: + // [VideoBackend] + bool Player_Create(const VideoBackendPlayerInfo& info, VideoBackendPlayer& player) override; + void Player_Destroy(VideoBackendPlayer& player) override; + void Player_UpdateInfo(VideoBackendPlayer& player, const VideoBackendPlayerInfo& info) override; + void Player_Play(VideoBackendPlayer& player) override; + void Player_Pause(VideoBackendPlayer& player) override; + void Player_Stop(VideoBackendPlayer& player) override; + void Player_Seek(VideoBackendPlayer& player, TimeSpan time) override; + TimeSpan Player_GetTime(const VideoBackendPlayer& player) override; + const Char* Base_Name() override; + bool Base_Init() override; + void Base_Update(TaskGraph* graph) override; + void Base_Dispose() override; +}; + +#endif diff --git a/Source/Engine/Video/Android/VideoBackendAndroid.cpp b/Source/Engine/Video/Android/VideoBackendAndroid.cpp index 46d04f062..bfa1296a3 100644 --- a/Source/Engine/Video/Android/VideoBackendAndroid.cpp +++ b/Source/Engine/Video/Android/VideoBackendAndroid.cpp @@ -574,7 +574,7 @@ bool VideoBackendAndroid::Base_Init() void VideoBackendAndroid::Base_Update(TaskGraph* graph) { - // Schedule work to update all videos models in async + // Schedule work to update all videos in async Function job; job.Bind(Android::UpdatePlayer); graph->DispatchJob(job, Android::Players.Count()); diff --git a/Source/Engine/Video/MF/VideoBackendMF.cpp b/Source/Engine/Video/MF/VideoBackendMF.cpp index ef020b858..29911575b 100644 --- a/Source/Engine/Video/MF/VideoBackendMF.cpp +++ b/Source/Engine/Video/MF/VideoBackendMF.cpp @@ -582,7 +582,7 @@ bool VideoBackendMF::Base_Init() void VideoBackendMF::Base_Update(TaskGraph* graph) { - // Schedule work to update all videos models in async + // Schedule work to update all videos in async Function job; job.Bind(MF::UpdatePlayer); graph->DispatchJob(job, MF::Players.Count()); diff --git a/Source/Engine/Video/Video.Build.cs b/Source/Engine/Video/Video.Build.cs index ee0fef30f..d9e15ca4b 100644 --- a/Source/Engine/Video/Video.Build.cs +++ b/Source/Engine/Video/Video.Build.cs @@ -34,6 +34,12 @@ public class Video : EngineModule options.OutputFiles.Add("mfreadwrite.lib"); options.OutputFiles.Add("mfuuid.lib"); break; + case TargetPlatform.Mac: + case TargetPlatform.iOS: + // AVFoundation + options.SourcePaths.Add(Path.Combine(FolderPath, "AV")); + options.CompileEnv.PreprocessorDefinitions.Add("VIDEO_API_AV"); + break; case TargetPlatform.PS4: options.SourcePaths.Add(Path.Combine(Globals.EngineRoot, "Source", "Platforms", "PS4", "Engine", "Video")); options.CompileEnv.PreprocessorDefinitions.Add("VIDEO_API_PS4"); diff --git a/Source/Engine/Video/Video.cpp b/Source/Engine/Video/Video.cpp index 1dbef13a0..cb2881923 100644 --- a/Source/Engine/Video/Video.cpp +++ b/Source/Engine/Video/Video.cpp @@ -22,6 +22,9 @@ #if VIDEO_API_MF #include "MF/VideoBackendMF.h" #endif +#if VIDEO_API_AV +#include "AV/VideoBackendAV.h" +#endif #if VIDEO_API_ANDROID #include "Android/VideoBackendAndroid.h" #endif @@ -109,13 +112,17 @@ protected: context->GPU->SetState(pso); context->GPU->DrawFullscreenTriangle(); } - else + else if (frame->Format() == _player->Format) { // Raw texture data upload uint32 rowPitch, slicePitch; frame->ComputePitch(0, rowPitch, slicePitch); context->GPU->UpdateTexture(frame, 0, 0, _player->VideoFrameMemory.Get(), rowPitch, slicePitch); } + else + { + LOG(Warning, "Incorrect video player data format {} for player texture format {}", ScriptingEnum::ToString(_player->Format), ScriptingEnum::ToString(_player->Frame->Format())); + } // Frame has been updated _player->FramesCount++; @@ -161,7 +168,6 @@ public: } bool Init() override; - void Update() override; void Dispose() override; }; @@ -187,11 +193,6 @@ bool VideoService::Init() return false; } -void VideoService::Update() -{ - PROFILE_CPU_NAMED("Video.Update"); -} - void VideoService::Dispose() { PROFILE_CPU_NAMED("Video.Dispose"); @@ -223,6 +224,9 @@ bool Video::CreatePlayerBackend(const VideoBackendPlayerInfo& info, VideoBackend #if VIDEO_API_MF TRY_USE_BACKEND(VideoBackendMF); #endif +#if VIDEO_API_AV + TRY_USE_BACKEND(VideoBackendAV); +#endif #if VIDEO_API_ANDROID TRY_USE_BACKEND(VideoBackendAndroid); #endif @@ -335,6 +339,8 @@ void VideoBackendPlayer::UpdateVideoFrame(Span data, TimeSpan time, TimeSp // Update output frame texture InitVideoFrame(); auto desc = GPUTextureDescription::New2D(Width, Height, PixelFormat::R8G8B8A8_UNorm, GPUTextureFlags::ShaderResource | GPUTextureFlags::RenderTarget); + if (!PixelFormatExtensions::IsVideo(Format)) + desc.Format = Format; // Use raw format reported by the backend (eg. BGRA) if (Frame->GetDescription() != desc) { if (Frame->Init(desc)) diff --git a/Source/Tools/Flax.Build/Platforms/Mac/MacToolchain.cs b/Source/Tools/Flax.Build/Platforms/Mac/MacToolchain.cs index 99854a1a1..00d7a4a07 100644 --- a/Source/Tools/Flax.Build/Platforms/Mac/MacToolchain.cs +++ b/Source/Tools/Flax.Build/Platforms/Mac/MacToolchain.cs @@ -45,10 +45,13 @@ namespace Flax.Build.Platforms options.LinkEnv.InputLibraries.Add("bz2"); options.LinkEnv.InputLibraries.Add("CoreFoundation.framework"); options.LinkEnv.InputLibraries.Add("CoreGraphics.framework"); + options.LinkEnv.InputLibraries.Add("CoreMedia.framework"); + options.LinkEnv.InputLibraries.Add("CoreVideo.framework"); options.LinkEnv.InputLibraries.Add("SystemConfiguration.framework"); options.LinkEnv.InputLibraries.Add("IOKit.framework"); options.LinkEnv.InputLibraries.Add("Cocoa.framework"); options.LinkEnv.InputLibraries.Add("QuartzCore.framework"); + options.LinkEnv.InputLibraries.Add("AVFoundation.framework"); } protected override void AddArgsCommon(BuildOptions options, List args) diff --git a/Source/Tools/Flax.Build/Platforms/iOS/iOSToolchain.cs b/Source/Tools/Flax.Build/Platforms/iOS/iOSToolchain.cs index 273382d7c..9addc55a0 100644 --- a/Source/Tools/Flax.Build/Platforms/iOS/iOSToolchain.cs +++ b/Source/Tools/Flax.Build/Platforms/iOS/iOSToolchain.cs @@ -47,10 +47,13 @@ namespace Flax.Build.Platforms options.LinkEnv.InputLibraries.Add("Foundation.framework"); options.LinkEnv.InputLibraries.Add("CoreFoundation.framework"); options.LinkEnv.InputLibraries.Add("CoreGraphics.framework"); + options.LinkEnv.InputLibraries.Add("CoreMedia.framework"); + options.LinkEnv.InputLibraries.Add("CoreVideo.framework"); options.LinkEnv.InputLibraries.Add("SystemConfiguration.framework"); options.LinkEnv.InputLibraries.Add("IOKit.framework"); options.LinkEnv.InputLibraries.Add("UIKit.framework"); options.LinkEnv.InputLibraries.Add("QuartzCore.framework"); + options.LinkEnv.InputLibraries.Add("AVFoundation.framework"); } protected override void AddArgsCommon(BuildOptions options, List args)