Phase 7

apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp

@@ -7,11 +7,14 @@
 #include "RuntimeEventDispatcher.h"
 
 #include <chrono>
+#include <cstring>
 #include <windows.h>
 
 VideoBackend::VideoBackend(RenderEngine& renderEngine, HealthTelemetry& healthTelemetry, RuntimeEventDispatcher& runtimeEventDispatcher) :
 	mHealthTelemetry(healthTelemetry),
 	mRuntimeEventDispatcher(runtimeEventDispatcher),
+	mPlayoutPolicy(NormalizeVideoPlayoutPolicy(VideoPlayoutPolicy())),
+	mReadyOutputQueue(mPlayoutPolicy),
 	mVideoIODevice(std::make_unique<DeckLinkSession>()),
 	mBridge(std::make_unique<OpenGLVideoIOBridge>(renderEngine))
 {
@@ -24,6 +27,8 @@ VideoBackend::~VideoBackend()
 
 void VideoBackend::ReleaseResources()
 {
+	StopOutputCompletionWorker();
+	mReadyOutputQueue.Clear();
 	if (mVideoIODevice)
 		mVideoIODevice->ReleaseResources();
 	if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped))
@@ -73,6 +78,8 @@ bool VideoBackend::ConfigureInput(const VideoFormat& inputVideoMode, std::string
 
 bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error)
 {
+	mPlayoutPolicy = NormalizeVideoPlayoutPolicy(mPlayoutPolicy);
+	mReadyOutputQueue.Configure(mPlayoutPolicy);
 	if (mLifecycle.State() != VideoBackendLifecycleState::Configuring)
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend output.");
 	if (!mVideoIODevice->ConfigureOutput(
@@ -90,11 +97,15 @@ bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool exte
 bool VideoBackend::Start()
 {
 	ApplyLifecycleTransition(VideoBackendLifecycleState::Prerolling, "Video backend preroll starting.");
+	StartOutputCompletionWorker();
 	const bool started = mVideoIODevice->Start();
 	if (started)
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Running, "Video backend started.");
 	else
+	{
+		StopOutputCompletionWorker();
 		ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend start failed." : StatusMessage());
+	}
 	return started;
 }
 
@@ -102,6 +113,7 @@ bool VideoBackend::Stop()
 {
 	ApplyLifecycleTransition(VideoBackendLifecycleState::Stopping, "Video backend stopping.");
 	const bool stopped = mVideoIODevice->Stop();
+	StopOutputCompletionWorker();
 	if (stopped)
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend stopped.");
 	else
@@ -134,9 +146,9 @@ bool VideoBackend::ScheduleOutputFrame(const VideoIOOutputFrame& frame)
 	return mVideoIODevice->ScheduleOutputFrame(frame);
 }
 
-void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result)
+void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth)
 {
-	mVideoIODevice->AccountForCompletionResult(result);
+	mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth);
 }
 
 bool VideoBackend::HasInputDevice() const
@@ -264,30 +276,156 @@ void VideoBackend::HandleInputFrame(const VideoIOFrame& frame)
 
 void VideoBackend::HandleOutputFrameCompletion(const VideoIOCompletion& completion)
 {
-	RecordFramePacing(completion.result);
-	PublishOutputFrameCompleted(completion);
+	{
+		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
+		if (!mOutputCompletionWorkerRunning || mOutputCompletionWorkerStopping)
+			return;
+		mPendingOutputCompletions.push_back(completion);
+	}
+	mOutputCompletionCondition.notify_one();
+}
 
-	VideoIOOutputFrame outputFrame;
-	if (!BeginOutputFrame(outputFrame))
+void VideoBackend::StartOutputCompletionWorker()
+{
+	std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
+	if (mOutputCompletionWorkerRunning)
 		return;
 
-	const VideoIOState& state = mVideoIODevice->State();
+	mPendingOutputCompletions.clear();
+	mReadyOutputQueue.Clear();
+	mNextReadyOutputFrameIndex = 0;
+	mOutputCompletionWorkerStopping = false;
+	mOutputCompletionWorkerRunning = true;
+	mOutputCompletionWorker = std::thread(&VideoBackend::OutputCompletionWorkerMain, this);
+}
+
+void VideoBackend::StopOutputCompletionWorker()
+{
+	bool shouldJoin = false;
+	{
+		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
+		if (mOutputCompletionWorkerRunning)
+			mOutputCompletionWorkerStopping = true;
+		shouldJoin = mOutputCompletionWorker.joinable();
+	}
+	mOutputCompletionCondition.notify_one();
+
+	if (shouldJoin)
+		mOutputCompletionWorker.join();
+}
+
+void VideoBackend::OutputCompletionWorkerMain()
+{
+	for (;;)
+	{
+		VideoIOCompletion completion;
+		{
+			std::unique_lock<std::mutex> lock(mOutputCompletionMutex);
+			mOutputCompletionCondition.wait(lock, [this]() {
+				return mOutputCompletionWorkerStopping || !mPendingOutputCompletions.empty();
+			});
+
+			if (mPendingOutputCompletions.empty())
+			{
+				if (mOutputCompletionWorkerStopping)
+				{
+					mOutputCompletionWorkerRunning = false;
+					return;
+				}
+				continue;
+			}
+
+			completion = mPendingOutputCompletions.front();
+			mPendingOutputCompletions.pop_front();
+		}
+
+		ProcessOutputFrameCompletion(completion);
+	}
+}
+
+void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion)
+{
+	RecordFramePacing(completion.result);
+	PublishOutputFrameCompleted(completion);
+	AccountForCompletionResult(completion.result, mReadyOutputQueue.GetMetrics().depth);
+
+	FillReadyOutputQueue(completion);
+	if (!ScheduleReadyOutputFrame())
+		ScheduleBlackUnderrunFrame();
+}
+
+bool VideoBackend::FillReadyOutputQueue(const VideoIOCompletion& completion)
+{
+	RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics();
+	bool filledAny = false;
+	while (metrics.depth < mPlayoutPolicy.targetReadyFrames)
+	{
+		if (!RenderReadyOutputFrame(mVideoIODevice->State(), completion))
+			return filledAny;
+		filledAny = true;
+		metrics = mReadyOutputQueue.GetMetrics();
+	}
+	return true;
+}
+
+bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion)
+{
+	VideoIOOutputFrame outputFrame;
+	if (!BeginOutputFrame(outputFrame))
+		return false;
+
 	bool rendered = true;
 	if (mBridge)
 		rendered = mBridge->RenderScheduledFrame(state, completion, outputFrame);
 
 	EndOutputFrame(outputFrame);
-	AccountForCompletionResult(completion.result);
 	if (!rendered)
 	{
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame.");
-		return;
+		return false;
 	}
 
-	// Schedule the next frame after render work is complete so device-side
-	// bookkeeping stays with the backend seam and the bridge stays render-only.
-	if (ScheduleOutputFrame(outputFrame))
-		PublishOutputFrameScheduled(outputFrame);
+	RenderOutputFrame readyFrame;
+	readyFrame.frame = outputFrame;
+	readyFrame.frameIndex = ++mNextReadyOutputFrameIndex;
+	return mReadyOutputQueue.Push(readyFrame);
+}
+
+bool VideoBackend::ScheduleReadyOutputFrame()
+{
+	RenderOutputFrame readyFrame;
+	if (!mReadyOutputQueue.TryPop(readyFrame))
+		return false;
+
+	if (!ScheduleOutputFrame(readyFrame.frame))
+		return false;
+
+	PublishOutputFrameScheduled(readyFrame.frame);
+	return true;
+}
+
+bool VideoBackend::ScheduleBlackUnderrunFrame()
+{
+	VideoIOOutputFrame outputFrame;
+	if (!BeginOutputFrame(outputFrame))
+	{
+		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: no output frame was available for fallback scheduling.");
+		return false;
+	}
+
+	if (outputFrame.bytes != nullptr && outputFrame.rowBytes > 0 && outputFrame.height > 0)
+		std::memset(outputFrame.bytes, 0, static_cast<std::size_t>(outputFrame.rowBytes) * outputFrame.height);
+	EndOutputFrame(outputFrame);
+
+	if (!ScheduleOutputFrame(outputFrame))
+	{
+		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: black fallback frame scheduling failed.");
+		return false;
+	}
+
+	ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: scheduled black fallback frame.");
+	PublishOutputFrameScheduled(outputFrame);
+	return true;
 }
 
 void VideoBackend::RecordFramePacing(VideoIOCompletionResult completionResult)