Phase 7

apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp

@@ -7,11 +7,14 @@
 #include "RuntimeEventDispatcher.h"
 
 #include <chrono>
+#include <cstring>
 #include <windows.h>
 
 VideoBackend::VideoBackend(RenderEngine& renderEngine, HealthTelemetry& healthTelemetry, RuntimeEventDispatcher& runtimeEventDispatcher) :
 	mHealthTelemetry(healthTelemetry),
 	mRuntimeEventDispatcher(runtimeEventDispatcher),
+	mPlayoutPolicy(NormalizeVideoPlayoutPolicy(VideoPlayoutPolicy())),
+	mReadyOutputQueue(mPlayoutPolicy),
 	mVideoIODevice(std::make_unique<DeckLinkSession>()),
 	mBridge(std::make_unique<OpenGLVideoIOBridge>(renderEngine))
 {
@@ -24,6 +27,8 @@ VideoBackend::~VideoBackend()
 
 void VideoBackend::ReleaseResources()
 {
+	StopOutputCompletionWorker();
+	mReadyOutputQueue.Clear();
 	if (mVideoIODevice)
 		mVideoIODevice->ReleaseResources();
 	if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped))
@@ -73,6 +78,8 @@ bool VideoBackend::ConfigureInput(const VideoFormat& inputVideoMode, std::string
 
 bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error)
 {
+	mPlayoutPolicy = NormalizeVideoPlayoutPolicy(mPlayoutPolicy);
+	mReadyOutputQueue.Configure(mPlayoutPolicy);
 	if (mLifecycle.State() != VideoBackendLifecycleState::Configuring)
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend output.");
 	if (!mVideoIODevice->ConfigureOutput(
@@ -90,11 +97,15 @@ bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool exte
 bool VideoBackend::Start()
 {
 	ApplyLifecycleTransition(VideoBackendLifecycleState::Prerolling, "Video backend preroll starting.");
+	StartOutputCompletionWorker();
 	const bool started = mVideoIODevice->Start();
 	if (started)
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Running, "Video backend started.");
 	else
+	{
+		StopOutputCompletionWorker();
 		ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend start failed." : StatusMessage());
+	}
 	return started;
 }
 
@@ -102,6 +113,7 @@ bool VideoBackend::Stop()
 {
 	ApplyLifecycleTransition(VideoBackendLifecycleState::Stopping, "Video backend stopping.");
 	const bool stopped = mVideoIODevice->Stop();
+	StopOutputCompletionWorker();
 	if (stopped)
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend stopped.");
 	else
@@ -134,9 +146,9 @@ bool VideoBackend::ScheduleOutputFrame(const VideoIOOutputFrame& frame)
 	return mVideoIODevice->ScheduleOutputFrame(frame);
 }
 
-void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result)
+void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth)
 {
-	mVideoIODevice->AccountForCompletionResult(result);
+	mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth);
 }
 
 bool VideoBackend::HasInputDevice() const
@@ -264,30 +276,156 @@ void VideoBackend::HandleInputFrame(const VideoIOFrame& frame)
 
 void VideoBackend::HandleOutputFrameCompletion(const VideoIOCompletion& completion)
 {
-	RecordFramePacing(completion.result);
-	PublishOutputFrameCompleted(completion);
+	{
+		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
+		if (!mOutputCompletionWorkerRunning || mOutputCompletionWorkerStopping)
+			return;
+		mPendingOutputCompletions.push_back(completion);
+	}
+	mOutputCompletionCondition.notify_one();
+}
 
-	VideoIOOutputFrame outputFrame;
-	if (!BeginOutputFrame(outputFrame))
+void VideoBackend::StartOutputCompletionWorker()
+{
+	std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
+	if (mOutputCompletionWorkerRunning)
 		return;
 
-	const VideoIOState& state = mVideoIODevice->State();
+	mPendingOutputCompletions.clear();
+	mReadyOutputQueue.Clear();
+	mNextReadyOutputFrameIndex = 0;
+	mOutputCompletionWorkerStopping = false;
+	mOutputCompletionWorkerRunning = true;
+	mOutputCompletionWorker = std::thread(&VideoBackend::OutputCompletionWorkerMain, this);
+}
+
+void VideoBackend::StopOutputCompletionWorker()
+{
+	bool shouldJoin = false;
+	{
+		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
+		if (mOutputCompletionWorkerRunning)
+			mOutputCompletionWorkerStopping = true;
+		shouldJoin = mOutputCompletionWorker.joinable();
+	}
+	mOutputCompletionCondition.notify_one();
+
+	if (shouldJoin)
+		mOutputCompletionWorker.join();
+}
+
+void VideoBackend::OutputCompletionWorkerMain()
+{
+	for (;;)
+	{
+		VideoIOCompletion completion;
+		{
+			std::unique_lock<std::mutex> lock(mOutputCompletionMutex);
+			mOutputCompletionCondition.wait(lock, [this]() {
+				return mOutputCompletionWorkerStopping || !mPendingOutputCompletions.empty();
+			});
+
+			if (mPendingOutputCompletions.empty())
+			{
+				if (mOutputCompletionWorkerStopping)
+				{
+					mOutputCompletionWorkerRunning = false;
+					return;
+				}
+				continue;
+			}
+
+			completion = mPendingOutputCompletions.front();
+			mPendingOutputCompletions.pop_front();
+		}
+
+		ProcessOutputFrameCompletion(completion);
+	}
+}
+
+void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion)
+{
+	RecordFramePacing(completion.result);
+	PublishOutputFrameCompleted(completion);
+	AccountForCompletionResult(completion.result, mReadyOutputQueue.GetMetrics().depth);
+
+	FillReadyOutputQueue(completion);
+	if (!ScheduleReadyOutputFrame())
+		ScheduleBlackUnderrunFrame();
+}
+
+bool VideoBackend::FillReadyOutputQueue(const VideoIOCompletion& completion)
+{
+	RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics();
+	bool filledAny = false;
+	while (metrics.depth < mPlayoutPolicy.targetReadyFrames)
+	{
+		if (!RenderReadyOutputFrame(mVideoIODevice->State(), completion))
+			return filledAny;
+		filledAny = true;
+		metrics = mReadyOutputQueue.GetMetrics();
+	}
+	return true;
+}
+
+bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion)
+{
+	VideoIOOutputFrame outputFrame;
+	if (!BeginOutputFrame(outputFrame))
+		return false;
+
 	bool rendered = true;
 	if (mBridge)
 		rendered = mBridge->RenderScheduledFrame(state, completion, outputFrame);
 
 	EndOutputFrame(outputFrame);
-	AccountForCompletionResult(completion.result);
 	if (!rendered)
 	{
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame.");
-		return;
+		return false;
 	}
 
-	// Schedule the next frame after render work is complete so device-side
-	// bookkeeping stays with the backend seam and the bridge stays render-only.
-	if (ScheduleOutputFrame(outputFrame))
-		PublishOutputFrameScheduled(outputFrame);
+	RenderOutputFrame readyFrame;
+	readyFrame.frame = outputFrame;
+	readyFrame.frameIndex = ++mNextReadyOutputFrameIndex;
+	return mReadyOutputQueue.Push(readyFrame);
+}
+
+bool VideoBackend::ScheduleReadyOutputFrame()
+{
+	RenderOutputFrame readyFrame;
+	if (!mReadyOutputQueue.TryPop(readyFrame))
+		return false;
+
+	if (!ScheduleOutputFrame(readyFrame.frame))
+		return false;
+
+	PublishOutputFrameScheduled(readyFrame.frame);
+	return true;
+}
+
+bool VideoBackend::ScheduleBlackUnderrunFrame()
+{
+	VideoIOOutputFrame outputFrame;
+	if (!BeginOutputFrame(outputFrame))
+	{
+		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: no output frame was available for fallback scheduling.");
+		return false;
+	}
+
+	if (outputFrame.bytes != nullptr && outputFrame.rowBytes > 0 && outputFrame.height > 0)
+		std::memset(outputFrame.bytes, 0, static_cast<std::size_t>(outputFrame.rowBytes) * outputFrame.height);
+	EndOutputFrame(outputFrame);
+
+	if (!ScheduleOutputFrame(outputFrame))
+	{
+		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: black fallback frame scheduling failed.");
+		return false;
+	}
+
+	ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: scheduled black fallback frame.");
+	PublishOutputFrameScheduled(outputFrame);
+	return true;
 }
 
 void VideoBackend::RecordFramePacing(VideoIOCompletionResult completionResult)

apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h

@@ -1,12 +1,18 @@
 #pragma once
 
+#include "RenderOutputQueue.h"
 #include "VideoBackendLifecycle.h"
 #include "VideoIOTypes.h"
+#include "VideoPlayoutPolicy.h"
 
 #include <chrono>
+#include <condition_variable>
 #include <cstdint>
+#include <deque>
 #include <memory>
+#include <mutex>
 #include <string>
+#include <thread>
 
 class HealthTelemetry;
 class OpenGLVideoIOBridge;
@@ -34,7 +40,7 @@ public:
 	bool BeginOutputFrame(VideoIOOutputFrame& frame);
 	void EndOutputFrame(VideoIOOutputFrame& frame);
 	bool ScheduleOutputFrame(const VideoIOOutputFrame& frame);
-	void AccountForCompletionResult(VideoIOCompletionResult result);
+	void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth);
 
 	bool HasInputDevice() const;
 	bool HasInputSource() const;
@@ -59,6 +65,14 @@ public:
 private:
 	void HandleInputFrame(const VideoIOFrame& frame);
 	void HandleOutputFrameCompletion(const VideoIOCompletion& completion);
+	void StartOutputCompletionWorker();
+	void StopOutputCompletionWorker();
+	void OutputCompletionWorkerMain();
+	void ProcessOutputFrameCompletion(const VideoIOCompletion& completion);
+	bool FillReadyOutputQueue(const VideoIOCompletion& completion);
+	bool RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion);
+	bool ScheduleReadyOutputFrame();
+	bool ScheduleBlackUnderrunFrame();
 	void RecordFramePacing(VideoIOCompletionResult completionResult);
 	bool ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message);
 	bool ApplyLifecycleFailure(const std::string& message);
@@ -74,8 +88,17 @@ private:
 	HealthTelemetry& mHealthTelemetry;
 	RuntimeEventDispatcher& mRuntimeEventDispatcher;
 	VideoBackendLifecycle mLifecycle;
+	VideoPlayoutPolicy mPlayoutPolicy;
+	RenderOutputQueue mReadyOutputQueue;
 	std::unique_ptr<VideoIODevice> mVideoIODevice;
 	std::unique_ptr<OpenGLVideoIOBridge> mBridge;
+	std::mutex mOutputCompletionMutex;
+	std::condition_variable mOutputCompletionCondition;
+	std::deque<VideoIOCompletion> mPendingOutputCompletions;
+	std::thread mOutputCompletionWorker;
+	bool mOutputCompletionWorkerRunning = false;
+	bool mOutputCompletionWorkerStopping = false;
+	uint64_t mNextReadyOutputFrameIndex = 0;
 	uint64_t mInputFrameIndex = 0;
 	uint64_t mOutputFrameScheduleIndex = 0;
 	uint64_t mOutputFrameCompletionIndex = 0;

apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h

@@ -105,7 +105,7 @@ public:
 	virtual bool BeginOutputFrame(VideoIOOutputFrame& frame) = 0;
 	virtual void EndOutputFrame(VideoIOOutputFrame& frame) = 0;
 	virtual bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) = 0;
-	virtual void AccountForCompletionResult(VideoIOCompletionResult result) = 0;
+	virtual void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) = 0;
 
 	bool HasInputDevice() const { return State().hasInputDevice; }
 	bool HasInputSource() const { return State().hasInputSource; }

apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.cpp

@@ -16,6 +16,9 @@ void VideoPlayoutScheduler::Configure(int64_t frameDuration, int64_t timeScale,
 void VideoPlayoutScheduler::Reset()
 {
 	mScheduledFrameIndex = 0;
+	mCompletedFrameIndex = 0;
+	mLateStreak = 0;
+	mDropStreak = 0;
 }
 
 VideoIOScheduleTime VideoPlayoutScheduler::NextScheduleTime()
@@ -29,10 +32,38 @@ VideoIOScheduleTime VideoPlayoutScheduler::NextScheduleTime()
 	return time;
 }
 
-void VideoPlayoutScheduler::AccountForCompletionResult(VideoIOCompletionResult result)
+VideoPlayoutRecoveryDecision VideoPlayoutScheduler::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth)
 {
-	if (result == VideoIOCompletionResult::DisplayedLate || result == VideoIOCompletionResult::Dropped)
-		mScheduledFrameIndex += mPolicy.lateOrDropCatchUpFrames;
+	++mCompletedFrameIndex;
+	if (result == VideoIOCompletionResult::DisplayedLate)
+		++mLateStreak;
+	else
+		mLateStreak = 0;
+	if (result == VideoIOCompletionResult::Dropped)
+		++mDropStreak;
+	else
+		mDropStreak = 0;
+
+	const uint64_t measuredLagFrames = MeasureLag(result, readyQueueDepth);
+	const uint64_t catchUpFrames = measuredLagFrames < mPolicy.lateOrDropCatchUpFrames
+		? measuredLagFrames
+		: mPolicy.lateOrDropCatchUpFrames;
+	if (catchUpFrames > 0)
+		mScheduledFrameIndex += catchUpFrames;
+
+	VideoPlayoutRecoveryDecision decision;
+	decision.result = result;
+	decision.completedFrameIndex = mCompletedFrameIndex;
+	decision.scheduledFrameIndex = mScheduledFrameIndex;
+	decision.readyQueueDepth = readyQueueDepth;
+	decision.scheduledLeadFrames = mScheduledFrameIndex > mCompletedFrameIndex
+		? mScheduledFrameIndex - mCompletedFrameIndex
+		: 0;
+	decision.measuredLagFrames = measuredLagFrames;
+	decision.catchUpFrames = catchUpFrames;
+	decision.lateStreak = mLateStreak;
+	decision.dropStreak = mDropStreak;
+	return decision;
 }
 
 double VideoPlayoutScheduler::FrameBudgetMilliseconds() const
@@ -41,3 +72,26 @@ double VideoPlayoutScheduler::FrameBudgetMilliseconds() const
 		? (static_cast<double>(mFrameDuration) * 1000.0) / static_cast<double>(mTimeScale)
 		: 0.0;
 }
+
+uint64_t VideoPlayoutScheduler::MeasureLag(VideoIOCompletionResult result, uint64_t readyQueueDepth) const
+{
+	if (result != VideoIOCompletionResult::DisplayedLate && result != VideoIOCompletionResult::Dropped)
+		return 0;
+
+	uint64_t lagFrames = 1;
+	if (result == VideoIOCompletionResult::DisplayedLate && mLateStreak > lagFrames)
+		lagFrames = mLateStreak;
+	if (result == VideoIOCompletionResult::Dropped && mDropStreak * 2 > lagFrames)
+		lagFrames = mDropStreak * 2;
+
+	if (mCompletedFrameIndex >= mScheduledFrameIndex)
+	{
+		const uint64_t scheduleLagFrames = mCompletedFrameIndex - mScheduledFrameIndex + 1;
+		if (scheduleLagFrames > lagFrames)
+			lagFrames = scheduleLagFrames;
+	}
+	if (readyQueueDepth < mPolicy.targetReadyFrames && mPolicy.targetReadyFrames - readyQueueDepth > lagFrames)
+		lagFrames = mPolicy.targetReadyFrames - readyQueueDepth;
+
+	return lagFrames;
+}

apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h

@@ -5,6 +5,19 @@
 
 #include <cstdint>
 
+struct VideoPlayoutRecoveryDecision
+{
+	VideoIOCompletionResult result = VideoIOCompletionResult::Completed;
+	uint64_t completedFrameIndex = 0;
+	uint64_t scheduledFrameIndex = 0;
+	uint64_t readyQueueDepth = 0;
+	uint64_t scheduledLeadFrames = 0;
+	uint64_t measuredLagFrames = 0;
+	uint64_t catchUpFrames = 0;
+	uint64_t lateStreak = 0;
+	uint64_t dropStreak = 0;
+};
+
 class VideoPlayoutScheduler
 {
 public:
@@ -12,15 +25,23 @@ public:
 	void Configure(int64_t frameDuration, int64_t timeScale, const VideoPlayoutPolicy& policy);
 	void Reset();
 	VideoIOScheduleTime NextScheduleTime();
-	void AccountForCompletionResult(VideoIOCompletionResult result);
+	VideoPlayoutRecoveryDecision AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth = 0);
 	double FrameBudgetMilliseconds() const;
 	uint64_t ScheduledFrameIndex() const { return mScheduledFrameIndex; }
+	uint64_t CompletedFrameIndex() const { return mCompletedFrameIndex; }
+	uint64_t LateStreak() const { return mLateStreak; }
+	uint64_t DropStreak() const { return mDropStreak; }
 	int64_t TimeScale() const { return mTimeScale; }
 	const VideoPlayoutPolicy& Policy() const { return mPolicy; }
 
 private:
+	uint64_t MeasureLag(VideoIOCompletionResult result, uint64_t readyQueueDepth) const;
+
 	int64_t mFrameDuration = 0;
 	int64_t mTimeScale = 0;
 	uint64_t mScheduledFrameIndex = 0;
+	uint64_t mCompletedFrameIndex = 0;
+	uint64_t mLateStreak = 0;
+	uint64_t mDropStreak = 0;
 	VideoPlayoutPolicy mPolicy;
 };

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp

@@ -498,9 +498,9 @@ void DeckLinkSession::EndOutputFrame(VideoIOOutputFrame& frame)
 	frame.bytes = nullptr;
 }
 
-void DeckLinkSession::AccountForCompletionResult(VideoIOCompletionResult completionResult)
+void DeckLinkSession::AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth)
 {
-	mScheduler.AccountForCompletionResult(completionResult);
+	mScheduler.AccountForCompletionResult(completionResult, readyQueueDepth);
 }
 
 bool DeckLinkSession::ScheduleOutputFrame(const VideoIOOutputFrame& frame)

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h

@@ -59,7 +59,7 @@ public:
 	const VideoIOState& State() const override { return mState; }
 	VideoIOState& MutableState() override { return mState; }
 	double FrameBudgetMilliseconds() const;
-	void AccountForCompletionResult(VideoIOCompletionResult completionResult) override;
+	void AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth) override;
 	bool BeginOutputFrame(VideoIOOutputFrame& frame) override;
 	void EndOutputFrame(VideoIOOutputFrame& frame) override;
 	bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) override;