Phase 7

CMakeLists.txt

@@ -168,6 +168,8 @@ set(APP_SOURCES
 	"${APP_DIR}/videoio/VideoBackendLifecycle.cpp"
 	"${APP_DIR}/videoio/VideoBackendLifecycle.h"
 	"${APP_DIR}/videoio/VideoIOTypes.h"
+	"${APP_DIR}/videoio/RenderOutputQueue.cpp"
+	"${APP_DIR}/videoio/RenderOutputQueue.h"
 	"${APP_DIR}/videoio/VideoPlayoutPolicy.h"
 	"${APP_DIR}/videoio/VideoPlayoutScheduler.cpp"
 	"${APP_DIR}/videoio/VideoPlayoutScheduler.h"
@@ -541,6 +543,23 @@ endif()
 
 add_test(NAME VideoPlayoutSchedulerTests COMMAND VideoPlayoutSchedulerTests)
 
+add_executable(RenderOutputQueueTests
+	"${APP_DIR}/videoio/RenderOutputQueue.cpp"
+	"${CMAKE_CURRENT_SOURCE_DIR}/tests/RenderOutputQueueTests.cpp"
+)
+
+target_include_directories(RenderOutputQueueTests PRIVATE
+	"${APP_DIR}"
+	"${APP_DIR}/videoio"
+	"${APP_DIR}/videoio/decklink"
+)
+
+if(MSVC)
+	target_compile_options(RenderOutputQueueTests PRIVATE /W3)
+endif()
+
+add_test(NAME RenderOutputQueueTests COMMAND RenderOutputQueueTests)
+
 add_executable(VideoBackendLifecycleTests
 	"${APP_DIR}/videoio/VideoBackendLifecycle.cpp"
 	"${CMAKE_CURRENT_SOURCE_DIR}/tests/VideoBackendLifecycleTests.cpp"

apps/LoopThroughWithOpenGLCompositing/videoio/RenderOutputQueue.cpp

@@ -0,0 +1,70 @@
+#include "RenderOutputQueue.h"
+
+RenderOutputQueue::RenderOutputQueue(const VideoPlayoutPolicy& policy) :
+	mPolicy(NormalizeVideoPlayoutPolicy(policy))
+{
+}
+
+void RenderOutputQueue::Configure(const VideoPlayoutPolicy& policy)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mPolicy = NormalizeVideoPlayoutPolicy(policy);
+	while (mReadyFrames.size() > CapacityLocked())
+	{
+		mReadyFrames.pop_front();
+		++mDroppedCount;
+	}
+}
+
+bool RenderOutputQueue::Push(RenderOutputFrame frame)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (mReadyFrames.size() >= CapacityLocked())
+	{
+		mReadyFrames.pop_front();
+		++mDroppedCount;
+	}
+
+	mReadyFrames.push_back(frame);
+	++mPushedCount;
+	return true;
+}
+
+bool RenderOutputQueue::TryPop(RenderOutputFrame& frame)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (mReadyFrames.empty())
+	{
+		++mUnderrunCount;
+		return false;
+	}
+
+	frame = mReadyFrames.front();
+	mReadyFrames.pop_front();
+	++mPoppedCount;
+	return true;
+}
+
+void RenderOutputQueue::Clear()
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mReadyFrames.clear();
+}
+
+RenderOutputQueueMetrics RenderOutputQueue::GetMetrics() const
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	RenderOutputQueueMetrics metrics;
+	metrics.depth = mReadyFrames.size();
+	metrics.capacity = CapacityLocked();
+	metrics.pushedCount = mPushedCount;
+	metrics.poppedCount = mPoppedCount;
+	metrics.droppedCount = mDroppedCount;
+	metrics.underrunCount = mUnderrunCount;
+	return metrics;
+}
+
+std::size_t RenderOutputQueue::CapacityLocked() const
+{
+	return static_cast<std::size_t>(mPolicy.maxReadyFrames);
+}

apps/LoopThroughWithOpenGLCompositing/videoio/RenderOutputQueue.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include "VideoIOTypes.h"
+#include "VideoPlayoutPolicy.h"
+
+#include <cstdint>
+#include <deque>
+#include <mutex>
+
+struct RenderOutputFrame
+{
+	VideoIOOutputFrame frame;
+	uint64_t frameIndex = 0;
+	bool stale = false;
+};
+
+struct RenderOutputQueueMetrics
+{
+	std::size_t depth = 0;
+	std::size_t capacity = 0;
+	uint64_t pushedCount = 0;
+	uint64_t poppedCount = 0;
+	uint64_t droppedCount = 0;
+	uint64_t underrunCount = 0;
+};
+
+class RenderOutputQueue
+{
+public:
+	explicit RenderOutputQueue(const VideoPlayoutPolicy& policy = VideoPlayoutPolicy());
+
+	void Configure(const VideoPlayoutPolicy& policy);
+	bool Push(RenderOutputFrame frame);
+	bool TryPop(RenderOutputFrame& frame);
+	void Clear();
+	RenderOutputQueueMetrics GetMetrics() const;
+
+private:
+	std::size_t CapacityLocked() const;
+
+	mutable std::mutex mMutex;
+	VideoPlayoutPolicy mPolicy;
+	std::deque<RenderOutputFrame> mReadyFrames;
+	uint64_t mPushedCount = 0;
+	uint64_t mPoppedCount = 0;
+	uint64_t mDroppedCount = 0;
+	uint64_t mUnderrunCount = 0;
+};

apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp

@@ -7,11 +7,14 @@
 #include "RuntimeEventDispatcher.h"
 
 #include <chrono>
+#include <cstring>
 #include <windows.h>
 
 VideoBackend::VideoBackend(RenderEngine& renderEngine, HealthTelemetry& healthTelemetry, RuntimeEventDispatcher& runtimeEventDispatcher) :
 	mHealthTelemetry(healthTelemetry),
 	mRuntimeEventDispatcher(runtimeEventDispatcher),
+	mPlayoutPolicy(NormalizeVideoPlayoutPolicy(VideoPlayoutPolicy())),
+	mReadyOutputQueue(mPlayoutPolicy),
 	mVideoIODevice(std::make_unique<DeckLinkSession>()),
 	mBridge(std::make_unique<OpenGLVideoIOBridge>(renderEngine))
 {
@@ -24,6 +27,8 @@ VideoBackend::~VideoBackend()
 
 void VideoBackend::ReleaseResources()
 {
+	StopOutputCompletionWorker();
+	mReadyOutputQueue.Clear();
 	if (mVideoIODevice)
 		mVideoIODevice->ReleaseResources();
 	if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped))
@@ -73,6 +78,8 @@ bool VideoBackend::ConfigureInput(const VideoFormat& inputVideoMode, std::string
 
 bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error)
 {
+	mPlayoutPolicy = NormalizeVideoPlayoutPolicy(mPlayoutPolicy);
+	mReadyOutputQueue.Configure(mPlayoutPolicy);
 	if (mLifecycle.State() != VideoBackendLifecycleState::Configuring)
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend output.");
 	if (!mVideoIODevice->ConfigureOutput(
@@ -90,11 +97,15 @@ bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool exte
 bool VideoBackend::Start()
 {
 	ApplyLifecycleTransition(VideoBackendLifecycleState::Prerolling, "Video backend preroll starting.");
+	StartOutputCompletionWorker();
 	const bool started = mVideoIODevice->Start();
 	if (started)
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Running, "Video backend started.");
 	else
+	{
+		StopOutputCompletionWorker();
 		ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend start failed." : StatusMessage());
+	}
 	return started;
 }
 
@@ -102,6 +113,7 @@ bool VideoBackend::Stop()
 {
 	ApplyLifecycleTransition(VideoBackendLifecycleState::Stopping, "Video backend stopping.");
 	const bool stopped = mVideoIODevice->Stop();
+	StopOutputCompletionWorker();
 	if (stopped)
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend stopped.");
 	else
@@ -134,9 +146,9 @@ bool VideoBackend::ScheduleOutputFrame(const VideoIOOutputFrame& frame)
 	return mVideoIODevice->ScheduleOutputFrame(frame);
 }
 
-void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result)
+void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth)
 {
-	mVideoIODevice->AccountForCompletionResult(result);
+	mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth);
 }
 
 bool VideoBackend::HasInputDevice() const
@@ -264,30 +276,156 @@ void VideoBackend::HandleInputFrame(const VideoIOFrame& frame)
 
 void VideoBackend::HandleOutputFrameCompletion(const VideoIOCompletion& completion)
 {
-	RecordFramePacing(completion.result);
-	PublishOutputFrameCompleted(completion);
+	{
+		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
+		if (!mOutputCompletionWorkerRunning || mOutputCompletionWorkerStopping)
+			return;
+		mPendingOutputCompletions.push_back(completion);
+	}
+	mOutputCompletionCondition.notify_one();
+}
 
-	VideoIOOutputFrame outputFrame;
-	if (!BeginOutputFrame(outputFrame))
+void VideoBackend::StartOutputCompletionWorker()
+{
+	std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
+	if (mOutputCompletionWorkerRunning)
 		return;
 
-	const VideoIOState& state = mVideoIODevice->State();
+	mPendingOutputCompletions.clear();
+	mReadyOutputQueue.Clear();
+	mNextReadyOutputFrameIndex = 0;
+	mOutputCompletionWorkerStopping = false;
+	mOutputCompletionWorkerRunning = true;
+	mOutputCompletionWorker = std::thread(&VideoBackend::OutputCompletionWorkerMain, this);
+}
+
+void VideoBackend::StopOutputCompletionWorker()
+{
+	bool shouldJoin = false;
+	{
+		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
+		if (mOutputCompletionWorkerRunning)
+			mOutputCompletionWorkerStopping = true;
+		shouldJoin = mOutputCompletionWorker.joinable();
+	}
+	mOutputCompletionCondition.notify_one();
+
+	if (shouldJoin)
+		mOutputCompletionWorker.join();
+}
+
+void VideoBackend::OutputCompletionWorkerMain()
+{
+	for (;;)
+	{
+		VideoIOCompletion completion;
+		{
+			std::unique_lock<std::mutex> lock(mOutputCompletionMutex);
+			mOutputCompletionCondition.wait(lock, [this]() {
+				return mOutputCompletionWorkerStopping || !mPendingOutputCompletions.empty();
+			});
+
+			if (mPendingOutputCompletions.empty())
+			{
+				if (mOutputCompletionWorkerStopping)
+				{
+					mOutputCompletionWorkerRunning = false;
+					return;
+				}
+				continue;
+			}
+
+			completion = mPendingOutputCompletions.front();
+			mPendingOutputCompletions.pop_front();
+		}
+
+		ProcessOutputFrameCompletion(completion);
+	}
+}
+
+void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion)
+{
+	RecordFramePacing(completion.result);
+	PublishOutputFrameCompleted(completion);
+	AccountForCompletionResult(completion.result, mReadyOutputQueue.GetMetrics().depth);
+
+	FillReadyOutputQueue(completion);
+	if (!ScheduleReadyOutputFrame())
+		ScheduleBlackUnderrunFrame();
+}
+
+bool VideoBackend::FillReadyOutputQueue(const VideoIOCompletion& completion)
+{
+	RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics();
+	bool filledAny = false;
+	while (metrics.depth < mPlayoutPolicy.targetReadyFrames)
+	{
+		if (!RenderReadyOutputFrame(mVideoIODevice->State(), completion))
+			return filledAny;
+		filledAny = true;
+		metrics = mReadyOutputQueue.GetMetrics();
+	}
+	return true;
+}
+
+bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion)
+{
+	VideoIOOutputFrame outputFrame;
+	if (!BeginOutputFrame(outputFrame))
+		return false;
+
 	bool rendered = true;
 	if (mBridge)
 		rendered = mBridge->RenderScheduledFrame(state, completion, outputFrame);
 
 	EndOutputFrame(outputFrame);
-	AccountForCompletionResult(completion.result);
 	if (!rendered)
 	{
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame.");
-		return;
+		return false;
 	}
 
-	// Schedule the next frame after render work is complete so device-side
-	// bookkeeping stays with the backend seam and the bridge stays render-only.
-	if (ScheduleOutputFrame(outputFrame))
-		PublishOutputFrameScheduled(outputFrame);
+	RenderOutputFrame readyFrame;
+	readyFrame.frame = outputFrame;
+	readyFrame.frameIndex = ++mNextReadyOutputFrameIndex;
+	return mReadyOutputQueue.Push(readyFrame);
+}
+
+bool VideoBackend::ScheduleReadyOutputFrame()
+{
+	RenderOutputFrame readyFrame;
+	if (!mReadyOutputQueue.TryPop(readyFrame))
+		return false;
+
+	if (!ScheduleOutputFrame(readyFrame.frame))
+		return false;
+
+	PublishOutputFrameScheduled(readyFrame.frame);
+	return true;
+}
+
+bool VideoBackend::ScheduleBlackUnderrunFrame()
+{
+	VideoIOOutputFrame outputFrame;
+	if (!BeginOutputFrame(outputFrame))
+	{
+		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: no output frame was available for fallback scheduling.");
+		return false;
+	}
+
+	if (outputFrame.bytes != nullptr && outputFrame.rowBytes > 0 && outputFrame.height > 0)
+		std::memset(outputFrame.bytes, 0, static_cast<std::size_t>(outputFrame.rowBytes) * outputFrame.height);
+	EndOutputFrame(outputFrame);
+
+	if (!ScheduleOutputFrame(outputFrame))
+	{
+		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: black fallback frame scheduling failed.");
+		return false;
+	}
+
+	ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: scheduled black fallback frame.");
+	PublishOutputFrameScheduled(outputFrame);
+	return true;
 }
 
 void VideoBackend::RecordFramePacing(VideoIOCompletionResult completionResult)

apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h

@@ -1,12 +1,18 @@
 #pragma once
 
+#include "RenderOutputQueue.h"
 #include "VideoBackendLifecycle.h"
 #include "VideoIOTypes.h"
+#include "VideoPlayoutPolicy.h"
 
 #include <chrono>
+#include <condition_variable>
 #include <cstdint>
+#include <deque>
 #include <memory>
+#include <mutex>
 #include <string>
+#include <thread>
 
 class HealthTelemetry;
 class OpenGLVideoIOBridge;
@@ -34,7 +40,7 @@ public:
 	bool BeginOutputFrame(VideoIOOutputFrame& frame);
 	void EndOutputFrame(VideoIOOutputFrame& frame);
 	bool ScheduleOutputFrame(const VideoIOOutputFrame& frame);
-	void AccountForCompletionResult(VideoIOCompletionResult result);
+	void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth);
 
 	bool HasInputDevice() const;
 	bool HasInputSource() const;
@@ -59,6 +65,14 @@ public:
 private:
 	void HandleInputFrame(const VideoIOFrame& frame);
 	void HandleOutputFrameCompletion(const VideoIOCompletion& completion);
+	void StartOutputCompletionWorker();
+	void StopOutputCompletionWorker();
+	void OutputCompletionWorkerMain();
+	void ProcessOutputFrameCompletion(const VideoIOCompletion& completion);
+	bool FillReadyOutputQueue(const VideoIOCompletion& completion);
+	bool RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion);
+	bool ScheduleReadyOutputFrame();
+	bool ScheduleBlackUnderrunFrame();
 	void RecordFramePacing(VideoIOCompletionResult completionResult);
 	bool ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message);
 	bool ApplyLifecycleFailure(const std::string& message);
@@ -74,8 +88,17 @@ private:
 	HealthTelemetry& mHealthTelemetry;
 	RuntimeEventDispatcher& mRuntimeEventDispatcher;
 	VideoBackendLifecycle mLifecycle;
+	VideoPlayoutPolicy mPlayoutPolicy;
+	RenderOutputQueue mReadyOutputQueue;
 	std::unique_ptr<VideoIODevice> mVideoIODevice;
 	std::unique_ptr<OpenGLVideoIOBridge> mBridge;
+	std::mutex mOutputCompletionMutex;
+	std::condition_variable mOutputCompletionCondition;
+	std::deque<VideoIOCompletion> mPendingOutputCompletions;
+	std::thread mOutputCompletionWorker;
+	bool mOutputCompletionWorkerRunning = false;
+	bool mOutputCompletionWorkerStopping = false;
+	uint64_t mNextReadyOutputFrameIndex = 0;
 	uint64_t mInputFrameIndex = 0;
 	uint64_t mOutputFrameScheduleIndex = 0;
 	uint64_t mOutputFrameCompletionIndex = 0;

apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h

@@ -105,7 +105,7 @@ public:
 	virtual bool BeginOutputFrame(VideoIOOutputFrame& frame) = 0;
 	virtual void EndOutputFrame(VideoIOOutputFrame& frame) = 0;
 	virtual bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) = 0;
-	virtual void AccountForCompletionResult(VideoIOCompletionResult result) = 0;
+	virtual void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) = 0;
 
 	bool HasInputDevice() const { return State().hasInputDevice; }
 	bool HasInputSource() const { return State().hasInputSource; }

apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.cpp

@@ -16,6 +16,9 @@ void VideoPlayoutScheduler::Configure(int64_t frameDuration, int64_t timeScale,
 void VideoPlayoutScheduler::Reset()
 {
 	mScheduledFrameIndex = 0;
+	mCompletedFrameIndex = 0;
+	mLateStreak = 0;
+	mDropStreak = 0;
 }
 
 VideoIOScheduleTime VideoPlayoutScheduler::NextScheduleTime()
@@ -29,10 +32,38 @@ VideoIOScheduleTime VideoPlayoutScheduler::NextScheduleTime()
 	return time;
 }
 
-void VideoPlayoutScheduler::AccountForCompletionResult(VideoIOCompletionResult result)
+VideoPlayoutRecoveryDecision VideoPlayoutScheduler::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth)
 {
-	if (result == VideoIOCompletionResult::DisplayedLate || result == VideoIOCompletionResult::Dropped)
-		mScheduledFrameIndex += mPolicy.lateOrDropCatchUpFrames;
+	++mCompletedFrameIndex;
+	if (result == VideoIOCompletionResult::DisplayedLate)
+		++mLateStreak;
+	else
+		mLateStreak = 0;
+	if (result == VideoIOCompletionResult::Dropped)
+		++mDropStreak;
+	else
+		mDropStreak = 0;
+
+	const uint64_t measuredLagFrames = MeasureLag(result, readyQueueDepth);
+	const uint64_t catchUpFrames = measuredLagFrames < mPolicy.lateOrDropCatchUpFrames
+		? measuredLagFrames
+		: mPolicy.lateOrDropCatchUpFrames;
+	if (catchUpFrames > 0)
+		mScheduledFrameIndex += catchUpFrames;
+
+	VideoPlayoutRecoveryDecision decision;
+	decision.result = result;
+	decision.completedFrameIndex = mCompletedFrameIndex;
+	decision.scheduledFrameIndex = mScheduledFrameIndex;
+	decision.readyQueueDepth = readyQueueDepth;
+	decision.scheduledLeadFrames = mScheduledFrameIndex > mCompletedFrameIndex
+		? mScheduledFrameIndex - mCompletedFrameIndex
+		: 0;
+	decision.measuredLagFrames = measuredLagFrames;
+	decision.catchUpFrames = catchUpFrames;
+	decision.lateStreak = mLateStreak;
+	decision.dropStreak = mDropStreak;
+	return decision;
 }
 
 double VideoPlayoutScheduler::FrameBudgetMilliseconds() const
@@ -41,3 +72,26 @@ double VideoPlayoutScheduler::FrameBudgetMilliseconds() const
 		? (static_cast<double>(mFrameDuration) * 1000.0) / static_cast<double>(mTimeScale)
 		: 0.0;
 }
+
+uint64_t VideoPlayoutScheduler::MeasureLag(VideoIOCompletionResult result, uint64_t readyQueueDepth) const
+{
+	if (result != VideoIOCompletionResult::DisplayedLate && result != VideoIOCompletionResult::Dropped)
+		return 0;
+
+	uint64_t lagFrames = 1;
+	if (result == VideoIOCompletionResult::DisplayedLate && mLateStreak > lagFrames)
+		lagFrames = mLateStreak;
+	if (result == VideoIOCompletionResult::Dropped && mDropStreak * 2 > lagFrames)
+		lagFrames = mDropStreak * 2;
+
+	if (mCompletedFrameIndex >= mScheduledFrameIndex)
+	{
+		const uint64_t scheduleLagFrames = mCompletedFrameIndex - mScheduledFrameIndex + 1;
+		if (scheduleLagFrames > lagFrames)
+			lagFrames = scheduleLagFrames;
+	}
+	if (readyQueueDepth < mPolicy.targetReadyFrames && mPolicy.targetReadyFrames - readyQueueDepth > lagFrames)
+		lagFrames = mPolicy.targetReadyFrames - readyQueueDepth;
+
+	return lagFrames;
+}

apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h

@@ -5,6 +5,19 @@
 
 #include <cstdint>
 
+struct VideoPlayoutRecoveryDecision
+{
+	VideoIOCompletionResult result = VideoIOCompletionResult::Completed;
+	uint64_t completedFrameIndex = 0;
+	uint64_t scheduledFrameIndex = 0;
+	uint64_t readyQueueDepth = 0;
+	uint64_t scheduledLeadFrames = 0;
+	uint64_t measuredLagFrames = 0;
+	uint64_t catchUpFrames = 0;
+	uint64_t lateStreak = 0;
+	uint64_t dropStreak = 0;
+};
+
 class VideoPlayoutScheduler
 {
 public:
@@ -12,15 +25,23 @@ public:
 	void Configure(int64_t frameDuration, int64_t timeScale, const VideoPlayoutPolicy& policy);
 	void Reset();
 	VideoIOScheduleTime NextScheduleTime();
-	void AccountForCompletionResult(VideoIOCompletionResult result);
+	VideoPlayoutRecoveryDecision AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth = 0);
 	double FrameBudgetMilliseconds() const;
 	uint64_t ScheduledFrameIndex() const { return mScheduledFrameIndex; }
+	uint64_t CompletedFrameIndex() const { return mCompletedFrameIndex; }
+	uint64_t LateStreak() const { return mLateStreak; }
+	uint64_t DropStreak() const { return mDropStreak; }
 	int64_t TimeScale() const { return mTimeScale; }
 	const VideoPlayoutPolicy& Policy() const { return mPolicy; }
 
 private:
+	uint64_t MeasureLag(VideoIOCompletionResult result, uint64_t readyQueueDepth) const;
+
 	int64_t mFrameDuration = 0;
 	int64_t mTimeScale = 0;
 	uint64_t mScheduledFrameIndex = 0;
+	uint64_t mCompletedFrameIndex = 0;
+	uint64_t mLateStreak = 0;
+	uint64_t mDropStreak = 0;
 	VideoPlayoutPolicy mPolicy;
 };

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp

@@ -498,9 +498,9 @@ void DeckLinkSession::EndOutputFrame(VideoIOOutputFrame& frame)
 	frame.bytes = nullptr;
 }
 
-void DeckLinkSession::AccountForCompletionResult(VideoIOCompletionResult completionResult)
+void DeckLinkSession::AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth)
 {
-	mScheduler.AccountForCompletionResult(completionResult);
+	mScheduler.AccountForCompletionResult(completionResult, readyQueueDepth);
 }
 
 bool DeckLinkSession::ScheduleOutputFrame(const VideoIOOutputFrame& frame)

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h

@@ -59,7 +59,7 @@ public:
 	const VideoIOState& State() const override { return mState; }
 	VideoIOState& MutableState() override { return mState; }
 	double FrameBudgetMilliseconds() const;
-	void AccountForCompletionResult(VideoIOCompletionResult completionResult) override;
+	void AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth) override;
 	bool BeginOutputFrame(VideoIOOutputFrame& frame) override;
 	void EndOutputFrame(VideoIOOutputFrame& frame) override;
 	bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) override;

docs/PHASE_7_BACKEND_LIFECYCLE_PLAYOUT_DESIGN.md

@@ -2,22 +2,23 @@
 
 This document expands Phase 7 of [ARCHITECTURE_RESILIENCE_REVIEW.md](/c:/Users/Aiden/Documents/GitHub/video-shader-toys/docs/ARCHITECTURE_RESILIENCE_REVIEW.md) into a concrete design target.
 
-Phase 4 made the render thread the sole owner of normal runtime GL work, but output timing is still callback-coupled: DeckLink completion callbacks synchronously request render-thread output production before scheduling the next hardware frame. Phase 7 should make backend lifecycle, buffer policy, playout headroom, and recovery explicit.
+Phase 4 made the render thread the sole owner of normal runtime GL work. Phase 7 Step 4 moved DeckLink completion processing onto a backend worker, so the callback no longer directly waits for render-thread output production. Phase 7 Step 5 added a bounded ready-frame queue inside that worker, so scheduling now consumes completed output frames and falls back explicitly on underrun. Phase 7 should make backend lifecycle, buffer policy, playout headroom, and recovery explicit.
 
 Phase 5 clarified that live parameter layering stops at final render-state composition. Phase 7 should keep backend lifecycle, output queue ownership, buffer reuse, temporal/feedback resources, and stale-frame/underrun policy outside the persisted/committed/transient parameter model.
 
 ## Status
 
 - Phase 7 design package: proposed.
-- Phase 7 implementation: Step 2 complete.
-- Current alignment: `VideoBackend`, `VideoIODevice`, `DeckLinkSession`, `VideoBackendLifecycle`, and `VideoPlayoutScheduler` exist. Phase 4 removed callback-thread GL ownership, but the DeckLink completion path still waits for render-thread output production.
+- Phase 7 implementation: Step 6 complete.
+- Current alignment: `VideoBackend`, `VideoIODevice`, `DeckLinkSession`, `VideoBackendLifecycle`, and `VideoPlayoutScheduler` exist. Phase 4 removed callback-thread GL ownership, Step 4 moved completion processing onto a backend worker, Step 5 uses `RenderOutputQueue` as the ready-frame handoff inside that worker, and Step 6 replaces fixed late/drop skip-ahead with measured recovery decisions.
 
 Current backend footholds:
 
 - `VideoBackend` wraps device discovery/configuration, start/stop, input callback handling, output completion handling, and telemetry publication.
 - `DeckLinkSession` owns DeckLink device handles, frame pool creation, preroll, keyer configuration, and scheduled playback.
 - `VideoPlayoutPolicy` names current frame pool, preroll, ready-frame, underrun, and catch-up policy defaults.
-- `VideoPlayoutScheduler` owns basic schedule time generation and simple late/drop skip-ahead behavior.
+- `RenderOutputQueue` names the future bounded ready-output-frame handoff and has pure queue tests.
+- `VideoPlayoutScheduler` owns schedule time generation, completion indexing, late/drop streaks, ready-queue pressure input, and measured recovery decisions.
 - `OpenGLVideoIOBridge` is the current adapter between `VideoBackend` and `RenderEngine`.
 - `HealthTelemetry` receives some signal, render, and pacing stats.
 
@@ -27,7 +28,7 @@ The current output path works only while render/readback stays comfortably insid
 
 The resilience review calls this the main remaining live-resilience risk after Phase 4:
 
-- output playout is still effectively render-on-demand from the DeckLink completion callback
+- output playout is still effectively filled on demand by a backend completion worker, but scheduling now consumes a bounded ready-frame queue
 - buffer pool size and preroll depth are not sourced from one policy
 - late/dropped recovery is a fixed skip rule
 - backend lifecycle is imperative rather than represented as explicit states
@@ -241,9 +242,17 @@ Introduce a bounded queue for completed output frames.
 
 Initial target:
 
-- pure queue tests
-- explicit depth/underrun metrics
-- no DeckLink dependency in queue tests
+- [x] pure queue tests
+- [x] explicit depth/underrun metrics
+- [x] no DeckLink dependency in queue tests
+
+Current implementation:
+
+- `RenderOutputQueue` owns a bounded FIFO of `RenderOutputFrame` values.
+- The queue is configured from `VideoPlayoutPolicy::maxReadyFrames`.
+- Queue metrics report depth, capacity, pushed, popped, dropped, and underrun counts.
+- Overflow drops the oldest ready frame, preserving the newest completed output for scheduling.
+- `RenderOutputQueueTests` cover ordering, bounded overflow, underrun counting, and capacity shrink behavior without DeckLink hardware.
 
 ### Step 4. Move Callback Toward Dequeue/Schedule
 
@@ -251,22 +260,36 @@ Stop producing frames directly in the completion callback path.
 
 Transitional target:
 
-- callback wakes/schedules a backend worker
-- worker consumes ready frames
+- [x] callback wakes/schedules a backend worker
+- [x] worker consumes ready frames
 
 Final target:
 
 - callback only records, recycles, dequeues, schedules
 
+Current implementation:
+
+- `VideoBackend::HandleOutputFrameCompletion(...)` now enqueues completion work and wakes an output-completion worker.
+- The output-completion worker drains pending completions and runs the existing render/schedule path.
+- This preserves behavior while removing the direct callback-thread wait on render-thread output production.
+- Step 5 now makes this worker consume ready frames from `RenderOutputQueue`; Step 4 remains the boundary that keeps output completion callbacks from doing render production directly.
+
 ### Step 5. Make Render Produce Ahead
 
 Teach render/output code to keep the ready queue filled to target headroom.
 
 Initial target:
 
-- render thread produces on demand until queue has target depth
-- callback does not synchronously wait for fresh render
-- stale/black fallback is explicit on underrun
+- [x] render thread produces on demand until queue has target depth
+- [x] callback does not synchronously wait for fresh render
+- [x] stale/black fallback is explicit on underrun
+
+Current implementation:
+
+- The backend output-completion worker fills `RenderOutputQueue` to `VideoPlayoutPolicy::targetReadyFrames`.
+- Scheduling now pops a ready frame from `RenderOutputQueue` instead of directly scheduling the freshly rendered frame.
+- If no ready frame can be produced, the worker schedules an explicit black fallback frame and reports degraded lifecycle state.
+- This is still demand-filled by the backend worker; a future pass can make render production more proactive or timer/pressure driven.
 
 ### Step 6. Replace Fixed Late/Drop Recovery
 
@@ -274,8 +297,16 @@ Replace fixed `+2` schedule-index recovery with measured lag/headroom accounting
 
 Initial target:
 
-- track scheduled index, completed index, queue depth, late streak, drop streak
-- recovery decisions use measured lag
+- [x] track scheduled index, completed index, queue depth, late streak, drop streak
+- [x] recovery decisions use measured lag
+
+Current implementation:
+
+- `VideoPlayoutRecoveryDecision` reports completion result, completed index, scheduled index, ready queue depth, scheduled lead, measured lag, catch-up frames, late streak, and drop streak.
+- `VideoPlayoutScheduler::AccountForCompletionResult(...)` now accepts ready queue depth and returns a recovery decision.
+- Recovery is measured from late/drop streaks, scheduled lead, and ready queue pressure, then capped by `VideoPlayoutPolicy::lateOrDropCatchUpFrames`.
+- `VideoBackend` passes the current ready queue depth into the video device completion-accounting call.
+- `VideoPlayoutSchedulerTests` cover measured late recovery, measured drop recovery, policy caps, completed-index tracking, and streak clearing.
 
 ### Step 7. Route Backend Health Structurally
 
@@ -330,10 +361,10 @@ Phase 7 can be considered complete once the project can say:
 
 - [x] backend lifecycle states and transitions are explicit
 - [x] playout policy owns preroll, pool size, headroom, and underrun behavior
-- [ ] output callbacks no longer synchronously wait for render production
-- [ ] render produces completed output frames into a bounded queue
-- [ ] underrun behavior is explicit and observable
-- [ ] late/drop recovery is measured rather than fixed skip-only
+- [x] output callbacks no longer synchronously wait for render production
+- [x] render produces completed output frames into a bounded queue
+- [x] underrun behavior is explicit and observable
+- [x] late/drop recovery is measured rather than fixed skip-only
 - [ ] backend health reports lifecycle, queue, underrun, late, and dropped state
 - [ ] queue/lifecycle/scheduler behavior has non-DeckLink tests
 

tests/RenderOutputQueueTests.cpp

@@ -0,0 +1,110 @@
+#include "RenderOutputQueue.h"
+
+#include <iostream>
+
+namespace
+{
+int gFailures = 0;
+
+void Expect(bool condition, const char* message)
+{
+	if (condition)
+		return;
+
+	std::cerr << "FAIL: " << message << "\n";
+	++gFailures;
+}
+
+RenderOutputFrame MakeFrame(uint64_t index)
+{
+	RenderOutputFrame frame;
+	frame.frameIndex = index;
+	frame.frame.nativeFrame = reinterpret_cast<void*>(static_cast<uintptr_t>(index + 1));
+	return frame;
+}
+
+void TestQueuePreservesOrdering()
+{
+	VideoPlayoutPolicy policy;
+	policy.maxReadyFrames = 3;
+	RenderOutputQueue queue(policy);
+
+	Expect(queue.Push(MakeFrame(1)), "first ready frame pushes");
+	Expect(queue.Push(MakeFrame(2)), "second ready frame pushes");
+
+	RenderOutputFrame frame;
+	Expect(queue.TryPop(frame), "first ready frame pops");
+	Expect(frame.frameIndex == 1, "queue pops first frame first");
+	Expect(queue.TryPop(frame), "second ready frame pops");
+	Expect(frame.frameIndex == 2, "queue pops second frame second");
+}
+
+void TestBoundedQueueDropsOldestFrame()
+{
+	VideoPlayoutPolicy policy;
+	policy.maxReadyFrames = 2;
+	RenderOutputQueue queue(policy);
+
+	queue.Push(MakeFrame(1));
+	queue.Push(MakeFrame(2));
+	queue.Push(MakeFrame(3));
+
+	RenderOutputQueueMetrics metrics = queue.GetMetrics();
+	Expect(metrics.depth == 2, "bounded queue depth stays at capacity");
+	Expect(metrics.droppedCount == 1, "bounded queue counts dropped oldest frame");
+
+	RenderOutputFrame frame;
+	Expect(queue.TryPop(frame), "bounded queue pops after drop");
+	Expect(frame.frameIndex == 2, "oldest frame was dropped when queue overflowed");
+}
+
+void TestUnderrunIsCounted()
+{
+	RenderOutputQueue queue;
+	RenderOutputFrame frame;
+	Expect(!queue.TryPop(frame), "empty queue reports underrun");
+
+	RenderOutputQueueMetrics metrics = queue.GetMetrics();
+	Expect(metrics.underrunCount == 1, "empty pop increments underrun count");
+}
+
+void TestConfigureShrinksDepthToNewCapacity()
+{
+	VideoPlayoutPolicy policy;
+	policy.maxReadyFrames = 4;
+	RenderOutputQueue queue(policy);
+	queue.Push(MakeFrame(1));
+	queue.Push(MakeFrame(2));
+	queue.Push(MakeFrame(3));
+
+	VideoPlayoutPolicy smallerPolicy;
+	smallerPolicy.targetReadyFrames = 1;
+	smallerPolicy.maxReadyFrames = 1;
+	queue.Configure(smallerPolicy);
+
+	RenderOutputQueueMetrics metrics = queue.GetMetrics();
+	Expect(metrics.depth == 1, "configure trims queue to new capacity");
+	Expect(metrics.droppedCount == 2, "configure counts trimmed frames as drops");
+
+	RenderOutputFrame frame;
+	Expect(queue.TryPop(frame), "trimmed queue still has newest frame");
+	Expect(frame.frameIndex == 3, "configure keeps newest ready frame");
+}
+}
+
+int main()
+{
+	TestQueuePreservesOrdering();
+	TestBoundedQueueDropsOldestFrame();
+	TestUnderrunIsCounted();
+	TestConfigureShrinksDepthToNewCapacity();
+
+	if (gFailures != 0)
+	{
+		std::cerr << gFailures << " render output queue test failure(s).\n";
+		return 1;
+	}
+
+	std::cout << "RenderOutputQueue tests passed.\n";
+	return 0;
+}

tests/VideoIODeviceFakeTests.cpp

@@ -92,13 +92,15 @@ public:
 		return true;
 	}
 
-	void AccountForCompletionResult(VideoIOCompletionResult result) override
+	void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) override
 	{
 		mLastCompletion = result;
+		mLastReadyQueueDepth = readyQueueDepth;
 	}
 
 	unsigned ScheduledFrames() const { return mScheduledFrames; }
 	VideoIOCompletionResult LastCompletion() const { return mLastCompletion; }
+	uint64_t LastReadyQueueDepth() const { return mLastReadyQueueDepth; }
 
 private:
 	VideoIOState mState;
@@ -108,6 +110,7 @@ private:
 	std::array<unsigned char, 7680> mOutputBytes = {};
 	unsigned mScheduledFrames = 0;
 	VideoIOCompletionResult mLastCompletion = VideoIOCompletionResult::Unknown;
+	uint64_t mLastReadyQueueDepth = 0;
 };
 }
 
@@ -132,13 +135,14 @@ int main()
 	VideoIOOutputFrame outputFrame;
 	Expect(device.BeginOutputFrame(outputFrame), "fake output frame can be acquired");
 	device.EndOutputFrame(outputFrame);
-	device.AccountForCompletionResult(VideoIOCompletionResult::Completed);
+	device.AccountForCompletionResult(VideoIOCompletionResult::Completed, 2);
 	Expect(device.ScheduleOutputFrame(outputFrame), "fake output frame can be scheduled");
 
 	Expect(inputSeen, "fake input callback emits generic frame");
 	Expect(outputSeen, "fake output callback emits generic completion");
 	Expect(device.ScheduledFrames() == 1, "fake backend schedules one frame");
 	Expect(device.LastCompletion() == VideoIOCompletionResult::Completed, "fake backend records generic completion");
+	Expect(device.LastReadyQueueDepth() == 2, "fake backend records ready queue depth");
 
 	if (gFailures != 0)
 	{

tests/VideoPlayoutSchedulerTests.cpp

@@ -37,30 +37,51 @@ void TestScheduleAdvancesFromZero()
 	Expect(third.streamTime == 2002, "third frame advances by two durations");
 }
 
-void TestLateAndDroppedSkipAhead()
+void TestLateAndDroppedRecoveryUsesMeasuredPressure()
 {
 	VideoPlayoutScheduler scheduler;
 	scheduler.Configure(1000, 50000);
 
 	(void)scheduler.NextScheduleTime();
-	scheduler.AccountForCompletionResult(VideoIOCompletionResult::DisplayedLate);
-	Expect(scheduler.NextScheduleTime().streamTime == 3000, "late completion preserves the existing two-frame skip policy");
+	VideoPlayoutRecoveryDecision lateDecision = scheduler.AccountForCompletionResult(VideoIOCompletionResult::DisplayedLate, 2);
+	Expect(lateDecision.catchUpFrames == 1, "single late completion catches up by measured one-frame lag");
+	Expect(lateDecision.lateStreak == 1, "late completion increments late streak");
+	Expect(scheduler.NextScheduleTime().streamTime == 2000, "single late recovery advances by measured lag");
 
-	scheduler.AccountForCompletionResult(VideoIOCompletionResult::Dropped);
-	Expect(scheduler.NextScheduleTime().streamTime == 6000, "dropped completion preserves the existing two-frame skip policy");
+	VideoPlayoutRecoveryDecision dropDecision = scheduler.AccountForCompletionResult(VideoIOCompletionResult::Dropped, 2);
+	Expect(dropDecision.catchUpFrames == 2, "dropped completion catches up by measured drop pressure");
+	Expect(dropDecision.lateStreak == 0, "dropped completion resets late streak");
+	Expect(dropDecision.dropStreak == 1, "dropped completion increments drop streak");
+	Expect(scheduler.NextScheduleTime().streamTime == 5000, "drop recovery advances by measured lag");
 }
 
-void TestLateAndDroppedRecoveryUsesPolicy()
+void TestMeasuredRecoveryIsCappedByPolicy()
 {
 	VideoPlayoutPolicy policy;
-	policy.lateOrDropCatchUpFrames = 4;
+	policy.lateOrDropCatchUpFrames = 1;
 
 	VideoPlayoutScheduler scheduler;
 	scheduler.Configure(1000, 50000, policy);
 
 	(void)scheduler.NextScheduleTime();
-	scheduler.AccountForCompletionResult(VideoIOCompletionResult::Dropped);
-	Expect(scheduler.NextScheduleTime().streamTime == 5000, "drop recovery uses policy catch-up frame count");
+	VideoPlayoutRecoveryDecision decision = scheduler.AccountForCompletionResult(VideoIOCompletionResult::Dropped, 0);
+	Expect(decision.measuredLagFrames > decision.catchUpFrames, "policy caps measured recovery");
+	Expect(decision.catchUpFrames == 1, "drop recovery obeys policy cap");
+	Expect(scheduler.NextScheduleTime().streamTime == 2000, "capped recovery advances by one frame");
+}
+
+void TestCleanCompletionTracksCompletedIndexAndClearsStreaks()
+{
+	VideoPlayoutScheduler scheduler;
+	scheduler.Configure(1000, 50000);
+
+	(void)scheduler.NextScheduleTime();
+	(void)scheduler.AccountForCompletionResult(VideoIOCompletionResult::DisplayedLate, 2);
+	VideoPlayoutRecoveryDecision decision = scheduler.AccountForCompletionResult(VideoIOCompletionResult::Completed, 2);
+	Expect(decision.completedFrameIndex == 2, "completion accounting tracks completed index");
+	Expect(decision.catchUpFrames == 0, "clean completion does not catch up");
+	Expect(decision.lateStreak == 0, "clean completion clears late streak");
+	Expect(decision.dropStreak == 0, "clean completion keeps drop streak clear");
 }
 
 void TestPolicyNormalization()
@@ -94,8 +115,9 @@ void TestFrameBudgets()
 int main()
 {
 	TestScheduleAdvancesFromZero();
-	TestLateAndDroppedSkipAhead();
-	TestLateAndDroppedRecoveryUsesPolicy();
+	TestLateAndDroppedRecoveryUsesMeasuredPressure();
+	TestMeasuredRecoveryIsCappedByPolicy();
+	TestCleanCompletionTracksCompletedIndexAndClearsStreaks();
 	TestPolicyNormalization();
 	TestFrameBudgets();