diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp index bf201d5..0d11f35 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp @@ -7,11 +7,14 @@ #include "RuntimeEventDispatcher.h" #include +#include #include VideoBackend::VideoBackend(RenderEngine& renderEngine, HealthTelemetry& healthTelemetry, RuntimeEventDispatcher& runtimeEventDispatcher) : mHealthTelemetry(healthTelemetry), mRuntimeEventDispatcher(runtimeEventDispatcher), + mPlayoutPolicy(NormalizeVideoPlayoutPolicy(VideoPlayoutPolicy())), + mReadyOutputQueue(mPlayoutPolicy), mVideoIODevice(std::make_unique()), mBridge(std::make_unique(renderEngine)) { @@ -24,6 +27,8 @@ VideoBackend::~VideoBackend() void VideoBackend::ReleaseResources() { + StopOutputCompletionWorker(); + mReadyOutputQueue.Clear(); if (mVideoIODevice) mVideoIODevice->ReleaseResources(); if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped)) @@ -73,6 +78,8 @@ bool VideoBackend::ConfigureInput(const VideoFormat& inputVideoMode, std::string bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error) { + mPlayoutPolicy = NormalizeVideoPlayoutPolicy(mPlayoutPolicy); + mReadyOutputQueue.Configure(mPlayoutPolicy); if (mLifecycle.State() != VideoBackendLifecycleState::Configuring) ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend output."); if (!mVideoIODevice->ConfigureOutput( @@ -90,11 +97,15 @@ bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool exte bool VideoBackend::Start() { ApplyLifecycleTransition(VideoBackendLifecycleState::Prerolling, "Video backend preroll starting."); + StartOutputCompletionWorker(); const bool started = mVideoIODevice->Start(); if (started) ApplyLifecycleTransition(VideoBackendLifecycleState::Running, "Video backend started."); else + { + StopOutputCompletionWorker(); ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend start failed." : StatusMessage()); + } return started; } @@ -102,6 +113,7 @@ bool VideoBackend::Stop() { ApplyLifecycleTransition(VideoBackendLifecycleState::Stopping, "Video backend stopping."); const bool stopped = mVideoIODevice->Stop(); + StopOutputCompletionWorker(); if (stopped) ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend stopped."); else @@ -134,9 +146,9 @@ bool VideoBackend::ScheduleOutputFrame(const VideoIOOutputFrame& frame) return mVideoIODevice->ScheduleOutputFrame(frame); } -void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result) +void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) { - mVideoIODevice->AccountForCompletionResult(result); + mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth); } bool VideoBackend::HasInputDevice() const @@ -264,30 +276,156 @@ void VideoBackend::HandleInputFrame(const VideoIOFrame& frame) void VideoBackend::HandleOutputFrameCompletion(const VideoIOCompletion& completion) { - RecordFramePacing(completion.result); - PublishOutputFrameCompleted(completion); + { + std::lock_guard lock(mOutputCompletionMutex); + if (!mOutputCompletionWorkerRunning || mOutputCompletionWorkerStopping) + return; + mPendingOutputCompletions.push_back(completion); + } + mOutputCompletionCondition.notify_one(); +} - VideoIOOutputFrame outputFrame; - if (!BeginOutputFrame(outputFrame)) +void VideoBackend::StartOutputCompletionWorker() +{ + std::lock_guard lock(mOutputCompletionMutex); + if (mOutputCompletionWorkerRunning) return; - const VideoIOState& state = mVideoIODevice->State(); + mPendingOutputCompletions.clear(); + mReadyOutputQueue.Clear(); + mNextReadyOutputFrameIndex = 0; + mOutputCompletionWorkerStopping = false; + mOutputCompletionWorkerRunning = true; + mOutputCompletionWorker = std::thread(&VideoBackend::OutputCompletionWorkerMain, this); +} + +void VideoBackend::StopOutputCompletionWorker() +{ + bool shouldJoin = false; + { + std::lock_guard lock(mOutputCompletionMutex); + if (mOutputCompletionWorkerRunning) + mOutputCompletionWorkerStopping = true; + shouldJoin = mOutputCompletionWorker.joinable(); + } + mOutputCompletionCondition.notify_one(); + + if (shouldJoin) + mOutputCompletionWorker.join(); +} + +void VideoBackend::OutputCompletionWorkerMain() +{ + for (;;) + { + VideoIOCompletion completion; + { + std::unique_lock lock(mOutputCompletionMutex); + mOutputCompletionCondition.wait(lock, [this]() { + return mOutputCompletionWorkerStopping || !mPendingOutputCompletions.empty(); + }); + + if (mPendingOutputCompletions.empty()) + { + if (mOutputCompletionWorkerStopping) + { + mOutputCompletionWorkerRunning = false; + return; + } + continue; + } + + completion = mPendingOutputCompletions.front(); + mPendingOutputCompletions.pop_front(); + } + + ProcessOutputFrameCompletion(completion); + } +} + +void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion) +{ + RecordFramePacing(completion.result); + PublishOutputFrameCompleted(completion); + AccountForCompletionResult(completion.result, mReadyOutputQueue.GetMetrics().depth); + + FillReadyOutputQueue(completion); + if (!ScheduleReadyOutputFrame()) + ScheduleBlackUnderrunFrame(); +} + +bool VideoBackend::FillReadyOutputQueue(const VideoIOCompletion& completion) +{ + RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics(); + bool filledAny = false; + while (metrics.depth < mPlayoutPolicy.targetReadyFrames) + { + if (!RenderReadyOutputFrame(mVideoIODevice->State(), completion)) + return filledAny; + filledAny = true; + metrics = mReadyOutputQueue.GetMetrics(); + } + return true; +} + +bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion) +{ + VideoIOOutputFrame outputFrame; + if (!BeginOutputFrame(outputFrame)) + return false; + bool rendered = true; if (mBridge) rendered = mBridge->RenderScheduledFrame(state, completion, outputFrame); EndOutputFrame(outputFrame); - AccountForCompletionResult(completion.result); if (!rendered) { ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame."); - return; + return false; } - // Schedule the next frame after render work is complete so device-side - // bookkeeping stays with the backend seam and the bridge stays render-only. - if (ScheduleOutputFrame(outputFrame)) - PublishOutputFrameScheduled(outputFrame); + RenderOutputFrame readyFrame; + readyFrame.frame = outputFrame; + readyFrame.frameIndex = ++mNextReadyOutputFrameIndex; + return mReadyOutputQueue.Push(readyFrame); +} + +bool VideoBackend::ScheduleReadyOutputFrame() +{ + RenderOutputFrame readyFrame; + if (!mReadyOutputQueue.TryPop(readyFrame)) + return false; + + if (!ScheduleOutputFrame(readyFrame.frame)) + return false; + + PublishOutputFrameScheduled(readyFrame.frame); + return true; +} + +bool VideoBackend::ScheduleBlackUnderrunFrame() +{ + VideoIOOutputFrame outputFrame; + if (!BeginOutputFrame(outputFrame)) + { + ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: no output frame was available for fallback scheduling."); + return false; + } + + if (outputFrame.bytes != nullptr && outputFrame.rowBytes > 0 && outputFrame.height > 0) + std::memset(outputFrame.bytes, 0, static_cast(outputFrame.rowBytes) * outputFrame.height); + EndOutputFrame(outputFrame); + + if (!ScheduleOutputFrame(outputFrame)) + { + ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: black fallback frame scheduling failed."); + return false; + } + + ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: scheduled black fallback frame."); + PublishOutputFrameScheduled(outputFrame); + return true; } void VideoBackend::RecordFramePacing(VideoIOCompletionResult completionResult) diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h index afc0c88..137d549 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h @@ -1,12 +1,18 @@ #pragma once +#include "RenderOutputQueue.h" #include "VideoBackendLifecycle.h" #include "VideoIOTypes.h" +#include "VideoPlayoutPolicy.h" #include +#include #include +#include #include +#include #include +#include class HealthTelemetry; class OpenGLVideoIOBridge; @@ -34,7 +40,7 @@ public: bool BeginOutputFrame(VideoIOOutputFrame& frame); void EndOutputFrame(VideoIOOutputFrame& frame); bool ScheduleOutputFrame(const VideoIOOutputFrame& frame); - void AccountForCompletionResult(VideoIOCompletionResult result); + void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth); bool HasInputDevice() const; bool HasInputSource() const; @@ -59,6 +65,14 @@ public: private: void HandleInputFrame(const VideoIOFrame& frame); void HandleOutputFrameCompletion(const VideoIOCompletion& completion); + void StartOutputCompletionWorker(); + void StopOutputCompletionWorker(); + void OutputCompletionWorkerMain(); + void ProcessOutputFrameCompletion(const VideoIOCompletion& completion); + bool FillReadyOutputQueue(const VideoIOCompletion& completion); + bool RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion); + bool ScheduleReadyOutputFrame(); + bool ScheduleBlackUnderrunFrame(); void RecordFramePacing(VideoIOCompletionResult completionResult); bool ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message); bool ApplyLifecycleFailure(const std::string& message); @@ -74,8 +88,17 @@ private: HealthTelemetry& mHealthTelemetry; RuntimeEventDispatcher& mRuntimeEventDispatcher; VideoBackendLifecycle mLifecycle; + VideoPlayoutPolicy mPlayoutPolicy; + RenderOutputQueue mReadyOutputQueue; std::unique_ptr mVideoIODevice; std::unique_ptr mBridge; + std::mutex mOutputCompletionMutex; + std::condition_variable mOutputCompletionCondition; + std::deque mPendingOutputCompletions; + std::thread mOutputCompletionWorker; + bool mOutputCompletionWorkerRunning = false; + bool mOutputCompletionWorkerStopping = false; + uint64_t mNextReadyOutputFrameIndex = 0; uint64_t mInputFrameIndex = 0; uint64_t mOutputFrameScheduleIndex = 0; uint64_t mOutputFrameCompletionIndex = 0; diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h index aa75fcf..3f10f5b 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h @@ -105,7 +105,7 @@ public: virtual bool BeginOutputFrame(VideoIOOutputFrame& frame) = 0; virtual void EndOutputFrame(VideoIOOutputFrame& frame) = 0; virtual bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) = 0; - virtual void AccountForCompletionResult(VideoIOCompletionResult result) = 0; + virtual void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) = 0; bool HasInputDevice() const { return State().hasInputDevice; } bool HasInputSource() const { return State().hasInputSource; } diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.cpp b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.cpp index f97af32..c0480b3 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.cpp @@ -16,6 +16,9 @@ void VideoPlayoutScheduler::Configure(int64_t frameDuration, int64_t timeScale, void VideoPlayoutScheduler::Reset() { mScheduledFrameIndex = 0; + mCompletedFrameIndex = 0; + mLateStreak = 0; + mDropStreak = 0; } VideoIOScheduleTime VideoPlayoutScheduler::NextScheduleTime() @@ -29,10 +32,38 @@ VideoIOScheduleTime VideoPlayoutScheduler::NextScheduleTime() return time; } -void VideoPlayoutScheduler::AccountForCompletionResult(VideoIOCompletionResult result) +VideoPlayoutRecoveryDecision VideoPlayoutScheduler::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) { - if (result == VideoIOCompletionResult::DisplayedLate || result == VideoIOCompletionResult::Dropped) - mScheduledFrameIndex += mPolicy.lateOrDropCatchUpFrames; + ++mCompletedFrameIndex; + if (result == VideoIOCompletionResult::DisplayedLate) + ++mLateStreak; + else + mLateStreak = 0; + if (result == VideoIOCompletionResult::Dropped) + ++mDropStreak; + else + mDropStreak = 0; + + const uint64_t measuredLagFrames = MeasureLag(result, readyQueueDepth); + const uint64_t catchUpFrames = measuredLagFrames < mPolicy.lateOrDropCatchUpFrames + ? measuredLagFrames + : mPolicy.lateOrDropCatchUpFrames; + if (catchUpFrames > 0) + mScheduledFrameIndex += catchUpFrames; + + VideoPlayoutRecoveryDecision decision; + decision.result = result; + decision.completedFrameIndex = mCompletedFrameIndex; + decision.scheduledFrameIndex = mScheduledFrameIndex; + decision.readyQueueDepth = readyQueueDepth; + decision.scheduledLeadFrames = mScheduledFrameIndex > mCompletedFrameIndex + ? mScheduledFrameIndex - mCompletedFrameIndex + : 0; + decision.measuredLagFrames = measuredLagFrames; + decision.catchUpFrames = catchUpFrames; + decision.lateStreak = mLateStreak; + decision.dropStreak = mDropStreak; + return decision; } double VideoPlayoutScheduler::FrameBudgetMilliseconds() const @@ -41,3 +72,26 @@ double VideoPlayoutScheduler::FrameBudgetMilliseconds() const ? (static_cast(mFrameDuration) * 1000.0) / static_cast(mTimeScale) : 0.0; } + +uint64_t VideoPlayoutScheduler::MeasureLag(VideoIOCompletionResult result, uint64_t readyQueueDepth) const +{ + if (result != VideoIOCompletionResult::DisplayedLate && result != VideoIOCompletionResult::Dropped) + return 0; + + uint64_t lagFrames = 1; + if (result == VideoIOCompletionResult::DisplayedLate && mLateStreak > lagFrames) + lagFrames = mLateStreak; + if (result == VideoIOCompletionResult::Dropped && mDropStreak * 2 > lagFrames) + lagFrames = mDropStreak * 2; + + if (mCompletedFrameIndex >= mScheduledFrameIndex) + { + const uint64_t scheduleLagFrames = mCompletedFrameIndex - mScheduledFrameIndex + 1; + if (scheduleLagFrames > lagFrames) + lagFrames = scheduleLagFrames; + } + if (readyQueueDepth < mPolicy.targetReadyFrames && mPolicy.targetReadyFrames - readyQueueDepth > lagFrames) + lagFrames = mPolicy.targetReadyFrames - readyQueueDepth; + + return lagFrames; +} diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h index 2c0d03a..738283b 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h @@ -5,6 +5,19 @@ #include +struct VideoPlayoutRecoveryDecision +{ + VideoIOCompletionResult result = VideoIOCompletionResult::Completed; + uint64_t completedFrameIndex = 0; + uint64_t scheduledFrameIndex = 0; + uint64_t readyQueueDepth = 0; + uint64_t scheduledLeadFrames = 0; + uint64_t measuredLagFrames = 0; + uint64_t catchUpFrames = 0; + uint64_t lateStreak = 0; + uint64_t dropStreak = 0; +}; + class VideoPlayoutScheduler { public: @@ -12,15 +25,23 @@ public: void Configure(int64_t frameDuration, int64_t timeScale, const VideoPlayoutPolicy& policy); void Reset(); VideoIOScheduleTime NextScheduleTime(); - void AccountForCompletionResult(VideoIOCompletionResult result); + VideoPlayoutRecoveryDecision AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth = 0); double FrameBudgetMilliseconds() const; uint64_t ScheduledFrameIndex() const { return mScheduledFrameIndex; } + uint64_t CompletedFrameIndex() const { return mCompletedFrameIndex; } + uint64_t LateStreak() const { return mLateStreak; } + uint64_t DropStreak() const { return mDropStreak; } int64_t TimeScale() const { return mTimeScale; } const VideoPlayoutPolicy& Policy() const { return mPolicy; } private: + uint64_t MeasureLag(VideoIOCompletionResult result, uint64_t readyQueueDepth) const; + int64_t mFrameDuration = 0; int64_t mTimeScale = 0; uint64_t mScheduledFrameIndex = 0; + uint64_t mCompletedFrameIndex = 0; + uint64_t mLateStreak = 0; + uint64_t mDropStreak = 0; VideoPlayoutPolicy mPolicy; }; diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp b/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp index 8d07e8a..f241530 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp @@ -498,9 +498,9 @@ void DeckLinkSession::EndOutputFrame(VideoIOOutputFrame& frame) frame.bytes = nullptr; } -void DeckLinkSession::AccountForCompletionResult(VideoIOCompletionResult completionResult) +void DeckLinkSession::AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth) { - mScheduler.AccountForCompletionResult(completionResult); + mScheduler.AccountForCompletionResult(completionResult, readyQueueDepth); } bool DeckLinkSession::ScheduleOutputFrame(const VideoIOOutputFrame& frame) diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h b/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h index 6bd55fe..2c25893 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h @@ -59,7 +59,7 @@ public: const VideoIOState& State() const override { return mState; } VideoIOState& MutableState() override { return mState; } double FrameBudgetMilliseconds() const; - void AccountForCompletionResult(VideoIOCompletionResult completionResult) override; + void AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth) override; bool BeginOutputFrame(VideoIOOutputFrame& frame) override; void EndOutputFrame(VideoIOOutputFrame& frame) override; bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) override; diff --git a/docs/PHASE_7_BACKEND_LIFECYCLE_PLAYOUT_DESIGN.md b/docs/PHASE_7_BACKEND_LIFECYCLE_PLAYOUT_DESIGN.md index 4f88046..b0a1654 100644 --- a/docs/PHASE_7_BACKEND_LIFECYCLE_PLAYOUT_DESIGN.md +++ b/docs/PHASE_7_BACKEND_LIFECYCLE_PLAYOUT_DESIGN.md @@ -2,15 +2,15 @@ This document expands Phase 7 of [ARCHITECTURE_RESILIENCE_REVIEW.md](/c:/Users/Aiden/Documents/GitHub/video-shader-toys/docs/ARCHITECTURE_RESILIENCE_REVIEW.md) into a concrete design target. -Phase 4 made the render thread the sole owner of normal runtime GL work, but output timing is still callback-coupled: DeckLink completion callbacks synchronously request render-thread output production before scheduling the next hardware frame. Phase 7 should make backend lifecycle, buffer policy, playout headroom, and recovery explicit. +Phase 4 made the render thread the sole owner of normal runtime GL work. Phase 7 Step 4 moved DeckLink completion processing onto a backend worker, so the callback no longer directly waits for render-thread output production. Phase 7 Step 5 added a bounded ready-frame queue inside that worker, so scheduling now consumes completed output frames and falls back explicitly on underrun. Phase 7 should make backend lifecycle, buffer policy, playout headroom, and recovery explicit. Phase 5 clarified that live parameter layering stops at final render-state composition. Phase 7 should keep backend lifecycle, output queue ownership, buffer reuse, temporal/feedback resources, and stale-frame/underrun policy outside the persisted/committed/transient parameter model. ## Status - Phase 7 design package: proposed. -- Phase 7 implementation: Step 3 complete. -- Current alignment: `VideoBackend`, `VideoIODevice`, `DeckLinkSession`, `VideoBackendLifecycle`, and `VideoPlayoutScheduler` exist. Phase 4 removed callback-thread GL ownership, but the DeckLink completion path still waits for render-thread output production. +- Phase 7 implementation: Step 6 complete. +- Current alignment: `VideoBackend`, `VideoIODevice`, `DeckLinkSession`, `VideoBackendLifecycle`, and `VideoPlayoutScheduler` exist. Phase 4 removed callback-thread GL ownership, Step 4 moved completion processing onto a backend worker, Step 5 uses `RenderOutputQueue` as the ready-frame handoff inside that worker, and Step 6 replaces fixed late/drop skip-ahead with measured recovery decisions. Current backend footholds: @@ -18,7 +18,7 @@ Current backend footholds: - `DeckLinkSession` owns DeckLink device handles, frame pool creation, preroll, keyer configuration, and scheduled playback. - `VideoPlayoutPolicy` names current frame pool, preroll, ready-frame, underrun, and catch-up policy defaults. - `RenderOutputQueue` names the future bounded ready-output-frame handoff and has pure queue tests. -- `VideoPlayoutScheduler` owns basic schedule time generation and simple late/drop skip-ahead behavior. +- `VideoPlayoutScheduler` owns schedule time generation, completion indexing, late/drop streaks, ready-queue pressure input, and measured recovery decisions. - `OpenGLVideoIOBridge` is the current adapter between `VideoBackend` and `RenderEngine`. - `HealthTelemetry` receives some signal, render, and pacing stats. @@ -28,7 +28,7 @@ The current output path works only while render/readback stays comfortably insid The resilience review calls this the main remaining live-resilience risk after Phase 4: -- output playout is still effectively render-on-demand from the DeckLink completion callback +- output playout is still effectively filled on demand by a backend completion worker, but scheduling now consumes a bounded ready-frame queue - buffer pool size and preroll depth are not sourced from one policy - late/dropped recovery is a fixed skip rule - backend lifecycle is imperative rather than represented as explicit states @@ -260,22 +260,36 @@ Stop producing frames directly in the completion callback path. Transitional target: -- callback wakes/schedules a backend worker -- worker consumes ready frames +- [x] callback wakes/schedules a backend worker +- [x] worker consumes ready frames Final target: - callback only records, recycles, dequeues, schedules +Current implementation: + +- `VideoBackend::HandleOutputFrameCompletion(...)` now enqueues completion work and wakes an output-completion worker. +- The output-completion worker drains pending completions and runs the existing render/schedule path. +- This preserves behavior while removing the direct callback-thread wait on render-thread output production. +- Step 5 now makes this worker consume ready frames from `RenderOutputQueue`; Step 4 remains the boundary that keeps output completion callbacks from doing render production directly. + ### Step 5. Make Render Produce Ahead Teach render/output code to keep the ready queue filled to target headroom. Initial target: -- render thread produces on demand until queue has target depth -- callback does not synchronously wait for fresh render -- stale/black fallback is explicit on underrun +- [x] render thread produces on demand until queue has target depth +- [x] callback does not synchronously wait for fresh render +- [x] stale/black fallback is explicit on underrun + +Current implementation: + +- The backend output-completion worker fills `RenderOutputQueue` to `VideoPlayoutPolicy::targetReadyFrames`. +- Scheduling now pops a ready frame from `RenderOutputQueue` instead of directly scheduling the freshly rendered frame. +- If no ready frame can be produced, the worker schedules an explicit black fallback frame and reports degraded lifecycle state. +- This is still demand-filled by the backend worker; a future pass can make render production more proactive or timer/pressure driven. ### Step 6. Replace Fixed Late/Drop Recovery @@ -283,8 +297,16 @@ Replace fixed `+2` schedule-index recovery with measured lag/headroom accounting Initial target: -- track scheduled index, completed index, queue depth, late streak, drop streak -- recovery decisions use measured lag +- [x] track scheduled index, completed index, queue depth, late streak, drop streak +- [x] recovery decisions use measured lag + +Current implementation: + +- `VideoPlayoutRecoveryDecision` reports completion result, completed index, scheduled index, ready queue depth, scheduled lead, measured lag, catch-up frames, late streak, and drop streak. +- `VideoPlayoutScheduler::AccountForCompletionResult(...)` now accepts ready queue depth and returns a recovery decision. +- Recovery is measured from late/drop streaks, scheduled lead, and ready queue pressure, then capped by `VideoPlayoutPolicy::lateOrDropCatchUpFrames`. +- `VideoBackend` passes the current ready queue depth into the video device completion-accounting call. +- `VideoPlayoutSchedulerTests` cover measured late recovery, measured drop recovery, policy caps, completed-index tracking, and streak clearing. ### Step 7. Route Backend Health Structurally @@ -339,10 +361,10 @@ Phase 7 can be considered complete once the project can say: - [x] backend lifecycle states and transitions are explicit - [x] playout policy owns preroll, pool size, headroom, and underrun behavior -- [ ] output callbacks no longer synchronously wait for render production -- [ ] render produces completed output frames into a bounded queue -- [ ] underrun behavior is explicit and observable -- [ ] late/drop recovery is measured rather than fixed skip-only +- [x] output callbacks no longer synchronously wait for render production +- [x] render produces completed output frames into a bounded queue +- [x] underrun behavior is explicit and observable +- [x] late/drop recovery is measured rather than fixed skip-only - [ ] backend health reports lifecycle, queue, underrun, late, and dropped state - [ ] queue/lifecycle/scheduler behavior has non-DeckLink tests diff --git a/tests/VideoIODeviceFakeTests.cpp b/tests/VideoIODeviceFakeTests.cpp index 6dd3788..d7942d9 100644 --- a/tests/VideoIODeviceFakeTests.cpp +++ b/tests/VideoIODeviceFakeTests.cpp @@ -92,13 +92,15 @@ public: return true; } - void AccountForCompletionResult(VideoIOCompletionResult result) override + void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) override { mLastCompletion = result; + mLastReadyQueueDepth = readyQueueDepth; } unsigned ScheduledFrames() const { return mScheduledFrames; } VideoIOCompletionResult LastCompletion() const { return mLastCompletion; } + uint64_t LastReadyQueueDepth() const { return mLastReadyQueueDepth; } private: VideoIOState mState; @@ -108,6 +110,7 @@ private: std::array mOutputBytes = {}; unsigned mScheduledFrames = 0; VideoIOCompletionResult mLastCompletion = VideoIOCompletionResult::Unknown; + uint64_t mLastReadyQueueDepth = 0; }; } @@ -132,13 +135,14 @@ int main() VideoIOOutputFrame outputFrame; Expect(device.BeginOutputFrame(outputFrame), "fake output frame can be acquired"); device.EndOutputFrame(outputFrame); - device.AccountForCompletionResult(VideoIOCompletionResult::Completed); + device.AccountForCompletionResult(VideoIOCompletionResult::Completed, 2); Expect(device.ScheduleOutputFrame(outputFrame), "fake output frame can be scheduled"); Expect(inputSeen, "fake input callback emits generic frame"); Expect(outputSeen, "fake output callback emits generic completion"); Expect(device.ScheduledFrames() == 1, "fake backend schedules one frame"); Expect(device.LastCompletion() == VideoIOCompletionResult::Completed, "fake backend records generic completion"); + Expect(device.LastReadyQueueDepth() == 2, "fake backend records ready queue depth"); if (gFailures != 0) { diff --git a/tests/VideoPlayoutSchedulerTests.cpp b/tests/VideoPlayoutSchedulerTests.cpp index 40bf66c..957e21e 100644 --- a/tests/VideoPlayoutSchedulerTests.cpp +++ b/tests/VideoPlayoutSchedulerTests.cpp @@ -37,30 +37,51 @@ void TestScheduleAdvancesFromZero() Expect(third.streamTime == 2002, "third frame advances by two durations"); } -void TestLateAndDroppedSkipAhead() +void TestLateAndDroppedRecoveryUsesMeasuredPressure() { VideoPlayoutScheduler scheduler; scheduler.Configure(1000, 50000); (void)scheduler.NextScheduleTime(); - scheduler.AccountForCompletionResult(VideoIOCompletionResult::DisplayedLate); - Expect(scheduler.NextScheduleTime().streamTime == 3000, "late completion preserves the existing two-frame skip policy"); + VideoPlayoutRecoveryDecision lateDecision = scheduler.AccountForCompletionResult(VideoIOCompletionResult::DisplayedLate, 2); + Expect(lateDecision.catchUpFrames == 1, "single late completion catches up by measured one-frame lag"); + Expect(lateDecision.lateStreak == 1, "late completion increments late streak"); + Expect(scheduler.NextScheduleTime().streamTime == 2000, "single late recovery advances by measured lag"); - scheduler.AccountForCompletionResult(VideoIOCompletionResult::Dropped); - Expect(scheduler.NextScheduleTime().streamTime == 6000, "dropped completion preserves the existing two-frame skip policy"); + VideoPlayoutRecoveryDecision dropDecision = scheduler.AccountForCompletionResult(VideoIOCompletionResult::Dropped, 2); + Expect(dropDecision.catchUpFrames == 2, "dropped completion catches up by measured drop pressure"); + Expect(dropDecision.lateStreak == 0, "dropped completion resets late streak"); + Expect(dropDecision.dropStreak == 1, "dropped completion increments drop streak"); + Expect(scheduler.NextScheduleTime().streamTime == 5000, "drop recovery advances by measured lag"); } -void TestLateAndDroppedRecoveryUsesPolicy() +void TestMeasuredRecoveryIsCappedByPolicy() { VideoPlayoutPolicy policy; - policy.lateOrDropCatchUpFrames = 4; + policy.lateOrDropCatchUpFrames = 1; VideoPlayoutScheduler scheduler; scheduler.Configure(1000, 50000, policy); (void)scheduler.NextScheduleTime(); - scheduler.AccountForCompletionResult(VideoIOCompletionResult::Dropped); - Expect(scheduler.NextScheduleTime().streamTime == 5000, "drop recovery uses policy catch-up frame count"); + VideoPlayoutRecoveryDecision decision = scheduler.AccountForCompletionResult(VideoIOCompletionResult::Dropped, 0); + Expect(decision.measuredLagFrames > decision.catchUpFrames, "policy caps measured recovery"); + Expect(decision.catchUpFrames == 1, "drop recovery obeys policy cap"); + Expect(scheduler.NextScheduleTime().streamTime == 2000, "capped recovery advances by one frame"); +} + +void TestCleanCompletionTracksCompletedIndexAndClearsStreaks() +{ + VideoPlayoutScheduler scheduler; + scheduler.Configure(1000, 50000); + + (void)scheduler.NextScheduleTime(); + (void)scheduler.AccountForCompletionResult(VideoIOCompletionResult::DisplayedLate, 2); + VideoPlayoutRecoveryDecision decision = scheduler.AccountForCompletionResult(VideoIOCompletionResult::Completed, 2); + Expect(decision.completedFrameIndex == 2, "completion accounting tracks completed index"); + Expect(decision.catchUpFrames == 0, "clean completion does not catch up"); + Expect(decision.lateStreak == 0, "clean completion clears late streak"); + Expect(decision.dropStreak == 0, "clean completion keeps drop streak clear"); } void TestPolicyNormalization() @@ -94,8 +115,9 @@ void TestFrameBudgets() int main() { TestScheduleAdvancesFromZero(); - TestLateAndDroppedSkipAhead(); - TestLateAndDroppedRecoveryUsesPolicy(); + TestLateAndDroppedRecoveryUsesMeasuredPressure(); + TestMeasuredRecoveryIsCappedByPolicy(); + TestCleanCompletionTracksCompletedIndexAndClearsStreaks(); TestPolicyNormalization(); TestFrameBudgets();