diff --git a/apps/LoopThroughWithOpenGLCompositing/runtime/presentation/RuntimeStatePresenter.cpp b/apps/LoopThroughWithOpenGLCompositing/runtime/presentation/RuntimeStatePresenter.cpp index 6c15e68..92101ef 100644 --- a/apps/LoopThroughWithOpenGLCompositing/runtime/presentation/RuntimeStatePresenter.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/runtime/presentation/RuntimeStatePresenter.cpp @@ -80,6 +80,35 @@ JsonValue RuntimeStatePresenter::BuildRuntimeStateValue(const RuntimeStore& runt performance.set("flushedFrameCount", JsonValue(static_cast(telemetrySnapshot.performance.flushedFrameCount))); root.set("performance", performance); + JsonValue readyQueue = JsonValue::MakeObject(); + readyQueue.set("depth", JsonValue(static_cast(telemetrySnapshot.backendPlayout.readyQueueDepth))); + readyQueue.set("capacity", JsonValue(static_cast(telemetrySnapshot.backendPlayout.readyQueueCapacity))); + readyQueue.set("pushedCount", JsonValue(static_cast(telemetrySnapshot.backendPlayout.readyQueuePushedCount))); + readyQueue.set("poppedCount", JsonValue(static_cast(telemetrySnapshot.backendPlayout.readyQueuePoppedCount))); + readyQueue.set("droppedCount", JsonValue(static_cast(telemetrySnapshot.backendPlayout.readyQueueDroppedCount))); + readyQueue.set("underrunCount", JsonValue(static_cast(telemetrySnapshot.backendPlayout.readyQueueUnderrunCount))); + + JsonValue recovery = JsonValue::MakeObject(); + recovery.set("completionResult", JsonValue(telemetrySnapshot.backendPlayout.completionResult)); + recovery.set("completedFrameIndex", JsonValue(static_cast(telemetrySnapshot.backendPlayout.completedFrameIndex))); + recovery.set("scheduledFrameIndex", JsonValue(static_cast(telemetrySnapshot.backendPlayout.scheduledFrameIndex))); + recovery.set("scheduledLeadFrames", JsonValue(static_cast(telemetrySnapshot.backendPlayout.scheduledLeadFrames))); + recovery.set("measuredLagFrames", JsonValue(static_cast(telemetrySnapshot.backendPlayout.measuredLagFrames))); + recovery.set("catchUpFrames", JsonValue(static_cast(telemetrySnapshot.backendPlayout.catchUpFrames))); + recovery.set("lateStreak", JsonValue(static_cast(telemetrySnapshot.backendPlayout.lateStreak))); + recovery.set("dropStreak", JsonValue(static_cast(telemetrySnapshot.backendPlayout.dropStreak))); + + JsonValue backendPlayout = JsonValue::MakeObject(); + backendPlayout.set("lifecycleState", JsonValue(telemetrySnapshot.backendPlayout.lifecycleState)); + backendPlayout.set("degraded", JsonValue(telemetrySnapshot.backendPlayout.degraded)); + backendPlayout.set("statusMessage", JsonValue(telemetrySnapshot.backendPlayout.statusMessage)); + backendPlayout.set("lateFrameCount", JsonValue(static_cast(telemetrySnapshot.backendPlayout.lateFrameCount))); + backendPlayout.set("droppedFrameCount", JsonValue(static_cast(telemetrySnapshot.backendPlayout.droppedFrameCount))); + backendPlayout.set("flushedFrameCount", JsonValue(static_cast(telemetrySnapshot.backendPlayout.flushedFrameCount))); + backendPlayout.set("readyQueue", readyQueue); + backendPlayout.set("recovery", recovery); + root.set("backendPlayout", backendPlayout); + JsonValue eventQueue = JsonValue::MakeObject(); eventQueue.set("name", JsonValue(telemetrySnapshot.runtimeEvents.queue.queueName)); eventQueue.set("depth", JsonValue(static_cast(telemetrySnapshot.runtimeEvents.queue.depth))); diff --git a/apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.cpp b/apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.cpp index 9cfb600..bd69917 100644 --- a/apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.cpp @@ -207,6 +207,72 @@ bool HealthTelemetry::TryRecordPersistenceWriteResult(bool succeeded, const std: return true; } +void HealthTelemetry::RecordBackendPlayoutHealth(const std::string& lifecycleState, const std::string& completionResult, + std::size_t readyQueueDepth, std::size_t readyQueueCapacity, uint64_t readyQueuePushedCount, + uint64_t readyQueuePoppedCount, uint64_t readyQueueDroppedCount, uint64_t readyQueueUnderrunCount, + uint64_t completedFrameIndex, uint64_t scheduledFrameIndex, uint64_t scheduledLeadFrames, + uint64_t measuredLagFrames, uint64_t catchUpFrames, uint64_t lateStreak, uint64_t dropStreak, + uint64_t lateFrameCount, uint64_t droppedFrameCount, uint64_t flushedFrameCount, + bool degraded, const std::string& statusMessage) +{ + std::lock_guard lock(mMutex); + mBackendPlayout.lifecycleState = lifecycleState; + mBackendPlayout.completionResult = completionResult; + mBackendPlayout.readyQueueDepth = readyQueueDepth; + mBackendPlayout.readyQueueCapacity = readyQueueCapacity; + mBackendPlayout.readyQueuePushedCount = readyQueuePushedCount; + mBackendPlayout.readyQueuePoppedCount = readyQueuePoppedCount; + mBackendPlayout.readyQueueDroppedCount = readyQueueDroppedCount; + mBackendPlayout.readyQueueUnderrunCount = readyQueueUnderrunCount; + mBackendPlayout.completedFrameIndex = completedFrameIndex; + mBackendPlayout.scheduledFrameIndex = scheduledFrameIndex; + mBackendPlayout.scheduledLeadFrames = scheduledLeadFrames; + mBackendPlayout.measuredLagFrames = measuredLagFrames; + mBackendPlayout.catchUpFrames = catchUpFrames; + mBackendPlayout.lateStreak = lateStreak; + mBackendPlayout.dropStreak = dropStreak; + mBackendPlayout.lateFrameCount = lateFrameCount; + mBackendPlayout.droppedFrameCount = droppedFrameCount; + mBackendPlayout.flushedFrameCount = flushedFrameCount; + mBackendPlayout.degraded = degraded; + mBackendPlayout.statusMessage = statusMessage; +} + +bool HealthTelemetry::TryRecordBackendPlayoutHealth(const std::string& lifecycleState, const std::string& completionResult, + std::size_t readyQueueDepth, std::size_t readyQueueCapacity, uint64_t readyQueuePushedCount, + uint64_t readyQueuePoppedCount, uint64_t readyQueueDroppedCount, uint64_t readyQueueUnderrunCount, + uint64_t completedFrameIndex, uint64_t scheduledFrameIndex, uint64_t scheduledLeadFrames, + uint64_t measuredLagFrames, uint64_t catchUpFrames, uint64_t lateStreak, uint64_t dropStreak, + uint64_t lateFrameCount, uint64_t droppedFrameCount, uint64_t flushedFrameCount, + bool degraded, const std::string& statusMessage) +{ + std::unique_lock lock(mMutex, std::try_to_lock); + if (!lock.owns_lock()) + return false; + + mBackendPlayout.lifecycleState = lifecycleState; + mBackendPlayout.completionResult = completionResult; + mBackendPlayout.readyQueueDepth = readyQueueDepth; + mBackendPlayout.readyQueueCapacity = readyQueueCapacity; + mBackendPlayout.readyQueuePushedCount = readyQueuePushedCount; + mBackendPlayout.readyQueuePoppedCount = readyQueuePoppedCount; + mBackendPlayout.readyQueueDroppedCount = readyQueueDroppedCount; + mBackendPlayout.readyQueueUnderrunCount = readyQueueUnderrunCount; + mBackendPlayout.completedFrameIndex = completedFrameIndex; + mBackendPlayout.scheduledFrameIndex = scheduledFrameIndex; + mBackendPlayout.scheduledLeadFrames = scheduledLeadFrames; + mBackendPlayout.measuredLagFrames = measuredLagFrames; + mBackendPlayout.catchUpFrames = catchUpFrames; + mBackendPlayout.lateStreak = lateStreak; + mBackendPlayout.dropStreak = dropStreak; + mBackendPlayout.lateFrameCount = lateFrameCount; + mBackendPlayout.droppedFrameCount = droppedFrameCount; + mBackendPlayout.flushedFrameCount = flushedFrameCount; + mBackendPlayout.degraded = degraded; + mBackendPlayout.statusMessage = statusMessage; + return true; +} + HealthTelemetry::SignalStatusSnapshot HealthTelemetry::GetSignalStatusSnapshot() const { std::lock_guard lock(mMutex); @@ -237,6 +303,12 @@ HealthTelemetry::PersistenceSnapshot HealthTelemetry::GetPersistenceSnapshot() c return mPersistence; } +HealthTelemetry::BackendPlayoutSnapshot HealthTelemetry::GetBackendPlayoutSnapshot() const +{ + std::lock_guard lock(mMutex); + return mBackendPlayout; +} + HealthTelemetry::Snapshot HealthTelemetry::GetSnapshot() const { std::lock_guard lock(mMutex); @@ -247,5 +319,6 @@ HealthTelemetry::Snapshot HealthTelemetry::GetSnapshot() const snapshot.performance = mPerformance; snapshot.runtimeEvents = mRuntimeEvents; snapshot.persistence = mPersistence; + snapshot.backendPlayout = mBackendPlayout; return snapshot; } diff --git a/apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.h b/apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.h index 3d609b9..0c86b42 100644 --- a/apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.h +++ b/apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.h @@ -81,6 +81,30 @@ public: std::string lastErrorMessage; }; + struct BackendPlayoutSnapshot + { + std::string lifecycleState = "NotStarted"; + std::string completionResult = "Unknown"; + std::size_t readyQueueDepth = 0; + std::size_t readyQueueCapacity = 0; + uint64_t readyQueuePushedCount = 0; + uint64_t readyQueuePoppedCount = 0; + uint64_t readyQueueDroppedCount = 0; + uint64_t readyQueueUnderrunCount = 0; + uint64_t completedFrameIndex = 0; + uint64_t scheduledFrameIndex = 0; + uint64_t scheduledLeadFrames = 0; + uint64_t measuredLagFrames = 0; + uint64_t catchUpFrames = 0; + uint64_t lateStreak = 0; + uint64_t dropStreak = 0; + uint64_t lateFrameCount = 0; + uint64_t droppedFrameCount = 0; + uint64_t flushedFrameCount = 0; + bool degraded = false; + std::string statusMessage; + }; + struct Snapshot { SignalStatusSnapshot signal; @@ -88,6 +112,7 @@ public: PerformanceSnapshot performance; RuntimeEventMetricsSnapshot runtimeEvents; PersistenceSnapshot persistence; + BackendPlayoutSnapshot backendPlayout; }; HealthTelemetry() = default; @@ -125,11 +150,27 @@ public: bool TryRecordPersistenceWriteResult(bool succeeded, const std::string& targetKind, const std::string& targetPath, const std::string& reason, const std::string& errorMessage, bool newerRequestPending); + void RecordBackendPlayoutHealth(const std::string& lifecycleState, const std::string& completionResult, + std::size_t readyQueueDepth, std::size_t readyQueueCapacity, uint64_t readyQueuePushedCount, + uint64_t readyQueuePoppedCount, uint64_t readyQueueDroppedCount, uint64_t readyQueueUnderrunCount, + uint64_t completedFrameIndex, uint64_t scheduledFrameIndex, uint64_t scheduledLeadFrames, + uint64_t measuredLagFrames, uint64_t catchUpFrames, uint64_t lateStreak, uint64_t dropStreak, + uint64_t lateFrameCount, uint64_t droppedFrameCount, uint64_t flushedFrameCount, + bool degraded, const std::string& statusMessage); + bool TryRecordBackendPlayoutHealth(const std::string& lifecycleState, const std::string& completionResult, + std::size_t readyQueueDepth, std::size_t readyQueueCapacity, uint64_t readyQueuePushedCount, + uint64_t readyQueuePoppedCount, uint64_t readyQueueDroppedCount, uint64_t readyQueueUnderrunCount, + uint64_t completedFrameIndex, uint64_t scheduledFrameIndex, uint64_t scheduledLeadFrames, + uint64_t measuredLagFrames, uint64_t catchUpFrames, uint64_t lateStreak, uint64_t dropStreak, + uint64_t lateFrameCount, uint64_t droppedFrameCount, uint64_t flushedFrameCount, + bool degraded, const std::string& statusMessage); + SignalStatusSnapshot GetSignalStatusSnapshot() const; VideoIOStatusSnapshot GetVideoIOStatusSnapshot() const; PerformanceSnapshot GetPerformanceSnapshot() const; RuntimeEventMetricsSnapshot GetRuntimeEventMetricsSnapshot() const; PersistenceSnapshot GetPersistenceSnapshot() const; + BackendPlayoutSnapshot GetBackendPlayoutSnapshot() const; Snapshot GetSnapshot() const; private: @@ -139,4 +180,5 @@ private: PerformanceSnapshot mPerformance; RuntimeEventMetricsSnapshot mRuntimeEvents; PersistenceSnapshot mPersistence; + BackendPlayoutSnapshot mBackendPlayout; }; diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp index 0d11f35..57fa22e 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp @@ -146,9 +146,9 @@ bool VideoBackend::ScheduleOutputFrame(const VideoIOOutputFrame& frame) return mVideoIODevice->ScheduleOutputFrame(frame); } -void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) +VideoPlayoutRecoveryDecision VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) { - mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth); + return mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth); } bool VideoBackend::HasInputDevice() const @@ -347,11 +347,38 @@ void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& complet { RecordFramePacing(completion.result); PublishOutputFrameCompleted(completion); - AccountForCompletionResult(completion.result, mReadyOutputQueue.GetMetrics().depth); + const VideoPlayoutRecoveryDecision recoveryDecision = AccountForCompletionResult(completion.result, mReadyOutputQueue.GetMetrics().depth); FillReadyOutputQueue(completion); if (!ScheduleReadyOutputFrame()) ScheduleBlackUnderrunFrame(); + RecordBackendPlayoutHealth(completion.result, recoveryDecision); +} + +void VideoBackend::RecordBackendPlayoutHealth(VideoIOCompletionResult result, const VideoPlayoutRecoveryDecision& recoveryDecision) +{ + const RenderOutputQueueMetrics queueMetrics = mReadyOutputQueue.GetMetrics(); + mHealthTelemetry.TryRecordBackendPlayoutHealth( + VideoBackendLifecycle::StateName(mLifecycle.State()), + CompletionResultName(result), + queueMetrics.depth, + queueMetrics.capacity, + queueMetrics.pushedCount, + queueMetrics.poppedCount, + queueMetrics.droppedCount, + queueMetrics.underrunCount, + recoveryDecision.completedFrameIndex, + recoveryDecision.scheduledFrameIndex, + recoveryDecision.scheduledLeadFrames, + recoveryDecision.measuredLagFrames, + recoveryDecision.catchUpFrames, + recoveryDecision.lateStreak, + recoveryDecision.dropStreak, + mLateFrameCount, + mDroppedFrameCount, + mFlushedFrameCount, + mLifecycle.State() == VideoBackendLifecycleState::Degraded, + StatusMessage()); } bool VideoBackend::FillReadyOutputQueue(const VideoIOCompletion& completion) diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h index 137d549..9b18f8a 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h @@ -40,7 +40,8 @@ public: bool BeginOutputFrame(VideoIOOutputFrame& frame); void EndOutputFrame(VideoIOOutputFrame& frame); bool ScheduleOutputFrame(const VideoIOOutputFrame& frame); - void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth); + VideoPlayoutRecoveryDecision AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth); + void RecordBackendPlayoutHealth(VideoIOCompletionResult result, const VideoPlayoutRecoveryDecision& recoveryDecision); bool HasInputDevice() const; bool HasInputSource() const; diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h index 3f10f5b..af75b57 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h @@ -86,6 +86,19 @@ struct VideoIOScheduleTime uint64_t frameIndex = 0; }; +struct VideoPlayoutRecoveryDecision +{ + VideoIOCompletionResult result = VideoIOCompletionResult::Completed; + uint64_t completedFrameIndex = 0; + uint64_t scheduledFrameIndex = 0; + uint64_t readyQueueDepth = 0; + uint64_t scheduledLeadFrames = 0; + uint64_t measuredLagFrames = 0; + uint64_t catchUpFrames = 0; + uint64_t lateStreak = 0; + uint64_t dropStreak = 0; +}; + class VideoIODevice { public: @@ -105,7 +118,7 @@ public: virtual bool BeginOutputFrame(VideoIOOutputFrame& frame) = 0; virtual void EndOutputFrame(VideoIOOutputFrame& frame) = 0; virtual bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) = 0; - virtual void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) = 0; + virtual VideoPlayoutRecoveryDecision AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) = 0; bool HasInputDevice() const { return State().hasInputDevice; } bool HasInputSource() const { return State().hasInputSource; } diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h index 738283b..d55aa1d 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h @@ -5,19 +5,6 @@ #include -struct VideoPlayoutRecoveryDecision -{ - VideoIOCompletionResult result = VideoIOCompletionResult::Completed; - uint64_t completedFrameIndex = 0; - uint64_t scheduledFrameIndex = 0; - uint64_t readyQueueDepth = 0; - uint64_t scheduledLeadFrames = 0; - uint64_t measuredLagFrames = 0; - uint64_t catchUpFrames = 0; - uint64_t lateStreak = 0; - uint64_t dropStreak = 0; -}; - class VideoPlayoutScheduler { public: diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp b/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp index f241530..2b31217 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp @@ -498,9 +498,9 @@ void DeckLinkSession::EndOutputFrame(VideoIOOutputFrame& frame) frame.bytes = nullptr; } -void DeckLinkSession::AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth) +VideoPlayoutRecoveryDecision DeckLinkSession::AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth) { - mScheduler.AccountForCompletionResult(completionResult, readyQueueDepth); + return mScheduler.AccountForCompletionResult(completionResult, readyQueueDepth); } bool DeckLinkSession::ScheduleOutputFrame(const VideoIOOutputFrame& frame) diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h b/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h index 2c25893..350742a 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h @@ -59,7 +59,7 @@ public: const VideoIOState& State() const override { return mState; } VideoIOState& MutableState() override { return mState; } double FrameBudgetMilliseconds() const; - void AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth) override; + VideoPlayoutRecoveryDecision AccountForCompletionResult(VideoIOCompletionResult completionResult, uint64_t readyQueueDepth) override; bool BeginOutputFrame(VideoIOOutputFrame& frame) override; void EndOutputFrame(VideoIOOutputFrame& frame) override; bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) override; diff --git a/docs/PHASE_7_BACKEND_LIFECYCLE_PLAYOUT_DESIGN.md b/docs/PHASE_7_BACKEND_LIFECYCLE_PLAYOUT_DESIGN.md index b0a1654..0d5d413 100644 --- a/docs/PHASE_7_BACKEND_LIFECYCLE_PLAYOUT_DESIGN.md +++ b/docs/PHASE_7_BACKEND_LIFECYCLE_PLAYOUT_DESIGN.md @@ -9,8 +9,8 @@ Phase 5 clarified that live parameter layering stops at final render-state compo ## Status - Phase 7 design package: proposed. -- Phase 7 implementation: Step 6 complete. -- Current alignment: `VideoBackend`, `VideoIODevice`, `DeckLinkSession`, `VideoBackendLifecycle`, and `VideoPlayoutScheduler` exist. Phase 4 removed callback-thread GL ownership, Step 4 moved completion processing onto a backend worker, Step 5 uses `RenderOutputQueue` as the ready-frame handoff inside that worker, and Step 6 replaces fixed late/drop skip-ahead with measured recovery decisions. +- Phase 7 implementation: complete. +- Current alignment: `VideoBackend`, `VideoIODevice`, `DeckLinkSession`, `VideoBackendLifecycle`, and `VideoPlayoutScheduler` exist. Phase 4 removed callback-thread GL ownership, Step 4 moved completion processing onto a backend worker, Step 5 uses `RenderOutputQueue` as the ready-frame handoff inside that worker, Step 6 replaces fixed late/drop skip-ahead with measured recovery decisions, and Step 7 reports backend playout health through `HealthTelemetry`. Current backend footholds: @@ -20,7 +20,7 @@ Current backend footholds: - `RenderOutputQueue` names the future bounded ready-output-frame handoff and has pure queue tests. - `VideoPlayoutScheduler` owns schedule time generation, completion indexing, late/drop streaks, ready-queue pressure input, and measured recovery decisions. - `OpenGLVideoIOBridge` is the current adapter between `VideoBackend` and `RenderEngine`. -- `HealthTelemetry` receives some signal, render, and pacing stats. +- `HealthTelemetry` receives signal, render, pacing, lifecycle, queue, underrun, late/drop, and scheduler recovery observations. ## Why Phase 7 Exists @@ -312,6 +312,20 @@ Current implementation: Publish backend lifecycle, queue depth, underrun, late/drop, and degraded-state observations through `HealthTelemetry`. +Initial target: + +- [x] backend lifecycle state is visible in health telemetry +- [x] ready queue depth, capacity, drops, and underruns are visible +- [x] late/drop streaks and scheduler recovery decisions are visible +- [x] runtime-state JSON exposes the backend playout health snapshot + +Current implementation: + +- `HealthTelemetry::BackendPlayoutSnapshot` captures lifecycle state, completion result, ready queue metrics, scheduler indices, scheduled lead, measured lag, catch-up frames, late/drop streaks, aggregate late/drop/flushed counts, degraded state, and status message. +- `VideoBackend::RecordBackendPlayoutHealth(...)` samples `RenderOutputQueue` metrics after each processed output completion and reports the latest scheduler recovery decision. +- `RuntimeStatePresenter` publishes the snapshot as `backendPlayout`, including `readyQueue` and `recovery` sections. +- `HealthTelemetryTests` cover backend playout health recording, try-record behavior, and inclusion in the full health snapshot. + ## Testing Strategy Recommended tests: @@ -365,8 +379,8 @@ Phase 7 can be considered complete once the project can say: - [x] render produces completed output frames into a bounded queue - [x] underrun behavior is explicit and observable - [x] late/drop recovery is measured rather than fixed skip-only -- [ ] backend health reports lifecycle, queue, underrun, late, and dropped state -- [ ] queue/lifecycle/scheduler behavior has non-DeckLink tests +- [x] backend health reports lifecycle, queue, underrun, late, and dropped state +- [x] queue/lifecycle/scheduler behavior has non-DeckLink tests ## Open Questions diff --git a/tests/HealthTelemetryTests.cpp b/tests/HealthTelemetryTests.cpp index 9a269f6..b1a626a 100644 --- a/tests/HealthTelemetryTests.cpp +++ b/tests/HealthTelemetryTests.cpp @@ -76,6 +76,77 @@ void TestPersistenceWriteHealth() Expect(persistence.lastWriteSucceeded, "persistence health records successful write state"); Expect(!persistence.unsavedChanges, "persistence health clears unsaved changes after latest successful write with no pending request"); } + +void TestBackendPlayoutHealth() +{ + HealthTelemetry telemetry; + telemetry.RecordBackendPlayoutHealth( + "Degraded", + "Dropped", + 1, + 4, + 12, + 10, + 2, + 1, + 8, + 11, + 3, + 2, + 2, + 1, + 2, + 5, + 3, + 1, + true, + "Output underrun"); + + const HealthTelemetry::BackendPlayoutSnapshot playout = telemetry.GetBackendPlayoutSnapshot(); + Expect(playout.lifecycleState == "Degraded", "backend playout health stores lifecycle state"); + Expect(playout.completionResult == "Dropped", "backend playout health stores completion result"); + Expect(playout.readyQueueDepth == 1, "backend playout health stores ready queue depth"); + Expect(playout.readyQueueCapacity == 4, "backend playout health stores ready queue capacity"); + Expect(playout.readyQueueDroppedCount == 2, "backend playout health stores queue dropped count"); + Expect(playout.readyQueueUnderrunCount == 1, "backend playout health stores queue underrun count"); + Expect(playout.completedFrameIndex == 8, "backend playout health stores completed index"); + Expect(playout.scheduledFrameIndex == 11, "backend playout health stores scheduled index"); + Expect(playout.measuredLagFrames == 2, "backend playout health stores measured lag"); + Expect(playout.catchUpFrames == 2, "backend playout health stores catch-up frames"); + Expect(playout.lateStreak == 1, "backend playout health stores late streak"); + Expect(playout.dropStreak == 2, "backend playout health stores drop streak"); + Expect(playout.lateFrameCount == 5, "backend playout health stores late frame count"); + Expect(playout.droppedFrameCount == 3, "backend playout health stores dropped frame count"); + Expect(playout.flushedFrameCount == 1, "backend playout health stores flushed frame count"); + Expect(playout.degraded, "backend playout health stores degraded state"); + Expect(playout.statusMessage == "Output underrun", "backend playout health stores status message"); + + Expect(telemetry.TryRecordBackendPlayoutHealth( + "Running", + "Completed", + 2, + 4, + 13, + 11, + 2, + 1, + 9, + 12, + 3, + 0, + 0, + 0, + 0, + 5, + 3, + 1, + false, + ""), + "try backend playout health succeeds when uncontended"); + const HealthTelemetry::Snapshot snapshot = telemetry.GetSnapshot(); + Expect(snapshot.backendPlayout.lifecycleState == "Running", "full health snapshot includes backend playout state"); + Expect(!snapshot.backendPlayout.degraded, "full health snapshot includes backend degraded state"); +} } int main() @@ -84,6 +155,7 @@ int main() TestRuntimeEventDispatchStats(); TestRuntimeEventTryRecord(); TestPersistenceWriteHealth(); + TestBackendPlayoutHealth(); if (gFailures != 0) { diff --git a/tests/VideoIODeviceFakeTests.cpp b/tests/VideoIODeviceFakeTests.cpp index d7942d9..a74fe62 100644 --- a/tests/VideoIODeviceFakeTests.cpp +++ b/tests/VideoIODeviceFakeTests.cpp @@ -92,10 +92,14 @@ public: return true; } - void AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) override + VideoPlayoutRecoveryDecision AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) override { mLastCompletion = result; mLastReadyQueueDepth = readyQueueDepth; + VideoPlayoutRecoveryDecision decision; + decision.result = result; + decision.readyQueueDepth = readyQueueDepth; + return decision; } unsigned ScheduledFrames() const { return mScheduledFrames; }