#include "VideoBackend.h" #include "DeckLinkSession.h" #include "OpenGLVideoIOBridge.h" #include "HealthTelemetry.h" #include "RenderEngine.h" #include "RuntimeEventDispatcher.h" #include #include #include #include #include #include VideoBackend::VideoBackend(RenderEngine& renderEngine, HealthTelemetry& healthTelemetry, RuntimeEventDispatcher& runtimeEventDispatcher) : mHealthTelemetry(healthTelemetry), mRuntimeEventDispatcher(runtimeEventDispatcher), mPlayoutPolicy(NormalizeVideoPlayoutPolicy(VideoPlayoutPolicy())), mOutputProductionController(mPlayoutPolicy), mReadyOutputQueue(mPlayoutPolicy), mVideoIODevice(std::make_unique()), mBridge(std::make_unique(renderEngine)), mInputCaptureDisabled(IsEnvironmentFlagEnabled("VST_DISABLE_INPUT_CAPTURE")) { } VideoBackend::~VideoBackend() { ReleaseResources(); } void VideoBackend::ReleaseResources() { StopOutputCompletionWorker(); mReadyOutputQueue.Clear(); if (mVideoIODevice) mVideoIODevice->ReleaseResources(); mSystemOutputFramePool.Clear(); if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped)) ApplyLifecycleFailure("Video backend resources released before lifecycle completed."); ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend resources released."); } VideoBackendLifecycleState VideoBackend::LifecycleState() const { return mLifecycle.State(); } bool VideoBackend::DiscoverDevicesAndModes(const VideoFormatSelection& videoModes, std::string& error) { ApplyLifecycleTransition(VideoBackendLifecycleState::Discovering, "Discovering video backend devices and modes."); if (mVideoIODevice->DiscoverDevicesAndModes(videoModes, error)) return ApplyLifecycleTransition(VideoBackendLifecycleState::Discovered, "Video backend devices and modes discovered."); ApplyLifecycleFailure(error.empty() ? "Video backend discovery failed." : error); return false; } bool VideoBackend::SelectPreferredFormats(const VideoFormatSelection& videoModes, bool outputAlphaRequired, std::string& error) { ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Selecting preferred video backend formats."); if (mVideoIODevice->SelectPreferredFormats(videoModes, outputAlphaRequired, error)) return true; ApplyLifecycleFailure(error.empty() ? "Video backend format selection failed." : error); return false; } bool VideoBackend::ConfigureInput(const VideoFormat& inputVideoMode, std::string& error) { if (mLifecycle.State() != VideoBackendLifecycleState::Configuring) ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend input."); if (mInputCaptureDisabled) { MutableState().hasInputSource = false; MutableState().statusMessage = "DeckLink input capture disabled by VST_DISABLE_INPUT_CAPTURE for output timing isolation."; return true; } if (!mVideoIODevice->ConfigureInput( [this](const VideoIOFrame& frame) { HandleInputFrame(frame); }, inputVideoMode, error)) { ApplyLifecycleFailure(error.empty() ? "Video backend input configuration failed." : error); return false; } return true; } bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error) { mPlayoutPolicy = NormalizeVideoPlayoutPolicy(mPlayoutPolicy); mOutputProductionController.Configure(mPlayoutPolicy); mReadyOutputQueue.Configure(mPlayoutPolicy); if (mLifecycle.State() != VideoBackendLifecycleState::Configuring) ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend output."); if (!mVideoIODevice->ConfigureOutput( [this](const VideoIOCompletion& completion) { HandleOutputFrameCompletion(completion); }, outputVideoMode, externalKeyingEnabled, error)) { ApplyLifecycleFailure(error.empty() ? "Video backend output configuration failed." : error); return false; } SystemOutputFramePoolConfig poolConfig; poolConfig.width = mVideoIODevice->OutputFrameWidth(); poolConfig.height = mVideoIODevice->OutputFrameHeight(); poolConfig.pixelFormat = mVideoIODevice->OutputPixelFormat(); poolConfig.rowBytes = mVideoIODevice->OutputFrameRowBytes(); poolConfig.capacity = mPlayoutPolicy.outputFramePoolSize; mSystemOutputFramePool.Configure(poolConfig); RecordSystemMemoryPlayoutStats(); return ApplyLifecycleTransition(VideoBackendLifecycleState::Configured, "Video backend configured."); } bool VideoBackend::Start() { ApplyLifecycleTransition(VideoBackendLifecycleState::Prerolling, "Video backend preroll starting."); if (!mVideoIODevice->PrepareOutputSchedule()) { ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend output schedule preparation failed." : StatusMessage()); return false; } StartOutputCompletionWorker(); StartOutputProducerWorker(); if (!WarmupOutputPreroll()) { StopOutputProducerWorker(); StopOutputCompletionWorker(); ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend preroll warmup failed." : StatusMessage()); return false; } if (!mInputCaptureDisabled && !mVideoIODevice->StartInputStreams()) { StopOutputProducerWorker(); StopOutputCompletionWorker(); ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend input stream start failed." : StatusMessage()); return false; } if (!mVideoIODevice->StartScheduledPlayback()) { StopOutputProducerWorker(); mVideoIODevice->Stop(); StopOutputCompletionWorker(); ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend scheduled playback start failed." : StatusMessage()); return false; } ApplyLifecycleTransition(VideoBackendLifecycleState::Running, "Video backend started."); return true; } bool VideoBackend::Stop() { ApplyLifecycleTransition(VideoBackendLifecycleState::Stopping, "Video backend stopping."); StopOutputProducerWorker(); const bool stopped = mVideoIODevice->Stop(); StopOutputCompletionWorker(); if (stopped) ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend stopped."); else ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend stop failed." : StatusMessage()); return stopped; } const VideoIOState& VideoBackend::State() const { return mVideoIODevice->State(); } VideoIOState& VideoBackend::MutableState() { return mVideoIODevice->MutableState(); } bool VideoBackend::BeginOutputFrame(VideoIOOutputFrame& frame) { return mVideoIODevice->BeginOutputFrame(frame); } void VideoBackend::EndOutputFrame(VideoIOOutputFrame& frame) { mVideoIODevice->EndOutputFrame(frame); } bool VideoBackend::ScheduleOutputFrame(const VideoIOOutputFrame& frame) { return mVideoIODevice->ScheduleOutputFrame(frame); } VideoPlayoutRecoveryDecision VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) { return mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth); } bool VideoBackend::HasInputDevice() const { return mVideoIODevice->HasInputDevice(); } bool VideoBackend::HasInputSource() const { if (mInputCaptureDisabled) return false; return mVideoIODevice->HasInputSource(); } unsigned VideoBackend::InputFrameWidth() const { return mVideoIODevice->InputFrameWidth(); } unsigned VideoBackend::InputFrameHeight() const { return mVideoIODevice->InputFrameHeight(); } unsigned VideoBackend::OutputFrameWidth() const { return mVideoIODevice->OutputFrameWidth(); } unsigned VideoBackend::OutputFrameHeight() const { return mVideoIODevice->OutputFrameHeight(); } unsigned VideoBackend::CaptureTextureWidth() const { return mVideoIODevice->CaptureTextureWidth(); } unsigned VideoBackend::OutputPackTextureWidth() const { return mVideoIODevice->OutputPackTextureWidth(); } VideoIOPixelFormat VideoBackend::InputPixelFormat() const { return mVideoIODevice->InputPixelFormat(); } const std::string& VideoBackend::InputDisplayModeName() const { return mVideoIODevice->InputDisplayModeName(); } const std::string& VideoBackend::OutputModelName() const { return mVideoIODevice->OutputModelName(); } bool VideoBackend::SupportsInternalKeying() const { return mVideoIODevice->SupportsInternalKeying(); } bool VideoBackend::SupportsExternalKeying() const { return mVideoIODevice->SupportsExternalKeying(); } bool VideoBackend::KeyerInterfaceAvailable() const { return mVideoIODevice->KeyerInterfaceAvailable(); } bool VideoBackend::ExternalKeyingActive() const { return mVideoIODevice->ExternalKeyingActive(); } const std::string& VideoBackend::StatusMessage() const { return mVideoIODevice->StatusMessage(); } bool VideoBackend::ShouldPrioritizeOutputOverPreview() const { const RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics(); return metrics.depth < static_cast(mPlayoutPolicy.targetReadyFrames); } void VideoBackend::SetStatusMessage(const std::string& message) { mVideoIODevice->SetStatusMessage(message); } void VideoBackend::PublishStatus(bool externalKeyingConfigured, const std::string& statusMessage) { if (!statusMessage.empty()) SetStatusMessage(statusMessage); mHealthTelemetry.ReportVideoIOStatus( "decklink", OutputModelName(), SupportsInternalKeying(), SupportsExternalKeying(), KeyerInterfaceAvailable(), externalKeyingConfigured, ExternalKeyingActive(), StatusMessage()); PublishBackendStateChanged(VideoBackendLifecycle::StateName(mLifecycle.State()), StatusMessage()); } void VideoBackend::ReportNoInputDeviceSignalStatus() { mHealthTelemetry.ReportSignalStatus( false, InputFrameWidth(), InputFrameHeight(), InputDisplayModeName()); PublishBackendStateChanged("no-input-device", "No input device is available."); } void VideoBackend::HandleInputFrame(const VideoIOFrame& frame) { if (mInputCaptureDisabled) return; const VideoIOState& state = mVideoIODevice->State(); mHealthTelemetry.TryReportSignalStatus(!frame.hasNoInputSource, state.inputFrameSize.width, state.inputFrameSize.height, state.inputDisplayModeName); PublishInputSignalChanged(frame, state); PublishInputFrameArrived(frame); if (mBridge) mBridge->UploadInputFrame(frame, state); } void VideoBackend::HandleOutputFrameCompletion(const VideoIOCompletion& completion) { { std::lock_guard lock(mOutputCompletionMutex); if (!mOutputCompletionWorkerRunning || mOutputCompletionWorkerStopping) return; mPendingOutputCompletions.push_back(completion); } mOutputCompletionCondition.notify_one(); } void VideoBackend::StartOutputCompletionWorker() { { std::lock_guard lock(mOutputCompletionMutex); if (mOutputCompletionWorkerRunning) return; mPendingOutputCompletions.clear(); mReadyOutputQueue.Clear(); mNextReadyOutputFrameIndex = 0; mHasReadyQueueDepthBaseline = false; mMinReadyQueueDepth = 0; mMaxReadyQueueDepth = 0; mReadyQueueZeroDepthCount = 0; mOutputRenderMilliseconds = 0.0; mSmoothedOutputRenderMilliseconds = 0.0; mMaxOutputRenderMilliseconds = 0.0; mOutputFrameAcquireMilliseconds = 0.0; mOutputFrameRenderRequestMilliseconds = 0.0; mOutputFrameEndAccessMilliseconds = 0.0; mLastLateStreak = 0; mLastDropStreak = 0; mOutputCompletionWorkerStopping = false; mOutputCompletionWorkerRunning = true; mOutputCompletionWorker = std::thread(&VideoBackend::OutputCompletionWorkerMain, this); } } void VideoBackend::StopOutputCompletionWorker() { StopOutputProducerWorker(); bool shouldJoin = false; { std::lock_guard lock(mOutputCompletionMutex); if (mOutputCompletionWorkerRunning) mOutputCompletionWorkerStopping = true; shouldJoin = mOutputCompletionWorker.joinable(); } mOutputCompletionCondition.notify_one(); if (shouldJoin) mOutputCompletionWorker.join(); } void VideoBackend::StartOutputProducerWorker() { std::lock_guard lock(mOutputProducerMutex); if (mOutputProducerWorkerRunning) return; const double frameBudgetMilliseconds = State().frameBudgetMilliseconds; const auto frameDuration = frameBudgetMilliseconds > 0.0 ? std::chrono::duration_cast( std::chrono::duration(frameBudgetMilliseconds)) : std::chrono::milliseconds(16); mRenderCadenceController.Configure(frameDuration, std::chrono::steady_clock::now()); mLastOutputProductionCompletion = VideoIOCompletion(); mLastOutputProductionTime = std::chrono::steady_clock::time_point(); mOutputProducerWorkerStopping = false; mOutputProducerWorkerRunning = true; mOutputProducerWorker = std::thread(&VideoBackend::OutputProducerWorkerMain, this); mOutputProducerCondition.notify_one(); } void VideoBackend::StopOutputProducerWorker() { bool shouldJoin = false; { std::lock_guard lock(mOutputProducerMutex); if (mOutputProducerWorkerRunning) mOutputProducerWorkerStopping = true; shouldJoin = mOutputProducerWorker.joinable(); } mOutputProducerCondition.notify_one(); if (shouldJoin) mOutputProducerWorker.join(); } void VideoBackend::NotifyOutputProducer() { mOutputProducerCondition.notify_one(); } bool VideoBackend::WarmupOutputPreroll() { const VideoPlayoutPolicy policy = NormalizeVideoPlayoutPolicy(mPlayoutPolicy); const std::size_t targetPrerollFrames = static_cast(policy.targetPrerollFrames); if (targetPrerollFrames == 0) return true; const double frameBudgetMilliseconds = State().frameBudgetMilliseconds > 0.0 ? State().frameBudgetMilliseconds : 16.0; const auto estimatedCadenceTime = std::chrono::duration_cast( std::chrono::duration(frameBudgetMilliseconds * static_cast(targetPrerollFrames + 2))); const auto timeout = (std::max)(std::chrono::milliseconds(1000), estimatedCadenceTime + std::chrono::milliseconds(500)); const auto deadline = std::chrono::steady_clock::now() + timeout; while (std::chrono::steady_clock::now() < deadline) { ScheduleReadyOutputFramesToTarget(); const SystemOutputFramePoolMetrics metrics = mSystemOutputFramePool.GetMetrics(); RecordSystemMemoryPlayoutStats(); if (metrics.scheduledCount >= targetPrerollFrames) return true; NotifyOutputProducer(); const auto waitDuration = (std::min)(OutputProducerWakeInterval(), std::chrono::milliseconds(5)); std::unique_lock lock(mOutputProducerMutex); mOutputProducerCondition.wait_for(lock, waitDuration); if (mOutputProducerWorkerStopping) return false; } SetStatusMessage("Timed out warming up DeckLink preroll from rendered system-memory frames."); return false; } void VideoBackend::OutputCompletionWorkerMain() { for (;;) { VideoIOCompletion completion; { std::unique_lock lock(mOutputCompletionMutex); mOutputCompletionCondition.wait(lock, [this]() { return mOutputCompletionWorkerStopping || !mPendingOutputCompletions.empty(); }); if (mPendingOutputCompletions.empty()) { if (mOutputCompletionWorkerStopping) { mOutputCompletionWorkerRunning = false; return; } continue; } completion = mPendingOutputCompletions.front(); mPendingOutputCompletions.pop_front(); } ProcessOutputFrameCompletion(completion); } } void VideoBackend::OutputProducerWorkerMain() { for (;;) { { std::lock_guard lock(mOutputProducerMutex); if (mOutputProducerWorkerStopping) { mOutputProducerWorkerRunning = false; return; } } ScheduleReadyOutputFramesToTarget(); const RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics(); RecordReadyQueueDepthSample(metrics); const auto now = std::chrono::steady_clock::now(); RenderCadenceDecision cadenceDecision = mRenderCadenceController.Tick(now); if (cadenceDecision.action == RenderCadenceAction::Wait) { const auto waitDuration = (std::min)( std::chrono::duration_cast(cadenceDecision.waitDuration), OutputProducerWakeInterval()); std::unique_lock lock(mOutputProducerMutex); mOutputProducerCondition.wait_for(lock, waitDuration); if (mOutputProducerWorkerStopping) { mOutputProducerWorkerRunning = false; return; } continue; } VideoIOCompletion completion; { std::lock_guard lock(mOutputProducerMutex); if (mOutputProducerWorkerStopping) continue; completion = mLastOutputProductionCompletion; } const std::size_t producedFrames = ProduceReadyOutputFrames(completion, 1); if (producedFrames > 0) { mLastOutputProductionTime = std::chrono::steady_clock::now(); ScheduleReadyOutputFramesToTarget(); continue; } { std::unique_lock lock(mOutputProducerMutex); mOutputProducerCondition.wait_for(lock, OutputProducerWakeInterval()); if (mOutputProducerWorkerStopping) { mOutputProducerWorkerRunning = false; return; } } } } std::chrono::milliseconds VideoBackend::OutputProducerWakeInterval() const { const double frameBudgetMilliseconds = State().frameBudgetMilliseconds; if (frameBudgetMilliseconds <= 0.0) return std::chrono::milliseconds(8); const int intervalMilliseconds = (std::max)(1, static_cast(std::floor(frameBudgetMilliseconds * 0.75))); return std::chrono::milliseconds(intervalMilliseconds); } void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion) { if (completion.outputFrameBuffer != nullptr) mSystemOutputFramePool.ReleaseSlotByBuffer(completion.outputFrameBuffer); RecordFramePacing(completion.result); PublishOutputFrameCompleted(completion); const RenderOutputQueueMetrics initialQueueMetrics = mReadyOutputQueue.GetMetrics(); RecordReadyQueueDepthSample(initialQueueMetrics); const VideoPlayoutRecoveryDecision recoveryDecision = AccountForCompletionResult(completion.result, initialQueueMetrics.depth); { std::lock_guard lock(mOutputMetricsMutex); mLastLateStreak = recoveryDecision.lateStreak; mLastDropStreak = recoveryDecision.dropStreak; } { std::lock_guard lock(mOutputProducerMutex); mLastOutputProductionCompletion = completion; } NotifyOutputProducer(); RecordBackendPlayoutHealth(completion.result, recoveryDecision); RecordSystemMemoryPlayoutStats(); } std::size_t VideoBackend::ScheduleReadyOutputFramesToTarget() { const std::size_t targetScheduledFrames = static_cast(mPlayoutPolicy.targetPrerollFrames); std::size_t scheduledFrames = 0; for (;;) { const SystemOutputFramePoolMetrics poolMetrics = mSystemOutputFramePool.GetMetrics(); if (poolMetrics.scheduledCount >= targetScheduledFrames) break; if (!ScheduleReadyOutputFrame()) break; ++scheduledFrames; } return scheduledFrames; } void VideoBackend::RecordBackendPlayoutHealth(VideoIOCompletionResult result, const VideoPlayoutRecoveryDecision& recoveryDecision) { const RenderOutputQueueMetrics queueMetrics = mReadyOutputQueue.GetMetrics(); std::size_t minReadyQueueDepth = 0; std::size_t maxReadyQueueDepth = 0; uint64_t readyQueueZeroDepthCount = 0; double outputRenderMilliseconds = 0.0; double smoothedOutputRenderMilliseconds = 0.0; double maxOutputRenderMilliseconds = 0.0; double outputFrameAcquireMilliseconds = 0.0; double outputFrameRenderRequestMilliseconds = 0.0; double outputFrameEndAccessMilliseconds = 0.0; { std::lock_guard lock(mOutputMetricsMutex); minReadyQueueDepth = mMinReadyQueueDepth; maxReadyQueueDepth = mMaxReadyQueueDepth; readyQueueZeroDepthCount = mReadyQueueZeroDepthCount; outputRenderMilliseconds = mOutputRenderMilliseconds; smoothedOutputRenderMilliseconds = mSmoothedOutputRenderMilliseconds; maxOutputRenderMilliseconds = mMaxOutputRenderMilliseconds; outputFrameAcquireMilliseconds = mOutputFrameAcquireMilliseconds; outputFrameRenderRequestMilliseconds = mOutputFrameRenderRequestMilliseconds; outputFrameEndAccessMilliseconds = mOutputFrameEndAccessMilliseconds; } mHealthTelemetry.TryRecordBackendPlayoutHealth( VideoBackendLifecycle::StateName(mLifecycle.State()), CompletionResultName(result), queueMetrics.depth, queueMetrics.capacity, queueMetrics.pushedCount, minReadyQueueDepth, maxReadyQueueDepth, readyQueueZeroDepthCount, queueMetrics.poppedCount, queueMetrics.droppedCount, queueMetrics.underrunCount, outputRenderMilliseconds, smoothedOutputRenderMilliseconds, maxOutputRenderMilliseconds, outputFrameAcquireMilliseconds, outputFrameRenderRequestMilliseconds, outputFrameEndAccessMilliseconds, recoveryDecision.completedFrameIndex, recoveryDecision.scheduledFrameIndex, recoveryDecision.scheduledLeadFrames, recoveryDecision.measuredLagFrames, recoveryDecision.catchUpFrames, recoveryDecision.lateStreak, recoveryDecision.dropStreak, mLateFrameCount, mDroppedFrameCount, mFlushedFrameCount, mLifecycle.State() == VideoBackendLifecycleState::Degraded, StatusMessage()); } std::size_t VideoBackend::ProduceReadyOutputFrames(const VideoIOCompletion& completion, std::size_t maxFrames) { if (maxFrames == 0) return 0; std::lock_guard productionLock(mOutputProductionMutex); RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics(); std::size_t producedFrames = 0; while (producedFrames < maxFrames) { if (!RenderReadyOutputFrame(mVideoIODevice->State(), completion)) break; ++producedFrames; metrics = mReadyOutputQueue.GetMetrics(); RecordReadyQueueDepthSample(metrics); } return producedFrames; } OutputProductionPressure VideoBackend::BuildOutputProductionPressure(const RenderOutputQueueMetrics& metrics) const { OutputProductionPressure pressure; pressure.readyQueueDepth = metrics.depth; pressure.readyQueueCapacity = metrics.capacity; pressure.readyQueueUnderrunCount = metrics.underrunCount; { std::lock_guard lock(mOutputMetricsMutex); pressure.lateStreak = mLastLateStreak; pressure.dropStreak = mLastDropStreak; } return pressure; } bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion) { const auto renderStart = std::chrono::steady_clock::now(); OutputFrameSlot outputSlot; VideoIOOutputFrame outputFrame; const auto acquireStart = std::chrono::steady_clock::now(); if (!mSystemOutputFramePool.AcquireFreeSlot(outputSlot)) { if (!mReadyOutputQueue.DropOldestFrame() || !mSystemOutputFramePool.AcquireFreeSlot(outputSlot)) return false; } outputFrame = outputSlot.frame; const auto acquireEnd = std::chrono::steady_clock::now(); bool rendered = true; const auto renderRequestStart = std::chrono::steady_clock::now(); if (mBridge) rendered = mBridge->RenderScheduledFrame(state, completion, outputFrame); const auto renderRequestEnd = std::chrono::steady_clock::now(); const auto endAccessStart = std::chrono::steady_clock::now(); const bool publishedReady = mSystemOutputFramePool.PublishReadySlot(outputSlot); const auto endAccessEnd = std::chrono::steady_clock::now(); const double acquireMilliseconds = std::chrono::duration_cast>(acquireEnd - acquireStart).count(); const double renderRequestMilliseconds = std::chrono::duration_cast>(renderRequestEnd - renderRequestStart).count(); const double endAccessMilliseconds = std::chrono::duration_cast>(endAccessEnd - endAccessStart).count(); if (!rendered) { mSystemOutputFramePool.ReleaseSlot(outputSlot); ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame."); const double renderMilliseconds = std::chrono::duration_cast>( std::chrono::steady_clock::now() - renderStart).count(); RecordOutputRenderDuration(renderMilliseconds, acquireMilliseconds, renderRequestMilliseconds, endAccessMilliseconds); return false; } if (!publishedReady) { mSystemOutputFramePool.ReleaseSlot(outputSlot); return false; } const double renderMilliseconds = std::chrono::duration_cast>( std::chrono::steady_clock::now() - renderStart).count(); RecordOutputRenderDuration(renderMilliseconds, acquireMilliseconds, renderRequestMilliseconds, endAccessMilliseconds); RenderOutputFrame readyFrame; readyFrame.frame = outputFrame; readyFrame.frameIndex = ++mNextReadyOutputFrameIndex; readyFrame.releaseFrame = [this](VideoIOOutputFrame& frame) { mSystemOutputFramePool.ReleaseSlotByBuffer(frame.bytes); }; const bool pushed = mReadyOutputQueue.Push(readyFrame); if (!pushed) mSystemOutputFramePool.ReleaseSlot(outputSlot); RecordSystemMemoryPlayoutStats(); return pushed; } bool VideoBackend::ScheduleReadyOutputFrame() { std::lock_guard schedulingLock(mOutputSchedulingMutex); RenderOutputFrame readyFrame; if (!mReadyOutputQueue.TryPop(readyFrame)) return false; RecordReadyQueueDepthSample(mReadyOutputQueue.GetMetrics()); if (!mSystemOutputFramePool.MarkScheduledByBuffer(readyFrame.frame.bytes)) { if (readyFrame.releaseFrame) readyFrame.releaseFrame(readyFrame.frame); return false; } if (!ScheduleOutputFrame(readyFrame.frame)) { RecordDeckLinkBufferTelemetry(); mSystemOutputFramePool.ReleaseSlotByBuffer(readyFrame.frame.bytes); return false; } RecordDeckLinkBufferTelemetry(); PublishOutputFrameScheduled(readyFrame.frame); RecordSystemMemoryPlayoutStats(); return true; } bool VideoBackend::ScheduleBlackUnderrunFrame() { VideoIOOutputFrame outputFrame; if (!BeginOutputFrame(outputFrame)) { ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: no output frame was available for fallback scheduling."); return false; } if (outputFrame.bytes != nullptr && outputFrame.rowBytes > 0 && outputFrame.height > 0) std::memset(outputFrame.bytes, 0, static_cast(outputFrame.rowBytes) * outputFrame.height); EndOutputFrame(outputFrame); if (!ScheduleOutputFrame(outputFrame)) { RecordDeckLinkBufferTelemetry(); ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: black fallback frame scheduling failed."); return false; } RecordDeckLinkBufferTelemetry(); ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: scheduled black fallback frame."); PublishOutputFrameScheduled(outputFrame); return true; } void VideoBackend::RecordFramePacing(VideoIOCompletionResult completionResult) { const auto now = std::chrono::steady_clock::now(); if (mLastPlayoutCompletionTime != std::chrono::steady_clock::time_point()) { mCompletionIntervalMilliseconds = std::chrono::duration_cast>(now - mLastPlayoutCompletionTime).count(); if (mSmoothedCompletionIntervalMilliseconds <= 0.0) mSmoothedCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds; else mSmoothedCompletionIntervalMilliseconds = mSmoothedCompletionIntervalMilliseconds * 0.9 + mCompletionIntervalMilliseconds * 0.1; if (mCompletionIntervalMilliseconds > mMaxCompletionIntervalMilliseconds) mMaxCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds; } mLastPlayoutCompletionTime = now; if (completionResult == VideoIOCompletionResult::DisplayedLate) ++mLateFrameCount; else if (completionResult == VideoIOCompletionResult::Dropped) ++mDroppedFrameCount; else if (completionResult == VideoIOCompletionResult::Flushed) ++mFlushedFrameCount; mHealthTelemetry.TryRecordFramePacingStats( mCompletionIntervalMilliseconds, mSmoothedCompletionIntervalMilliseconds, mMaxCompletionIntervalMilliseconds, mLateFrameCount, mDroppedFrameCount, mFlushedFrameCount); PublishTimingSample("VideoBackend", "completionInterval", mCompletionIntervalMilliseconds, "ms"); PublishTimingSample("VideoBackend", "smoothedCompletionInterval", mSmoothedCompletionIntervalMilliseconds, "ms"); } void VideoBackend::RecordReadyQueueDepthSample(const RenderOutputQueueMetrics& metrics) { std::lock_guard lock(mOutputMetricsMutex); if (!mHasReadyQueueDepthBaseline) { mHasReadyQueueDepthBaseline = true; mMinReadyQueueDepth = metrics.depth; mMaxReadyQueueDepth = metrics.depth; } else { mMinReadyQueueDepth = (std::min)(mMinReadyQueueDepth, metrics.depth); mMaxReadyQueueDepth = (std::max)(mMaxReadyQueueDepth, metrics.depth); } if (metrics.depth == 0) ++mReadyQueueZeroDepthCount; } void VideoBackend::RecordDeckLinkBufferTelemetry() { if (!mVideoIODevice) return; const VideoIOState& state = mVideoIODevice->State(); mHealthTelemetry.TryRecordDeckLinkBufferTelemetry( state.actualDeckLinkBufferedFramesAvailable, state.actualDeckLinkBufferedFrames, static_cast(mPlayoutPolicy.targetPrerollFrames), state.deckLinkScheduleCallMilliseconds, state.deckLinkScheduleFailureCount); } void VideoBackend::RecordSystemMemoryPlayoutStats() { const SystemOutputFramePoolMetrics poolMetrics = mSystemOutputFramePool.GetMetrics(); const RenderOutputQueueMetrics queueMetrics = mReadyOutputQueue.GetMetrics(); RecordDeckLinkBufferTelemetry(); mHealthTelemetry.TryRecordSystemMemoryPlayoutStats( poolMetrics.freeCount, poolMetrics.readyCount, poolMetrics.scheduledCount, poolMetrics.readyUnderrunCount, 0, queueMetrics.droppedCount, 0.0, 0.0); } void VideoBackend::RecordOutputRenderDuration(double renderMilliseconds, double acquireMilliseconds, double renderRequestMilliseconds, double endAccessMilliseconds) { std::lock_guard lock(mOutputMetricsMutex); mOutputRenderMilliseconds = (std::max)(renderMilliseconds, 0.0); if (mSmoothedOutputRenderMilliseconds <= 0.0) mSmoothedOutputRenderMilliseconds = mOutputRenderMilliseconds; else mSmoothedOutputRenderMilliseconds = mSmoothedOutputRenderMilliseconds * 0.9 + mOutputRenderMilliseconds * 0.1; mMaxOutputRenderMilliseconds = (std::max)(mMaxOutputRenderMilliseconds, mOutputRenderMilliseconds); mOutputFrameAcquireMilliseconds = (std::max)(acquireMilliseconds, 0.0); mOutputFrameRenderRequestMilliseconds = (std::max)(renderRequestMilliseconds, 0.0); mOutputFrameEndAccessMilliseconds = (std::max)(endAccessMilliseconds, 0.0); PublishTimingSample("VideoBackend", "outputRender", mOutputRenderMilliseconds, "ms"); PublishTimingSample("VideoBackend", "smoothedOutputRender", mSmoothedOutputRenderMilliseconds, "ms"); } bool VideoBackend::ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message) { const VideoBackendLifecycleTransition transition = mLifecycle.TransitionTo(state, message); if (!transition.accepted) { PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), transition.errorMessage); return false; } PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), message); return true; } bool VideoBackend::ApplyLifecycleFailure(const std::string& message) { const VideoBackendLifecycleTransition transition = mLifecycle.Fail(message); if (!transition.accepted) { PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), transition.errorMessage); return false; } PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), message); return true; } void VideoBackend::PublishBackendStateChanged(const std::string& state, const std::string& message) { try { BackendStateChangedEvent event; event.backendName = "decklink"; event.state = state; event.message = message; if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("BackendStateChanged event publish failed.\n"); } catch (...) { OutputDebugStringA("BackendStateChanged event publish threw.\n"); } } void VideoBackend::PublishInputSignalChanged(const VideoIOFrame& frame, const VideoIOState& state) { const bool hasSignal = !frame.hasNoInputSource; const unsigned width = state.inputFrameSize.width; const unsigned height = state.inputFrameSize.height; if (mHasLastInputSignal && mLastInputSignal == hasSignal && mLastInputSignalWidth == width && mLastInputSignalHeight == height && mLastInputSignalModeName == state.inputDisplayModeName) { return; } mHasLastInputSignal = true; mLastInputSignal = hasSignal; mLastInputSignalWidth = width; mLastInputSignalHeight = height; mLastInputSignalModeName = state.inputDisplayModeName; try { InputSignalChangedEvent event; event.hasSignal = hasSignal; event.width = width; event.height = height; event.modeName = state.inputDisplayModeName; if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("InputSignalChanged event publish failed.\n"); } catch (...) { OutputDebugStringA("InputSignalChanged event publish threw.\n"); } } void VideoBackend::PublishInputFrameArrived(const VideoIOFrame& frame) { try { InputFrameArrivedEvent event; event.frameIndex = ++mInputFrameIndex; event.width = frame.width; event.height = frame.height; event.rowBytes = frame.rowBytes; event.pixelFormat = PixelFormatName(frame.pixelFormat); event.hasNoInputSource = frame.hasNoInputSource; if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("InputFrameArrived event publish failed.\n"); } catch (...) { OutputDebugStringA("InputFrameArrived event publish threw.\n"); } } void VideoBackend::PublishOutputFrameScheduled(const VideoIOOutputFrame& frame) { try { OutputFrameScheduledEvent event; event.frameIndex = ++mOutputFrameScheduleIndex; (void)frame; if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("OutputFrameScheduled event publish failed.\n"); } catch (...) { OutputDebugStringA("OutputFrameScheduled event publish threw.\n"); } } void VideoBackend::PublishOutputFrameCompleted(const VideoIOCompletion& completion) { try { OutputFrameCompletedEvent event; event.frameIndex = ++mOutputFrameCompletionIndex; event.result = CompletionResultName(completion.result); if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("OutputFrameCompleted event publish failed.\n"); } catch (...) { OutputDebugStringA("OutputFrameCompleted event publish threw.\n"); } } void VideoBackend::PublishTimingSample(const std::string& subsystem, const std::string& metric, double value, const std::string& unit) { try { TimingSampleRecordedEvent event; event.subsystem = subsystem; event.metric = metric; event.value = value; event.unit = unit; if (!mRuntimeEventDispatcher.PublishPayload(event, "HealthTelemetry")) OutputDebugStringA("TimingSampleRecorded event publish failed.\n"); } catch (...) { OutputDebugStringA("TimingSampleRecorded event publish threw.\n"); } } std::string VideoBackend::CompletionResultName(VideoIOCompletionResult result) { switch (result) { case VideoIOCompletionResult::Completed: return "Completed"; case VideoIOCompletionResult::DisplayedLate: return "DisplayedLate"; case VideoIOCompletionResult::Dropped: return "Dropped"; case VideoIOCompletionResult::Flushed: return "Flushed"; case VideoIOCompletionResult::Unknown: default: return "Unknown"; } } std::string VideoBackend::PixelFormatName(VideoIOPixelFormat pixelFormat) { return std::string(VideoIOPixelFormatName(pixelFormat)); } bool VideoBackend::IsEnvironmentFlagEnabled(const char* name) { if (name == nullptr || name[0] == '\0') return false; char* value = nullptr; std::size_t valueSize = 0; if (_dupenv_s(&value, &valueSize, name) != 0 || value == nullptr) return false; const std::string flag(value); std::free(value); return flag == "1" || flag == "true" || flag == "TRUE" || flag == "yes" || flag == "on"; }