#include "VideoBackend.h" #include "DeckLinkSession.h" #include "OpenGLVideoIOBridge.h" #include "HealthTelemetry.h" #include "RenderEngine.h" #include "RuntimeEventDispatcher.h" #include #include #include VideoBackend::VideoBackend(RenderEngine& renderEngine, HealthTelemetry& healthTelemetry, RuntimeEventDispatcher& runtimeEventDispatcher) : mHealthTelemetry(healthTelemetry), mRuntimeEventDispatcher(runtimeEventDispatcher), mPlayoutPolicy(NormalizeVideoPlayoutPolicy(VideoPlayoutPolicy())), mReadyOutputQueue(mPlayoutPolicy), mVideoIODevice(std::make_unique()), mBridge(std::make_unique(renderEngine)) { } VideoBackend::~VideoBackend() { ReleaseResources(); } void VideoBackend::ReleaseResources() { StopOutputCompletionWorker(); mReadyOutputQueue.Clear(); if (mVideoIODevice) mVideoIODevice->ReleaseResources(); if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped)) ApplyLifecycleFailure("Video backend resources released before lifecycle completed."); ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend resources released."); } VideoBackendLifecycleState VideoBackend::LifecycleState() const { return mLifecycle.State(); } bool VideoBackend::DiscoverDevicesAndModes(const VideoFormatSelection& videoModes, std::string& error) { ApplyLifecycleTransition(VideoBackendLifecycleState::Discovering, "Discovering video backend devices and modes."); if (mVideoIODevice->DiscoverDevicesAndModes(videoModes, error)) return ApplyLifecycleTransition(VideoBackendLifecycleState::Discovered, "Video backend devices and modes discovered."); ApplyLifecycleFailure(error.empty() ? "Video backend discovery failed." : error); return false; } bool VideoBackend::SelectPreferredFormats(const VideoFormatSelection& videoModes, bool outputAlphaRequired, std::string& error) { ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Selecting preferred video backend formats."); if (mVideoIODevice->SelectPreferredFormats(videoModes, outputAlphaRequired, error)) return true; ApplyLifecycleFailure(error.empty() ? "Video backend format selection failed." : error); return false; } bool VideoBackend::ConfigureInput(const VideoFormat& inputVideoMode, std::string& error) { if (mLifecycle.State() != VideoBackendLifecycleState::Configuring) ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend input."); if (!mVideoIODevice->ConfigureInput( [this](const VideoIOFrame& frame) { HandleInputFrame(frame); }, inputVideoMode, error)) { ApplyLifecycleFailure(error.empty() ? "Video backend input configuration failed." : error); return false; } return true; } bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error) { mPlayoutPolicy = NormalizeVideoPlayoutPolicy(mPlayoutPolicy); mReadyOutputQueue.Configure(mPlayoutPolicy); if (mLifecycle.State() != VideoBackendLifecycleState::Configuring) ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend output."); if (!mVideoIODevice->ConfigureOutput( [this](const VideoIOCompletion& completion) { HandleOutputFrameCompletion(completion); }, outputVideoMode, externalKeyingEnabled, error)) { ApplyLifecycleFailure(error.empty() ? "Video backend output configuration failed." : error); return false; } return ApplyLifecycleTransition(VideoBackendLifecycleState::Configured, "Video backend configured."); } bool VideoBackend::Start() { ApplyLifecycleTransition(VideoBackendLifecycleState::Prerolling, "Video backend preroll starting."); StartOutputCompletionWorker(); const bool started = mVideoIODevice->Start(); if (started) ApplyLifecycleTransition(VideoBackendLifecycleState::Running, "Video backend started."); else { StopOutputCompletionWorker(); ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend start failed." : StatusMessage()); } return started; } bool VideoBackend::Stop() { ApplyLifecycleTransition(VideoBackendLifecycleState::Stopping, "Video backend stopping."); const bool stopped = mVideoIODevice->Stop(); StopOutputCompletionWorker(); if (stopped) ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend stopped."); else ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend stop failed." : StatusMessage()); return stopped; } const VideoIOState& VideoBackend::State() const { return mVideoIODevice->State(); } VideoIOState& VideoBackend::MutableState() { return mVideoIODevice->MutableState(); } bool VideoBackend::BeginOutputFrame(VideoIOOutputFrame& frame) { return mVideoIODevice->BeginOutputFrame(frame); } void VideoBackend::EndOutputFrame(VideoIOOutputFrame& frame) { mVideoIODevice->EndOutputFrame(frame); } bool VideoBackend::ScheduleOutputFrame(const VideoIOOutputFrame& frame) { return mVideoIODevice->ScheduleOutputFrame(frame); } VideoPlayoutRecoveryDecision VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth) { return mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth); } bool VideoBackend::HasInputDevice() const { return mVideoIODevice->HasInputDevice(); } bool VideoBackend::HasInputSource() const { return mVideoIODevice->HasInputSource(); } unsigned VideoBackend::InputFrameWidth() const { return mVideoIODevice->InputFrameWidth(); } unsigned VideoBackend::InputFrameHeight() const { return mVideoIODevice->InputFrameHeight(); } unsigned VideoBackend::OutputFrameWidth() const { return mVideoIODevice->OutputFrameWidth(); } unsigned VideoBackend::OutputFrameHeight() const { return mVideoIODevice->OutputFrameHeight(); } unsigned VideoBackend::CaptureTextureWidth() const { return mVideoIODevice->CaptureTextureWidth(); } unsigned VideoBackend::OutputPackTextureWidth() const { return mVideoIODevice->OutputPackTextureWidth(); } VideoIOPixelFormat VideoBackend::InputPixelFormat() const { return mVideoIODevice->InputPixelFormat(); } const std::string& VideoBackend::InputDisplayModeName() const { return mVideoIODevice->InputDisplayModeName(); } const std::string& VideoBackend::OutputModelName() const { return mVideoIODevice->OutputModelName(); } bool VideoBackend::SupportsInternalKeying() const { return mVideoIODevice->SupportsInternalKeying(); } bool VideoBackend::SupportsExternalKeying() const { return mVideoIODevice->SupportsExternalKeying(); } bool VideoBackend::KeyerInterfaceAvailable() const { return mVideoIODevice->KeyerInterfaceAvailable(); } bool VideoBackend::ExternalKeyingActive() const { return mVideoIODevice->ExternalKeyingActive(); } const std::string& VideoBackend::StatusMessage() const { return mVideoIODevice->StatusMessage(); } void VideoBackend::SetStatusMessage(const std::string& message) { mVideoIODevice->SetStatusMessage(message); } void VideoBackend::PublishStatus(bool externalKeyingConfigured, const std::string& statusMessage) { if (!statusMessage.empty()) SetStatusMessage(statusMessage); mHealthTelemetry.ReportVideoIOStatus( "decklink", OutputModelName(), SupportsInternalKeying(), SupportsExternalKeying(), KeyerInterfaceAvailable(), externalKeyingConfigured, ExternalKeyingActive(), StatusMessage()); PublishBackendStateChanged(VideoBackendLifecycle::StateName(mLifecycle.State()), StatusMessage()); } void VideoBackend::ReportNoInputDeviceSignalStatus() { mHealthTelemetry.ReportSignalStatus( false, InputFrameWidth(), InputFrameHeight(), InputDisplayModeName()); PublishBackendStateChanged("no-input-device", "No input device is available."); } void VideoBackend::HandleInputFrame(const VideoIOFrame& frame) { const VideoIOState& state = mVideoIODevice->State(); mHealthTelemetry.TryReportSignalStatus(!frame.hasNoInputSource, state.inputFrameSize.width, state.inputFrameSize.height, state.inputDisplayModeName); PublishInputSignalChanged(frame, state); PublishInputFrameArrived(frame); if (mBridge) mBridge->UploadInputFrame(frame, state); } void VideoBackend::HandleOutputFrameCompletion(const VideoIOCompletion& completion) { { std::lock_guard lock(mOutputCompletionMutex); if (!mOutputCompletionWorkerRunning || mOutputCompletionWorkerStopping) return; mPendingOutputCompletions.push_back(completion); } mOutputCompletionCondition.notify_one(); } void VideoBackend::StartOutputCompletionWorker() { std::lock_guard lock(mOutputCompletionMutex); if (mOutputCompletionWorkerRunning) return; mPendingOutputCompletions.clear(); mReadyOutputQueue.Clear(); mNextReadyOutputFrameIndex = 0; mOutputCompletionWorkerStopping = false; mOutputCompletionWorkerRunning = true; mOutputCompletionWorker = std::thread(&VideoBackend::OutputCompletionWorkerMain, this); } void VideoBackend::StopOutputCompletionWorker() { bool shouldJoin = false; { std::lock_guard lock(mOutputCompletionMutex); if (mOutputCompletionWorkerRunning) mOutputCompletionWorkerStopping = true; shouldJoin = mOutputCompletionWorker.joinable(); } mOutputCompletionCondition.notify_one(); if (shouldJoin) mOutputCompletionWorker.join(); } void VideoBackend::OutputCompletionWorkerMain() { for (;;) { VideoIOCompletion completion; { std::unique_lock lock(mOutputCompletionMutex); mOutputCompletionCondition.wait(lock, [this]() { return mOutputCompletionWorkerStopping || !mPendingOutputCompletions.empty(); }); if (mPendingOutputCompletions.empty()) { if (mOutputCompletionWorkerStopping) { mOutputCompletionWorkerRunning = false; return; } continue; } completion = mPendingOutputCompletions.front(); mPendingOutputCompletions.pop_front(); } ProcessOutputFrameCompletion(completion); } } void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion) { RecordFramePacing(completion.result); PublishOutputFrameCompleted(completion); const VideoPlayoutRecoveryDecision recoveryDecision = AccountForCompletionResult(completion.result, mReadyOutputQueue.GetMetrics().depth); FillReadyOutputQueue(completion); if (!ScheduleReadyOutputFrame()) ScheduleBlackUnderrunFrame(); RecordBackendPlayoutHealth(completion.result, recoveryDecision); } void VideoBackend::RecordBackendPlayoutHealth(VideoIOCompletionResult result, const VideoPlayoutRecoveryDecision& recoveryDecision) { const RenderOutputQueueMetrics queueMetrics = mReadyOutputQueue.GetMetrics(); mHealthTelemetry.TryRecordBackendPlayoutHealth( VideoBackendLifecycle::StateName(mLifecycle.State()), CompletionResultName(result), queueMetrics.depth, queueMetrics.capacity, queueMetrics.pushedCount, queueMetrics.poppedCount, queueMetrics.droppedCount, queueMetrics.underrunCount, recoveryDecision.completedFrameIndex, recoveryDecision.scheduledFrameIndex, recoveryDecision.scheduledLeadFrames, recoveryDecision.measuredLagFrames, recoveryDecision.catchUpFrames, recoveryDecision.lateStreak, recoveryDecision.dropStreak, mLateFrameCount, mDroppedFrameCount, mFlushedFrameCount, mLifecycle.State() == VideoBackendLifecycleState::Degraded, StatusMessage()); } bool VideoBackend::FillReadyOutputQueue(const VideoIOCompletion& completion) { RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics(); bool filledAny = false; while (metrics.depth < mPlayoutPolicy.targetReadyFrames) { if (!RenderReadyOutputFrame(mVideoIODevice->State(), completion)) return filledAny; filledAny = true; metrics = mReadyOutputQueue.GetMetrics(); } return true; } bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion) { VideoIOOutputFrame outputFrame; if (!BeginOutputFrame(outputFrame)) return false; bool rendered = true; if (mBridge) rendered = mBridge->RenderScheduledFrame(state, completion, outputFrame); EndOutputFrame(outputFrame); if (!rendered) { ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame."); return false; } RenderOutputFrame readyFrame; readyFrame.frame = outputFrame; readyFrame.frameIndex = ++mNextReadyOutputFrameIndex; return mReadyOutputQueue.Push(readyFrame); } bool VideoBackend::ScheduleReadyOutputFrame() { RenderOutputFrame readyFrame; if (!mReadyOutputQueue.TryPop(readyFrame)) return false; if (!ScheduleOutputFrame(readyFrame.frame)) return false; PublishOutputFrameScheduled(readyFrame.frame); return true; } bool VideoBackend::ScheduleBlackUnderrunFrame() { VideoIOOutputFrame outputFrame; if (!BeginOutputFrame(outputFrame)) { ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: no output frame was available for fallback scheduling."); return false; } if (outputFrame.bytes != nullptr && outputFrame.rowBytes > 0 && outputFrame.height > 0) std::memset(outputFrame.bytes, 0, static_cast(outputFrame.rowBytes) * outputFrame.height); EndOutputFrame(outputFrame); if (!ScheduleOutputFrame(outputFrame)) { ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: black fallback frame scheduling failed."); return false; } ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: scheduled black fallback frame."); PublishOutputFrameScheduled(outputFrame); return true; } void VideoBackend::RecordFramePacing(VideoIOCompletionResult completionResult) { const auto now = std::chrono::steady_clock::now(); if (mLastPlayoutCompletionTime != std::chrono::steady_clock::time_point()) { mCompletionIntervalMilliseconds = std::chrono::duration_cast>(now - mLastPlayoutCompletionTime).count(); if (mSmoothedCompletionIntervalMilliseconds <= 0.0) mSmoothedCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds; else mSmoothedCompletionIntervalMilliseconds = mSmoothedCompletionIntervalMilliseconds * 0.9 + mCompletionIntervalMilliseconds * 0.1; if (mCompletionIntervalMilliseconds > mMaxCompletionIntervalMilliseconds) mMaxCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds; } mLastPlayoutCompletionTime = now; if (completionResult == VideoIOCompletionResult::DisplayedLate) ++mLateFrameCount; else if (completionResult == VideoIOCompletionResult::Dropped) ++mDroppedFrameCount; else if (completionResult == VideoIOCompletionResult::Flushed) ++mFlushedFrameCount; mHealthTelemetry.TryRecordFramePacingStats( mCompletionIntervalMilliseconds, mSmoothedCompletionIntervalMilliseconds, mMaxCompletionIntervalMilliseconds, mLateFrameCount, mDroppedFrameCount, mFlushedFrameCount); PublishTimingSample("VideoBackend", "completionInterval", mCompletionIntervalMilliseconds, "ms"); PublishTimingSample("VideoBackend", "smoothedCompletionInterval", mSmoothedCompletionIntervalMilliseconds, "ms"); } bool VideoBackend::ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message) { const VideoBackendLifecycleTransition transition = mLifecycle.TransitionTo(state, message); if (!transition.accepted) { PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), transition.errorMessage); return false; } PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), message); return true; } bool VideoBackend::ApplyLifecycleFailure(const std::string& message) { const VideoBackendLifecycleTransition transition = mLifecycle.Fail(message); if (!transition.accepted) { PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), transition.errorMessage); return false; } PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), message); return true; } void VideoBackend::PublishBackendStateChanged(const std::string& state, const std::string& message) { try { BackendStateChangedEvent event; event.backendName = "decklink"; event.state = state; event.message = message; if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("BackendStateChanged event publish failed.\n"); } catch (...) { OutputDebugStringA("BackendStateChanged event publish threw.\n"); } } void VideoBackend::PublishInputSignalChanged(const VideoIOFrame& frame, const VideoIOState& state) { const bool hasSignal = !frame.hasNoInputSource; const unsigned width = state.inputFrameSize.width; const unsigned height = state.inputFrameSize.height; if (mHasLastInputSignal && mLastInputSignal == hasSignal && mLastInputSignalWidth == width && mLastInputSignalHeight == height && mLastInputSignalModeName == state.inputDisplayModeName) { return; } mHasLastInputSignal = true; mLastInputSignal = hasSignal; mLastInputSignalWidth = width; mLastInputSignalHeight = height; mLastInputSignalModeName = state.inputDisplayModeName; try { InputSignalChangedEvent event; event.hasSignal = hasSignal; event.width = width; event.height = height; event.modeName = state.inputDisplayModeName; if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("InputSignalChanged event publish failed.\n"); } catch (...) { OutputDebugStringA("InputSignalChanged event publish threw.\n"); } } void VideoBackend::PublishInputFrameArrived(const VideoIOFrame& frame) { try { InputFrameArrivedEvent event; event.frameIndex = ++mInputFrameIndex; event.width = frame.width; event.height = frame.height; event.rowBytes = frame.rowBytes; event.pixelFormat = PixelFormatName(frame.pixelFormat); event.hasNoInputSource = frame.hasNoInputSource; if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("InputFrameArrived event publish failed.\n"); } catch (...) { OutputDebugStringA("InputFrameArrived event publish threw.\n"); } } void VideoBackend::PublishOutputFrameScheduled(const VideoIOOutputFrame& frame) { try { OutputFrameScheduledEvent event; event.frameIndex = ++mOutputFrameScheduleIndex; (void)frame; if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("OutputFrameScheduled event publish failed.\n"); } catch (...) { OutputDebugStringA("OutputFrameScheduled event publish threw.\n"); } } void VideoBackend::PublishOutputFrameCompleted(const VideoIOCompletion& completion) { try { OutputFrameCompletedEvent event; event.frameIndex = ++mOutputFrameCompletionIndex; event.result = CompletionResultName(completion.result); if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend")) OutputDebugStringA("OutputFrameCompleted event publish failed.\n"); } catch (...) { OutputDebugStringA("OutputFrameCompleted event publish threw.\n"); } } void VideoBackend::PublishTimingSample(const std::string& subsystem, const std::string& metric, double value, const std::string& unit) { try { TimingSampleRecordedEvent event; event.subsystem = subsystem; event.metric = metric; event.value = value; event.unit = unit; if (!mRuntimeEventDispatcher.PublishPayload(event, "HealthTelemetry")) OutputDebugStringA("TimingSampleRecorded event publish failed.\n"); } catch (...) { OutputDebugStringA("TimingSampleRecorded event publish threw.\n"); } } std::string VideoBackend::CompletionResultName(VideoIOCompletionResult result) { switch (result) { case VideoIOCompletionResult::Completed: return "Completed"; case VideoIOCompletionResult::DisplayedLate: return "DisplayedLate"; case VideoIOCompletionResult::Dropped: return "Dropped"; case VideoIOCompletionResult::Flushed: return "Flushed"; case VideoIOCompletionResult::Unknown: default: return "Unknown"; } } std::string VideoBackend::PixelFormatName(VideoIOPixelFormat pixelFormat) { return std::string(VideoIOPixelFormatName(pixelFormat)); }