video-shader-toys/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp

#include "VideoBackend.h"

#include "DeckLinkSession.h"
#include "OpenGLVideoIOBridge.h"
#include "HealthTelemetry.h"
#include "RenderEngine.h"
#include "RuntimeEventDispatcher.h"

#include <algorithm>
#include <chrono>
#include <cstring>
#include <cmath>
#include <windows.h>

VideoBackend::VideoBackend(RenderEngine& renderEngine, HealthTelemetry& healthTelemetry, RuntimeEventDispatcher& runtimeEventDispatcher) :
	mHealthTelemetry(healthTelemetry),
	mRuntimeEventDispatcher(runtimeEventDispatcher),
	mPlayoutPolicy(NormalizeVideoPlayoutPolicy(VideoPlayoutPolicy())),
	mOutputProductionController(mPlayoutPolicy),
	mReadyOutputQueue(mPlayoutPolicy),
	mVideoIODevice(std::make_unique<DeckLinkSession>()),
	mBridge(std::make_unique<OpenGLVideoIOBridge>(renderEngine))
{
}

VideoBackend::~VideoBackend()
{
	ReleaseResources();
}

void VideoBackend::ReleaseResources()
{
	StopOutputCompletionWorker();
	mReadyOutputQueue.Clear();
	if (mVideoIODevice)
		mVideoIODevice->ReleaseResources();
	mSystemOutputFramePool.Clear();
	if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped))
		ApplyLifecycleFailure("Video backend resources released before lifecycle completed.");
	ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend resources released.");
}

VideoBackendLifecycleState VideoBackend::LifecycleState() const
{
	return mLifecycle.State();
}

bool VideoBackend::DiscoverDevicesAndModes(const VideoFormatSelection& videoModes, std::string& error)
{
	ApplyLifecycleTransition(VideoBackendLifecycleState::Discovering, "Discovering video backend devices and modes.");
	if (mVideoIODevice->DiscoverDevicesAndModes(videoModes, error))
		return ApplyLifecycleTransition(VideoBackendLifecycleState::Discovered, "Video backend devices and modes discovered.");

	ApplyLifecycleFailure(error.empty() ? "Video backend discovery failed." : error);
	return false;
}

bool VideoBackend::SelectPreferredFormats(const VideoFormatSelection& videoModes, bool outputAlphaRequired, std::string& error)
{
	ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Selecting preferred video backend formats.");
	if (mVideoIODevice->SelectPreferredFormats(videoModes, outputAlphaRequired, error))
		return true;

	ApplyLifecycleFailure(error.empty() ? "Video backend format selection failed." : error);
	return false;
}

bool VideoBackend::ConfigureInput(const VideoFormat& inputVideoMode, std::string& error)
{
	if (mLifecycle.State() != VideoBackendLifecycleState::Configuring)
		ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend input.");
	if (!mVideoIODevice->ConfigureInput(
		[this](const VideoIOFrame& frame) { HandleInputFrame(frame); },
		inputVideoMode,
		error))
	{
		ApplyLifecycleFailure(error.empty() ? "Video backend input configuration failed." : error);
		return false;
	}
	return true;
}

bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error)
{
	mPlayoutPolicy = NormalizeVideoPlayoutPolicy(mPlayoutPolicy);
	mOutputProductionController.Configure(mPlayoutPolicy);
	mReadyOutputQueue.Configure(mPlayoutPolicy);
	if (mLifecycle.State() != VideoBackendLifecycleState::Configuring)
		ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend output.");
	if (!mVideoIODevice->ConfigureOutput(
		[this](const VideoIOCompletion& completion) { HandleOutputFrameCompletion(completion); },
		outputVideoMode,
		externalKeyingEnabled,
		error))
	{
		ApplyLifecycleFailure(error.empty() ? "Video backend output configuration failed." : error);
		return false;
	}
	SystemOutputFramePoolConfig poolConfig;
	poolConfig.width = mVideoIODevice->OutputFrameWidth();
	poolConfig.height = mVideoIODevice->OutputFrameHeight();
	poolConfig.pixelFormat = mVideoIODevice->OutputPixelFormat();
	poolConfig.rowBytes = mVideoIODevice->OutputFrameRowBytes();
	poolConfig.capacity = mPlayoutPolicy.outputFramePoolSize;
	mSystemOutputFramePool.Configure(poolConfig);
	RecordSystemMemoryPlayoutStats();
	return ApplyLifecycleTransition(VideoBackendLifecycleState::Configured, "Video backend configured.");
}

bool VideoBackend::Start()
{
	ApplyLifecycleTransition(VideoBackendLifecycleState::Prerolling, "Video backend preroll starting.");
	StartOutputCompletionWorker();
	const bool started = mVideoIODevice->Start();
	if (started)
	{
		StartOutputProducerWorker();
		ApplyLifecycleTransition(VideoBackendLifecycleState::Running, "Video backend started.");
	}
	else
	{
		StopOutputCompletionWorker();
		ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend start failed." : StatusMessage());
	}
	return started;
}

bool VideoBackend::Stop()
{
	ApplyLifecycleTransition(VideoBackendLifecycleState::Stopping, "Video backend stopping.");
	StopOutputProducerWorker();
	const bool stopped = mVideoIODevice->Stop();
	StopOutputCompletionWorker();
	if (stopped)
		ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend stopped.");
	else
		ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend stop failed." : StatusMessage());
	return stopped;
}

const VideoIOState& VideoBackend::State() const
{
	return mVideoIODevice->State();
}

VideoIOState& VideoBackend::MutableState()
{
	return mVideoIODevice->MutableState();
}

bool VideoBackend::BeginOutputFrame(VideoIOOutputFrame& frame)
{
	return mVideoIODevice->BeginOutputFrame(frame);
}

void VideoBackend::EndOutputFrame(VideoIOOutputFrame& frame)
{
	mVideoIODevice->EndOutputFrame(frame);
}

bool VideoBackend::ScheduleOutputFrame(const VideoIOOutputFrame& frame)
{
	return mVideoIODevice->ScheduleOutputFrame(frame);
}

VideoPlayoutRecoveryDecision VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth)
{
	return mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth);
}

bool VideoBackend::HasInputDevice() const
{
	return mVideoIODevice->HasInputDevice();
}

bool VideoBackend::HasInputSource() const
{
	return mVideoIODevice->HasInputSource();
}

unsigned VideoBackend::InputFrameWidth() const
{
	return mVideoIODevice->InputFrameWidth();
}

unsigned VideoBackend::InputFrameHeight() const
{
	return mVideoIODevice->InputFrameHeight();
}

unsigned VideoBackend::OutputFrameWidth() const
{
	return mVideoIODevice->OutputFrameWidth();
}

unsigned VideoBackend::OutputFrameHeight() const
{
	return mVideoIODevice->OutputFrameHeight();
}

unsigned VideoBackend::CaptureTextureWidth() const
{
	return mVideoIODevice->CaptureTextureWidth();
}

unsigned VideoBackend::OutputPackTextureWidth() const
{
	return mVideoIODevice->OutputPackTextureWidth();
}

VideoIOPixelFormat VideoBackend::InputPixelFormat() const
{
	return mVideoIODevice->InputPixelFormat();
}

const std::string& VideoBackend::InputDisplayModeName() const
{
	return mVideoIODevice->InputDisplayModeName();
}

const std::string& VideoBackend::OutputModelName() const
{
	return mVideoIODevice->OutputModelName();
}

bool VideoBackend::SupportsInternalKeying() const
{
	return mVideoIODevice->SupportsInternalKeying();
}

bool VideoBackend::SupportsExternalKeying() const
{
	return mVideoIODevice->SupportsExternalKeying();
}

bool VideoBackend::KeyerInterfaceAvailable() const
{
	return mVideoIODevice->KeyerInterfaceAvailable();
}

bool VideoBackend::ExternalKeyingActive() const
{
	return mVideoIODevice->ExternalKeyingActive();
}

const std::string& VideoBackend::StatusMessage() const
{
	return mVideoIODevice->StatusMessage();
}

bool VideoBackend::ShouldPrioritizeOutputOverPreview() const
{
	const RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics();
	return metrics.depth < static_cast<std::size_t>(mPlayoutPolicy.targetReadyFrames);
}

void VideoBackend::SetStatusMessage(const std::string& message)
{
	mVideoIODevice->SetStatusMessage(message);
}

void VideoBackend::PublishStatus(bool externalKeyingConfigured, const std::string& statusMessage)
{
	if (!statusMessage.empty())
		SetStatusMessage(statusMessage);

	mHealthTelemetry.ReportVideoIOStatus(
		"decklink",
		OutputModelName(),
		SupportsInternalKeying(),
		SupportsExternalKeying(),
		KeyerInterfaceAvailable(),
		externalKeyingConfigured,
		ExternalKeyingActive(),
		StatusMessage());
	PublishBackendStateChanged(VideoBackendLifecycle::StateName(mLifecycle.State()), StatusMessage());
}

void VideoBackend::ReportNoInputDeviceSignalStatus()
{
	mHealthTelemetry.ReportSignalStatus(
		false,
		InputFrameWidth(),
		InputFrameHeight(),
		InputDisplayModeName());
	PublishBackendStateChanged("no-input-device", "No input device is available.");
}

void VideoBackend::HandleInputFrame(const VideoIOFrame& frame)
{
	const VideoIOState& state = mVideoIODevice->State();
	mHealthTelemetry.TryReportSignalStatus(!frame.hasNoInputSource, state.inputFrameSize.width, state.inputFrameSize.height, state.inputDisplayModeName);
	PublishInputSignalChanged(frame, state);
	PublishInputFrameArrived(frame);

	if (mBridge)
		mBridge->UploadInputFrame(frame, state);
}

void VideoBackend::HandleOutputFrameCompletion(const VideoIOCompletion& completion)
{
	{
		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
		if (!mOutputCompletionWorkerRunning || mOutputCompletionWorkerStopping)
			return;
		mPendingOutputCompletions.push_back(completion);
	}
	mOutputCompletionCondition.notify_one();
}

void VideoBackend::StartOutputCompletionWorker()
{
	{
		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
		if (mOutputCompletionWorkerRunning)
			return;

		mPendingOutputCompletions.clear();
		mReadyOutputQueue.Clear();
		mNextReadyOutputFrameIndex = 0;
		mHasReadyQueueDepthBaseline = false;
		mMinReadyQueueDepth = 0;
		mMaxReadyQueueDepth = 0;
		mReadyQueueZeroDepthCount = 0;
		mOutputRenderMilliseconds = 0.0;
		mSmoothedOutputRenderMilliseconds = 0.0;
		mMaxOutputRenderMilliseconds = 0.0;
		mOutputFrameAcquireMilliseconds = 0.0;
		mOutputFrameRenderRequestMilliseconds = 0.0;
		mOutputFrameEndAccessMilliseconds = 0.0;
		mLastLateStreak = 0;
		mLastDropStreak = 0;
		mOutputCompletionWorkerStopping = false;
		mOutputCompletionWorkerRunning = true;
		mOutputCompletionWorker = std::thread(&VideoBackend::OutputCompletionWorkerMain, this);
	}
}

void VideoBackend::StopOutputCompletionWorker()
{
	StopOutputProducerWorker();

	bool shouldJoin = false;
	{
		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
		if (mOutputCompletionWorkerRunning)
			mOutputCompletionWorkerStopping = true;
		shouldJoin = mOutputCompletionWorker.joinable();
	}
	mOutputCompletionCondition.notify_one();

	if (shouldJoin)
		mOutputCompletionWorker.join();
}

void VideoBackend::StartOutputProducerWorker()
{
	std::lock_guard<std::mutex> lock(mOutputProducerMutex);
	if (mOutputProducerWorkerRunning)
		return;

	const double frameBudgetMilliseconds = State().frameBudgetMilliseconds;
	const auto frameDuration = frameBudgetMilliseconds > 0.0
		? std::chrono::duration_cast<RenderCadenceController::Duration>(
			std::chrono::duration<double, std::milli>(frameBudgetMilliseconds))
		: std::chrono::milliseconds(16);
	mRenderCadenceController.Configure(frameDuration, std::chrono::steady_clock::now());
	mLastOutputProductionCompletion = VideoIOCompletion();
	mLastOutputProductionTime = std::chrono::steady_clock::time_point();
	mOutputProducerWorkerStopping = false;
	mOutputProducerWorkerRunning = true;
	mOutputProducerWorker = std::thread(&VideoBackend::OutputProducerWorkerMain, this);
	mOutputProducerCondition.notify_one();
}

void VideoBackend::StopOutputProducerWorker()
{
	bool shouldJoin = false;
	{
		std::lock_guard<std::mutex> lock(mOutputProducerMutex);
		if (mOutputProducerWorkerRunning)
			mOutputProducerWorkerStopping = true;
		shouldJoin = mOutputProducerWorker.joinable();
	}
	mOutputProducerCondition.notify_one();

	if (shouldJoin)
		mOutputProducerWorker.join();
}

void VideoBackend::NotifyOutputProducer()
{
	mOutputProducerCondition.notify_one();
}

void VideoBackend::OutputCompletionWorkerMain()
{
	for (;;)
	{
		VideoIOCompletion completion;
		{
			std::unique_lock<std::mutex> lock(mOutputCompletionMutex);
			mOutputCompletionCondition.wait(lock, [this]() {
				return mOutputCompletionWorkerStopping || !mPendingOutputCompletions.empty();
			});

			if (mPendingOutputCompletions.empty())
			{
				if (mOutputCompletionWorkerStopping)
				{
					mOutputCompletionWorkerRunning = false;
					return;
				}
				continue;
			}

			completion = mPendingOutputCompletions.front();
			mPendingOutputCompletions.pop_front();
		}

		ProcessOutputFrameCompletion(completion);
	}
}

void VideoBackend::OutputProducerWorkerMain()
{
	for (;;)
	{
		{
			std::lock_guard<std::mutex> lock(mOutputProducerMutex);
			if (mOutputProducerWorkerStopping)
			{
				mOutputProducerWorkerRunning = false;
				return;
			}
		}

		ScheduleReadyOutputFramesToTarget();

		const RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics();
		RecordReadyQueueDepthSample(metrics);

		const auto now = std::chrono::steady_clock::now();
		RenderCadenceDecision cadenceDecision = mRenderCadenceController.Tick(now);
		if (cadenceDecision.action == RenderCadenceAction::Wait)
		{
			const auto waitDuration = (std::min)(
				std::chrono::duration_cast<std::chrono::milliseconds>(cadenceDecision.waitDuration),
				OutputProducerWakeInterval());
			std::unique_lock<std::mutex> lock(mOutputProducerMutex);
			mOutputProducerCondition.wait_for(lock, waitDuration);
			if (mOutputProducerWorkerStopping)
			{
				mOutputProducerWorkerRunning = false;
				return;
			}
			continue;
		}

		VideoIOCompletion completion;
		{
			std::lock_guard<std::mutex> lock(mOutputProducerMutex);
			if (mOutputProducerWorkerStopping)
				continue;
			completion = mLastOutputProductionCompletion;
		}

		const std::size_t producedFrames = ProduceReadyOutputFrames(completion, 1);
		if (producedFrames > 0)
		{
			mLastOutputProductionTime = std::chrono::steady_clock::now();
			ScheduleReadyOutputFramesToTarget();
			continue;
		}

		{
			std::unique_lock<std::mutex> lock(mOutputProducerMutex);
			mOutputProducerCondition.wait_for(lock, OutputProducerWakeInterval());
			if (mOutputProducerWorkerStopping)
			{
				mOutputProducerWorkerRunning = false;
				return;
			}
		}
	}
}

std::chrono::milliseconds VideoBackend::OutputProducerWakeInterval() const
{
	const double frameBudgetMilliseconds = State().frameBudgetMilliseconds;
	if (frameBudgetMilliseconds <= 0.0)
		return std::chrono::milliseconds(8);

	const int intervalMilliseconds = (std::max)(1, static_cast<int>(std::floor(frameBudgetMilliseconds * 0.75)));
	return std::chrono::milliseconds(intervalMilliseconds);
}

void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion)
{
	if (completion.outputFrameBuffer != nullptr)
		mSystemOutputFramePool.ReleaseSlotByBuffer(completion.outputFrameBuffer);
	RecordFramePacing(completion.result);
	PublishOutputFrameCompleted(completion);
	const RenderOutputQueueMetrics initialQueueMetrics = mReadyOutputQueue.GetMetrics();
	RecordReadyQueueDepthSample(initialQueueMetrics);
	const VideoPlayoutRecoveryDecision recoveryDecision = AccountForCompletionResult(completion.result, initialQueueMetrics.depth);
	{
		std::lock_guard<std::mutex> lock(mOutputMetricsMutex);
		mLastLateStreak = recoveryDecision.lateStreak;
		mLastDropStreak = recoveryDecision.dropStreak;
	}
	{
		std::lock_guard<std::mutex> lock(mOutputProducerMutex);
		mLastOutputProductionCompletion = completion;
	}
	NotifyOutputProducer();

	RecordBackendPlayoutHealth(completion.result, recoveryDecision);
	RecordSystemMemoryPlayoutStats();
}

std::size_t VideoBackend::ScheduleReadyOutputFramesToTarget()
{
	const std::size_t targetScheduledFrames = static_cast<std::size_t>(mPlayoutPolicy.targetPrerollFrames);
	std::size_t scheduledFrames = 0;
	for (;;)
	{
		const SystemOutputFramePoolMetrics poolMetrics = mSystemOutputFramePool.GetMetrics();
		if (poolMetrics.scheduledCount >= targetScheduledFrames)
			break;
		if (!ScheduleReadyOutputFrame())
			break;
		++scheduledFrames;
	}
	return scheduledFrames;
}

void VideoBackend::RecordBackendPlayoutHealth(VideoIOCompletionResult result, const VideoPlayoutRecoveryDecision& recoveryDecision)
{
	const RenderOutputQueueMetrics queueMetrics = mReadyOutputQueue.GetMetrics();
	std::size_t minReadyQueueDepth = 0;
	std::size_t maxReadyQueueDepth = 0;
	uint64_t readyQueueZeroDepthCount = 0;
	double outputRenderMilliseconds = 0.0;
	double smoothedOutputRenderMilliseconds = 0.0;
	double maxOutputRenderMilliseconds = 0.0;
	double outputFrameAcquireMilliseconds = 0.0;
	double outputFrameRenderRequestMilliseconds = 0.0;
	double outputFrameEndAccessMilliseconds = 0.0;
	{
		std::lock_guard<std::mutex> lock(mOutputMetricsMutex);
		minReadyQueueDepth = mMinReadyQueueDepth;
		maxReadyQueueDepth = mMaxReadyQueueDepth;
		readyQueueZeroDepthCount = mReadyQueueZeroDepthCount;
		outputRenderMilliseconds = mOutputRenderMilliseconds;
		smoothedOutputRenderMilliseconds = mSmoothedOutputRenderMilliseconds;
		maxOutputRenderMilliseconds = mMaxOutputRenderMilliseconds;
		outputFrameAcquireMilliseconds = mOutputFrameAcquireMilliseconds;
		outputFrameRenderRequestMilliseconds = mOutputFrameRenderRequestMilliseconds;
		outputFrameEndAccessMilliseconds = mOutputFrameEndAccessMilliseconds;
	}

	mHealthTelemetry.TryRecordBackendPlayoutHealth(
		VideoBackendLifecycle::StateName(mLifecycle.State()),
		CompletionResultName(result),
		queueMetrics.depth,
		queueMetrics.capacity,
		queueMetrics.pushedCount,
		minReadyQueueDepth,
		maxReadyQueueDepth,
		readyQueueZeroDepthCount,
		queueMetrics.poppedCount,
		queueMetrics.droppedCount,
		queueMetrics.underrunCount,
		outputRenderMilliseconds,
		smoothedOutputRenderMilliseconds,
		maxOutputRenderMilliseconds,
		outputFrameAcquireMilliseconds,
		outputFrameRenderRequestMilliseconds,
		outputFrameEndAccessMilliseconds,
		recoveryDecision.completedFrameIndex,
		recoveryDecision.scheduledFrameIndex,
		recoveryDecision.scheduledLeadFrames,
		recoveryDecision.measuredLagFrames,
		recoveryDecision.catchUpFrames,
		recoveryDecision.lateStreak,
		recoveryDecision.dropStreak,
		mLateFrameCount,
		mDroppedFrameCount,
		mFlushedFrameCount,
		mLifecycle.State() == VideoBackendLifecycleState::Degraded,
		StatusMessage());
}

std::size_t VideoBackend::ProduceReadyOutputFrames(const VideoIOCompletion& completion, std::size_t maxFrames)
{
	if (maxFrames == 0)
		return 0;

	std::lock_guard<std::mutex> productionLock(mOutputProductionMutex);
	RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics();
	std::size_t producedFrames = 0;
	while (producedFrames < maxFrames)
	{
		if (!RenderReadyOutputFrame(mVideoIODevice->State(), completion))
			break;
		++producedFrames;
		metrics = mReadyOutputQueue.GetMetrics();
		RecordReadyQueueDepthSample(metrics);
	}
	return producedFrames;
}

OutputProductionPressure VideoBackend::BuildOutputProductionPressure(const RenderOutputQueueMetrics& metrics) const
{
	OutputProductionPressure pressure;
	pressure.readyQueueDepth = metrics.depth;
	pressure.readyQueueCapacity = metrics.capacity;
	pressure.readyQueueUnderrunCount = metrics.underrunCount;
	{
		std::lock_guard<std::mutex> lock(mOutputMetricsMutex);
		pressure.lateStreak = mLastLateStreak;
		pressure.dropStreak = mLastDropStreak;
	}
	return pressure;
}

bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion)
{
	const auto renderStart = std::chrono::steady_clock::now();
	OutputFrameSlot outputSlot;
	VideoIOOutputFrame outputFrame;
	const auto acquireStart = std::chrono::steady_clock::now();
	if (!mSystemOutputFramePool.AcquireFreeSlot(outputSlot))
	{
		if (!mReadyOutputQueue.DropOldestFrame() || !mSystemOutputFramePool.AcquireFreeSlot(outputSlot))
			return false;
	}
	outputFrame = outputSlot.frame;
	const auto acquireEnd = std::chrono::steady_clock::now();

	bool rendered = true;
	const auto renderRequestStart = std::chrono::steady_clock::now();
	if (mBridge)
		rendered = mBridge->RenderScheduledFrame(state, completion, outputFrame);
	const auto renderRequestEnd = std::chrono::steady_clock::now();

	const auto endAccessStart = std::chrono::steady_clock::now();
	const bool publishedReady = mSystemOutputFramePool.PublishReadySlot(outputSlot);
	const auto endAccessEnd = std::chrono::steady_clock::now();
	const double acquireMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(acquireEnd - acquireStart).count();
	const double renderRequestMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(renderRequestEnd - renderRequestStart).count();
	const double endAccessMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(endAccessEnd - endAccessStart).count();

	if (!rendered)
	{
		mSystemOutputFramePool.ReleaseSlot(outputSlot);
		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame.");
		const double renderMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(
			std::chrono::steady_clock::now() - renderStart).count();
		RecordOutputRenderDuration(renderMilliseconds, acquireMilliseconds, renderRequestMilliseconds, endAccessMilliseconds);
		return false;
	}

	if (!publishedReady)
	{
		mSystemOutputFramePool.ReleaseSlot(outputSlot);
		return false;
	}

	const double renderMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(
		std::chrono::steady_clock::now() - renderStart).count();
	RecordOutputRenderDuration(renderMilliseconds, acquireMilliseconds, renderRequestMilliseconds, endAccessMilliseconds);

	RenderOutputFrame readyFrame;
	readyFrame.frame = outputFrame;
	readyFrame.frameIndex = ++mNextReadyOutputFrameIndex;
	readyFrame.releaseFrame = [this](VideoIOOutputFrame& frame) {
		mSystemOutputFramePool.ReleaseSlotByBuffer(frame.bytes);
	};
	const bool pushed = mReadyOutputQueue.Push(readyFrame);
	if (!pushed)
		mSystemOutputFramePool.ReleaseSlot(outputSlot);
	RecordSystemMemoryPlayoutStats();
	return pushed;
}

bool VideoBackend::ScheduleReadyOutputFrame()
{
	std::lock_guard<std::mutex> schedulingLock(mOutputSchedulingMutex);
	RenderOutputFrame readyFrame;
	if (!mReadyOutputQueue.TryPop(readyFrame))
		return false;
	RecordReadyQueueDepthSample(mReadyOutputQueue.GetMetrics());

	if (!mSystemOutputFramePool.MarkScheduledByBuffer(readyFrame.frame.bytes))
	{
		if (readyFrame.releaseFrame)
			readyFrame.releaseFrame(readyFrame.frame);
		return false;
	}

	if (!ScheduleOutputFrame(readyFrame.frame))
	{
		RecordDeckLinkBufferTelemetry();
		mSystemOutputFramePool.ReleaseSlotByBuffer(readyFrame.frame.bytes);
		return false;
	}

	RecordDeckLinkBufferTelemetry();
	PublishOutputFrameScheduled(readyFrame.frame);
	RecordSystemMemoryPlayoutStats();
	return true;
}

bool VideoBackend::ScheduleBlackUnderrunFrame()
{
	VideoIOOutputFrame outputFrame;
	if (!BeginOutputFrame(outputFrame))
	{
		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: no output frame was available for fallback scheduling.");
		return false;
	}

	if (outputFrame.bytes != nullptr && outputFrame.rowBytes > 0 && outputFrame.height > 0)
		std::memset(outputFrame.bytes, 0, static_cast<std::size_t>(outputFrame.rowBytes) * outputFrame.height);
	EndOutputFrame(outputFrame);

	if (!ScheduleOutputFrame(outputFrame))
	{
		RecordDeckLinkBufferTelemetry();
		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: black fallback frame scheduling failed.");
		return false;
	}

	RecordDeckLinkBufferTelemetry();
	ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: scheduled black fallback frame.");
	PublishOutputFrameScheduled(outputFrame);
	return true;
}

void VideoBackend::RecordFramePacing(VideoIOCompletionResult completionResult)
{
	const auto now = std::chrono::steady_clock::now();
	if (mLastPlayoutCompletionTime != std::chrono::steady_clock::time_point())
	{
		mCompletionIntervalMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(now - mLastPlayoutCompletionTime).count();
		if (mSmoothedCompletionIntervalMilliseconds <= 0.0)
			mSmoothedCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds;
		else
			mSmoothedCompletionIntervalMilliseconds = mSmoothedCompletionIntervalMilliseconds * 0.9 + mCompletionIntervalMilliseconds * 0.1;
		if (mCompletionIntervalMilliseconds > mMaxCompletionIntervalMilliseconds)
			mMaxCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds;
	}
	mLastPlayoutCompletionTime = now;

	if (completionResult == VideoIOCompletionResult::DisplayedLate)
		++mLateFrameCount;
	else if (completionResult == VideoIOCompletionResult::Dropped)
		++mDroppedFrameCount;
	else if (completionResult == VideoIOCompletionResult::Flushed)
		++mFlushedFrameCount;

	mHealthTelemetry.TryRecordFramePacingStats(
		mCompletionIntervalMilliseconds,
		mSmoothedCompletionIntervalMilliseconds,
		mMaxCompletionIntervalMilliseconds,
		mLateFrameCount,
		mDroppedFrameCount,
		mFlushedFrameCount);
	PublishTimingSample("VideoBackend", "completionInterval", mCompletionIntervalMilliseconds, "ms");
	PublishTimingSample("VideoBackend", "smoothedCompletionInterval", mSmoothedCompletionIntervalMilliseconds, "ms");
}

void VideoBackend::RecordReadyQueueDepthSample(const RenderOutputQueueMetrics& metrics)
{
	std::lock_guard<std::mutex> lock(mOutputMetricsMutex);
	if (!mHasReadyQueueDepthBaseline)
	{
		mHasReadyQueueDepthBaseline = true;
		mMinReadyQueueDepth = metrics.depth;
		mMaxReadyQueueDepth = metrics.depth;
	}
	else
	{
		mMinReadyQueueDepth = (std::min)(mMinReadyQueueDepth, metrics.depth);
		mMaxReadyQueueDepth = (std::max)(mMaxReadyQueueDepth, metrics.depth);
	}

	if (metrics.depth == 0)
		++mReadyQueueZeroDepthCount;
}

void VideoBackend::RecordDeckLinkBufferTelemetry()
{
	if (!mVideoIODevice)
		return;

	const VideoIOState& state = mVideoIODevice->State();
	mHealthTelemetry.TryRecordDeckLinkBufferTelemetry(
		state.actualDeckLinkBufferedFramesAvailable,
		state.actualDeckLinkBufferedFrames,
		static_cast<std::size_t>(mPlayoutPolicy.targetPrerollFrames),
		state.deckLinkScheduleCallMilliseconds,
		state.deckLinkScheduleFailureCount);
}

void VideoBackend::RecordSystemMemoryPlayoutStats()
{
	const SystemOutputFramePoolMetrics poolMetrics = mSystemOutputFramePool.GetMetrics();
	const RenderOutputQueueMetrics queueMetrics = mReadyOutputQueue.GetMetrics();
	RecordDeckLinkBufferTelemetry();
	mHealthTelemetry.TryRecordSystemMemoryPlayoutStats(
		poolMetrics.freeCount,
		poolMetrics.readyCount,
		poolMetrics.scheduledCount,
		poolMetrics.readyUnderrunCount,
		0,
		queueMetrics.droppedCount,
		0.0,
		0.0);
}

void VideoBackend::RecordOutputRenderDuration(double renderMilliseconds, double acquireMilliseconds, double renderRequestMilliseconds, double endAccessMilliseconds)
{
	std::lock_guard<std::mutex> lock(mOutputMetricsMutex);
	mOutputRenderMilliseconds = (std::max)(renderMilliseconds, 0.0);
	if (mSmoothedOutputRenderMilliseconds <= 0.0)
		mSmoothedOutputRenderMilliseconds = mOutputRenderMilliseconds;
	else
		mSmoothedOutputRenderMilliseconds = mSmoothedOutputRenderMilliseconds * 0.9 + mOutputRenderMilliseconds * 0.1;
	mMaxOutputRenderMilliseconds = (std::max)(mMaxOutputRenderMilliseconds, mOutputRenderMilliseconds);
	mOutputFrameAcquireMilliseconds = (std::max)(acquireMilliseconds, 0.0);
	mOutputFrameRenderRequestMilliseconds = (std::max)(renderRequestMilliseconds, 0.0);
	mOutputFrameEndAccessMilliseconds = (std::max)(endAccessMilliseconds, 0.0);

	PublishTimingSample("VideoBackend", "outputRender", mOutputRenderMilliseconds, "ms");
	PublishTimingSample("VideoBackend", "smoothedOutputRender", mSmoothedOutputRenderMilliseconds, "ms");
}

bool VideoBackend::ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message)
{
	const VideoBackendLifecycleTransition transition = mLifecycle.TransitionTo(state, message);
	if (!transition.accepted)
	{
		PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), transition.errorMessage);
		return false;
	}

	PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), message);
	return true;
}

bool VideoBackend::ApplyLifecycleFailure(const std::string& message)
{
	const VideoBackendLifecycleTransition transition = mLifecycle.Fail(message);
	if (!transition.accepted)
	{
		PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), transition.errorMessage);
		return false;
	}

	PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), message);
	return true;
}

void VideoBackend::PublishBackendStateChanged(const std::string& state, const std::string& message)
{
	try
	{
		BackendStateChangedEvent event;
		event.backendName = "decklink";
		event.state = state;
		event.message = message;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("BackendStateChanged event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("BackendStateChanged event publish threw.\n");
	}
}

void VideoBackend::PublishInputSignalChanged(const VideoIOFrame& frame, const VideoIOState& state)
{
	const bool hasSignal = !frame.hasNoInputSource;
	const unsigned width = state.inputFrameSize.width;
	const unsigned height = state.inputFrameSize.height;
	if (mHasLastInputSignal &&
		mLastInputSignal == hasSignal &&
		mLastInputSignalWidth == width &&
		mLastInputSignalHeight == height &&
		mLastInputSignalModeName == state.inputDisplayModeName)
	{
		return;
	}

	mHasLastInputSignal = true;
	mLastInputSignal = hasSignal;
	mLastInputSignalWidth = width;
	mLastInputSignalHeight = height;
	mLastInputSignalModeName = state.inputDisplayModeName;

	try
	{
		InputSignalChangedEvent event;
		event.hasSignal = hasSignal;
		event.width = width;
		event.height = height;
		event.modeName = state.inputDisplayModeName;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("InputSignalChanged event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("InputSignalChanged event publish threw.\n");
	}
}

void VideoBackend::PublishInputFrameArrived(const VideoIOFrame& frame)
{
	try
	{
		InputFrameArrivedEvent event;
		event.frameIndex = ++mInputFrameIndex;
		event.width = frame.width;
		event.height = frame.height;
		event.rowBytes = frame.rowBytes;
		event.pixelFormat = PixelFormatName(frame.pixelFormat);
		event.hasNoInputSource = frame.hasNoInputSource;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("InputFrameArrived event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("InputFrameArrived event publish threw.\n");
	}
}

void VideoBackend::PublishOutputFrameScheduled(const VideoIOOutputFrame& frame)
{
	try
	{
		OutputFrameScheduledEvent event;
		event.frameIndex = ++mOutputFrameScheduleIndex;
		(void)frame;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("OutputFrameScheduled event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("OutputFrameScheduled event publish threw.\n");
	}
}

void VideoBackend::PublishOutputFrameCompleted(const VideoIOCompletion& completion)
{
	try
	{
		OutputFrameCompletedEvent event;
		event.frameIndex = ++mOutputFrameCompletionIndex;
		event.result = CompletionResultName(completion.result);
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("OutputFrameCompleted event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("OutputFrameCompleted event publish threw.\n");
	}
}

void VideoBackend::PublishTimingSample(const std::string& subsystem, const std::string& metric, double value, const std::string& unit)
{
	try
	{
		TimingSampleRecordedEvent event;
		event.subsystem = subsystem;
		event.metric = metric;
		event.value = value;
		event.unit = unit;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "HealthTelemetry"))
			OutputDebugStringA("TimingSampleRecorded event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("TimingSampleRecorded event publish threw.\n");
	}
}

std::string VideoBackend::CompletionResultName(VideoIOCompletionResult result)
{
	switch (result)
	{
	case VideoIOCompletionResult::Completed:
		return "Completed";
	case VideoIOCompletionResult::DisplayedLate:
		return "DisplayedLate";
	case VideoIOCompletionResult::Dropped:
		return "Dropped";
	case VideoIOCompletionResult::Flushed:
		return "Flushed";
	case VideoIOCompletionResult::Unknown:
	default:
		return "Unknown";
	}
}

std::string VideoBackend::PixelFormatName(VideoIOPixelFormat pixelFormat)
{
	return std::string(VideoIOPixelFormatName(pixelFormat));
}