video-shader-toys/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp

#include "VideoBackend.h"

#include "DeckLinkSession.h"
#include "OpenGLVideoIOBridge.h"
#include "HealthTelemetry.h"
#include "RenderEngine.h"
#include "RuntimeEventDispatcher.h"

#include <chrono>
#include <cstring>
#include <windows.h>

VideoBackend::VideoBackend(RenderEngine& renderEngine, HealthTelemetry& healthTelemetry, RuntimeEventDispatcher& runtimeEventDispatcher) :
	mHealthTelemetry(healthTelemetry),
	mRuntimeEventDispatcher(runtimeEventDispatcher),
	mPlayoutPolicy(NormalizeVideoPlayoutPolicy(VideoPlayoutPolicy())),
	mReadyOutputQueue(mPlayoutPolicy),
	mVideoIODevice(std::make_unique<DeckLinkSession>()),
	mBridge(std::make_unique<OpenGLVideoIOBridge>(renderEngine))
{
}

VideoBackend::~VideoBackend()
{
	ReleaseResources();
}

void VideoBackend::ReleaseResources()
{
	StopOutputCompletionWorker();
	mReadyOutputQueue.Clear();
	if (mVideoIODevice)
		mVideoIODevice->ReleaseResources();
	if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped))
		ApplyLifecycleFailure("Video backend resources released before lifecycle completed.");
	ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend resources released.");
}

VideoBackendLifecycleState VideoBackend::LifecycleState() const
{
	return mLifecycle.State();
}

bool VideoBackend::DiscoverDevicesAndModes(const VideoFormatSelection& videoModes, std::string& error)
{
	ApplyLifecycleTransition(VideoBackendLifecycleState::Discovering, "Discovering video backend devices and modes.");
	if (mVideoIODevice->DiscoverDevicesAndModes(videoModes, error))
		return ApplyLifecycleTransition(VideoBackendLifecycleState::Discovered, "Video backend devices and modes discovered.");

	ApplyLifecycleFailure(error.empty() ? "Video backend discovery failed." : error);
	return false;
}

bool VideoBackend::SelectPreferredFormats(const VideoFormatSelection& videoModes, bool outputAlphaRequired, std::string& error)
{
	ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Selecting preferred video backend formats.");
	if (mVideoIODevice->SelectPreferredFormats(videoModes, outputAlphaRequired, error))
		return true;

	ApplyLifecycleFailure(error.empty() ? "Video backend format selection failed." : error);
	return false;
}

bool VideoBackend::ConfigureInput(const VideoFormat& inputVideoMode, std::string& error)
{
	if (mLifecycle.State() != VideoBackendLifecycleState::Configuring)
		ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend input.");
	if (!mVideoIODevice->ConfigureInput(
		[this](const VideoIOFrame& frame) { HandleInputFrame(frame); },
		inputVideoMode,
		error))
	{
		ApplyLifecycleFailure(error.empty() ? "Video backend input configuration failed." : error);
		return false;
	}
	return true;
}

bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error)
{
	mPlayoutPolicy = NormalizeVideoPlayoutPolicy(mPlayoutPolicy);
	mReadyOutputQueue.Configure(mPlayoutPolicy);
	if (mLifecycle.State() != VideoBackendLifecycleState::Configuring)
		ApplyLifecycleTransition(VideoBackendLifecycleState::Configuring, "Configuring video backend output.");
	if (!mVideoIODevice->ConfigureOutput(
		[this](const VideoIOCompletion& completion) { HandleOutputFrameCompletion(completion); },
		outputVideoMode,
		externalKeyingEnabled,
		error))
	{
		ApplyLifecycleFailure(error.empty() ? "Video backend output configuration failed." : error);
		return false;
	}
	return ApplyLifecycleTransition(VideoBackendLifecycleState::Configured, "Video backend configured.");
}

bool VideoBackend::Start()
{
	ApplyLifecycleTransition(VideoBackendLifecycleState::Prerolling, "Video backend preroll starting.");
	StartOutputCompletionWorker();
	const bool started = mVideoIODevice->Start();
	if (started)
		ApplyLifecycleTransition(VideoBackendLifecycleState::Running, "Video backend started.");
	else
	{
		StopOutputCompletionWorker();
		ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend start failed." : StatusMessage());
	}
	return started;
}

bool VideoBackend::Stop()
{
	ApplyLifecycleTransition(VideoBackendLifecycleState::Stopping, "Video backend stopping.");
	const bool stopped = mVideoIODevice->Stop();
	StopOutputCompletionWorker();
	if (stopped)
		ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend stopped.");
	else
		ApplyLifecycleFailure(StatusMessage().empty() ? "Video backend stop failed." : StatusMessage());
	return stopped;
}

const VideoIOState& VideoBackend::State() const
{
	return mVideoIODevice->State();
}

VideoIOState& VideoBackend::MutableState()
{
	return mVideoIODevice->MutableState();
}

bool VideoBackend::BeginOutputFrame(VideoIOOutputFrame& frame)
{
	return mVideoIODevice->BeginOutputFrame(frame);
}

void VideoBackend::EndOutputFrame(VideoIOOutputFrame& frame)
{
	mVideoIODevice->EndOutputFrame(frame);
}

bool VideoBackend::ScheduleOutputFrame(const VideoIOOutputFrame& frame)
{
	return mVideoIODevice->ScheduleOutputFrame(frame);
}

void VideoBackend::AccountForCompletionResult(VideoIOCompletionResult result, uint64_t readyQueueDepth)
{
	mVideoIODevice->AccountForCompletionResult(result, readyQueueDepth);
}

bool VideoBackend::HasInputDevice() const
{
	return mVideoIODevice->HasInputDevice();
}

bool VideoBackend::HasInputSource() const
{
	return mVideoIODevice->HasInputSource();
}

unsigned VideoBackend::InputFrameWidth() const
{
	return mVideoIODevice->InputFrameWidth();
}

unsigned VideoBackend::InputFrameHeight() const
{
	return mVideoIODevice->InputFrameHeight();
}

unsigned VideoBackend::OutputFrameWidth() const
{
	return mVideoIODevice->OutputFrameWidth();
}

unsigned VideoBackend::OutputFrameHeight() const
{
	return mVideoIODevice->OutputFrameHeight();
}

unsigned VideoBackend::CaptureTextureWidth() const
{
	return mVideoIODevice->CaptureTextureWidth();
}

unsigned VideoBackend::OutputPackTextureWidth() const
{
	return mVideoIODevice->OutputPackTextureWidth();
}

VideoIOPixelFormat VideoBackend::InputPixelFormat() const
{
	return mVideoIODevice->InputPixelFormat();
}

const std::string& VideoBackend::InputDisplayModeName() const
{
	return mVideoIODevice->InputDisplayModeName();
}

const std::string& VideoBackend::OutputModelName() const
{
	return mVideoIODevice->OutputModelName();
}

bool VideoBackend::SupportsInternalKeying() const
{
	return mVideoIODevice->SupportsInternalKeying();
}

bool VideoBackend::SupportsExternalKeying() const
{
	return mVideoIODevice->SupportsExternalKeying();
}

bool VideoBackend::KeyerInterfaceAvailable() const
{
	return mVideoIODevice->KeyerInterfaceAvailable();
}

bool VideoBackend::ExternalKeyingActive() const
{
	return mVideoIODevice->ExternalKeyingActive();
}

const std::string& VideoBackend::StatusMessage() const
{
	return mVideoIODevice->StatusMessage();
}

void VideoBackend::SetStatusMessage(const std::string& message)
{
	mVideoIODevice->SetStatusMessage(message);
}

void VideoBackend::PublishStatus(bool externalKeyingConfigured, const std::string& statusMessage)
{
	if (!statusMessage.empty())
		SetStatusMessage(statusMessage);

	mHealthTelemetry.ReportVideoIOStatus(
		"decklink",
		OutputModelName(),
		SupportsInternalKeying(),
		SupportsExternalKeying(),
		KeyerInterfaceAvailable(),
		externalKeyingConfigured,
		ExternalKeyingActive(),
		StatusMessage());
	PublishBackendStateChanged(VideoBackendLifecycle::StateName(mLifecycle.State()), StatusMessage());
}

void VideoBackend::ReportNoInputDeviceSignalStatus()
{
	mHealthTelemetry.ReportSignalStatus(
		false,
		InputFrameWidth(),
		InputFrameHeight(),
		InputDisplayModeName());
	PublishBackendStateChanged("no-input-device", "No input device is available.");
}

void VideoBackend::HandleInputFrame(const VideoIOFrame& frame)
{
	const VideoIOState& state = mVideoIODevice->State();
	mHealthTelemetry.TryReportSignalStatus(!frame.hasNoInputSource, state.inputFrameSize.width, state.inputFrameSize.height, state.inputDisplayModeName);
	PublishInputSignalChanged(frame, state);
	PublishInputFrameArrived(frame);

	if (mBridge)
		mBridge->UploadInputFrame(frame, state);
}

void VideoBackend::HandleOutputFrameCompletion(const VideoIOCompletion& completion)
{
	{
		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
		if (!mOutputCompletionWorkerRunning || mOutputCompletionWorkerStopping)
			return;
		mPendingOutputCompletions.push_back(completion);
	}
	mOutputCompletionCondition.notify_one();
}

void VideoBackend::StartOutputCompletionWorker()
{
	std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
	if (mOutputCompletionWorkerRunning)
		return;

	mPendingOutputCompletions.clear();
	mReadyOutputQueue.Clear();
	mNextReadyOutputFrameIndex = 0;
	mOutputCompletionWorkerStopping = false;
	mOutputCompletionWorkerRunning = true;
	mOutputCompletionWorker = std::thread(&VideoBackend::OutputCompletionWorkerMain, this);
}

void VideoBackend::StopOutputCompletionWorker()
{
	bool shouldJoin = false;
	{
		std::lock_guard<std::mutex> lock(mOutputCompletionMutex);
		if (mOutputCompletionWorkerRunning)
			mOutputCompletionWorkerStopping = true;
		shouldJoin = mOutputCompletionWorker.joinable();
	}
	mOutputCompletionCondition.notify_one();

	if (shouldJoin)
		mOutputCompletionWorker.join();
}

void VideoBackend::OutputCompletionWorkerMain()
{
	for (;;)
	{
		VideoIOCompletion completion;
		{
			std::unique_lock<std::mutex> lock(mOutputCompletionMutex);
			mOutputCompletionCondition.wait(lock, [this]() {
				return mOutputCompletionWorkerStopping || !mPendingOutputCompletions.empty();
			});

			if (mPendingOutputCompletions.empty())
			{
				if (mOutputCompletionWorkerStopping)
				{
					mOutputCompletionWorkerRunning = false;
					return;
				}
				continue;
			}

			completion = mPendingOutputCompletions.front();
			mPendingOutputCompletions.pop_front();
		}

		ProcessOutputFrameCompletion(completion);
	}
}

void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion)
{
	RecordFramePacing(completion.result);
	PublishOutputFrameCompleted(completion);
	AccountForCompletionResult(completion.result, mReadyOutputQueue.GetMetrics().depth);

	FillReadyOutputQueue(completion);
	if (!ScheduleReadyOutputFrame())
		ScheduleBlackUnderrunFrame();
}

bool VideoBackend::FillReadyOutputQueue(const VideoIOCompletion& completion)
{
	RenderOutputQueueMetrics metrics = mReadyOutputQueue.GetMetrics();
	bool filledAny = false;
	while (metrics.depth < mPlayoutPolicy.targetReadyFrames)
	{
		if (!RenderReadyOutputFrame(mVideoIODevice->State(), completion))
			return filledAny;
		filledAny = true;
		metrics = mReadyOutputQueue.GetMetrics();
	}
	return true;
}

bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion)
{
	VideoIOOutputFrame outputFrame;
	if (!BeginOutputFrame(outputFrame))
		return false;

	bool rendered = true;
	if (mBridge)
		rendered = mBridge->RenderScheduledFrame(state, completion, outputFrame);

	EndOutputFrame(outputFrame);
	if (!rendered)
	{
		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame.");
		return false;
	}

	RenderOutputFrame readyFrame;
	readyFrame.frame = outputFrame;
	readyFrame.frameIndex = ++mNextReadyOutputFrameIndex;
	return mReadyOutputQueue.Push(readyFrame);
}

bool VideoBackend::ScheduleReadyOutputFrame()
{
	RenderOutputFrame readyFrame;
	if (!mReadyOutputQueue.TryPop(readyFrame))
		return false;

	if (!ScheduleOutputFrame(readyFrame.frame))
		return false;

	PublishOutputFrameScheduled(readyFrame.frame);
	return true;
}

bool VideoBackend::ScheduleBlackUnderrunFrame()
{
	VideoIOOutputFrame outputFrame;
	if (!BeginOutputFrame(outputFrame))
	{
		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: no output frame was available for fallback scheduling.");
		return false;
	}

	if (outputFrame.bytes != nullptr && outputFrame.rowBytes > 0 && outputFrame.height > 0)
		std::memset(outputFrame.bytes, 0, static_cast<std::size_t>(outputFrame.rowBytes) * outputFrame.height);
	EndOutputFrame(outputFrame);

	if (!ScheduleOutputFrame(outputFrame))
	{
		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: black fallback frame scheduling failed.");
		return false;
	}

	ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: scheduled black fallback frame.");
	PublishOutputFrameScheduled(outputFrame);
	return true;
}

void VideoBackend::RecordFramePacing(VideoIOCompletionResult completionResult)
{
	const auto now = std::chrono::steady_clock::now();
	if (mLastPlayoutCompletionTime != std::chrono::steady_clock::time_point())
	{
		mCompletionIntervalMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(now - mLastPlayoutCompletionTime).count();
		if (mSmoothedCompletionIntervalMilliseconds <= 0.0)
			mSmoothedCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds;
		else
			mSmoothedCompletionIntervalMilliseconds = mSmoothedCompletionIntervalMilliseconds * 0.9 + mCompletionIntervalMilliseconds * 0.1;
		if (mCompletionIntervalMilliseconds > mMaxCompletionIntervalMilliseconds)
			mMaxCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds;
	}
	mLastPlayoutCompletionTime = now;

	if (completionResult == VideoIOCompletionResult::DisplayedLate)
		++mLateFrameCount;
	else if (completionResult == VideoIOCompletionResult::Dropped)
		++mDroppedFrameCount;
	else if (completionResult == VideoIOCompletionResult::Flushed)
		++mFlushedFrameCount;

	mHealthTelemetry.TryRecordFramePacingStats(
		mCompletionIntervalMilliseconds,
		mSmoothedCompletionIntervalMilliseconds,
		mMaxCompletionIntervalMilliseconds,
		mLateFrameCount,
		mDroppedFrameCount,
		mFlushedFrameCount);
	PublishTimingSample("VideoBackend", "completionInterval", mCompletionIntervalMilliseconds, "ms");
	PublishTimingSample("VideoBackend", "smoothedCompletionInterval", mSmoothedCompletionIntervalMilliseconds, "ms");
}

bool VideoBackend::ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message)
{
	const VideoBackendLifecycleTransition transition = mLifecycle.TransitionTo(state, message);
	if (!transition.accepted)
	{
		PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), transition.errorMessage);
		return false;
	}

	PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), message);
	return true;
}

bool VideoBackend::ApplyLifecycleFailure(const std::string& message)
{
	const VideoBackendLifecycleTransition transition = mLifecycle.Fail(message);
	if (!transition.accepted)
	{
		PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), transition.errorMessage);
		return false;
	}

	PublishBackendStateChanged(VideoBackendLifecycle::StateName(transition.current), message);
	return true;
}

void VideoBackend::PublishBackendStateChanged(const std::string& state, const std::string& message)
{
	try
	{
		BackendStateChangedEvent event;
		event.backendName = "decklink";
		event.state = state;
		event.message = message;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("BackendStateChanged event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("BackendStateChanged event publish threw.\n");
	}
}

void VideoBackend::PublishInputSignalChanged(const VideoIOFrame& frame, const VideoIOState& state)
{
	const bool hasSignal = !frame.hasNoInputSource;
	const unsigned width = state.inputFrameSize.width;
	const unsigned height = state.inputFrameSize.height;
	if (mHasLastInputSignal &&
		mLastInputSignal == hasSignal &&
		mLastInputSignalWidth == width &&
		mLastInputSignalHeight == height &&
		mLastInputSignalModeName == state.inputDisplayModeName)
	{
		return;
	}

	mHasLastInputSignal = true;
	mLastInputSignal = hasSignal;
	mLastInputSignalWidth = width;
	mLastInputSignalHeight = height;
	mLastInputSignalModeName = state.inputDisplayModeName;

	try
	{
		InputSignalChangedEvent event;
		event.hasSignal = hasSignal;
		event.width = width;
		event.height = height;
		event.modeName = state.inputDisplayModeName;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("InputSignalChanged event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("InputSignalChanged event publish threw.\n");
	}
}

void VideoBackend::PublishInputFrameArrived(const VideoIOFrame& frame)
{
	try
	{
		InputFrameArrivedEvent event;
		event.frameIndex = ++mInputFrameIndex;
		event.width = frame.width;
		event.height = frame.height;
		event.rowBytes = frame.rowBytes;
		event.pixelFormat = PixelFormatName(frame.pixelFormat);
		event.hasNoInputSource = frame.hasNoInputSource;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("InputFrameArrived event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("InputFrameArrived event publish threw.\n");
	}
}

void VideoBackend::PublishOutputFrameScheduled(const VideoIOOutputFrame& frame)
{
	try
	{
		OutputFrameScheduledEvent event;
		event.frameIndex = ++mOutputFrameScheduleIndex;
		(void)frame;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("OutputFrameScheduled event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("OutputFrameScheduled event publish threw.\n");
	}
}

void VideoBackend::PublishOutputFrameCompleted(const VideoIOCompletion& completion)
{
	try
	{
		OutputFrameCompletedEvent event;
		event.frameIndex = ++mOutputFrameCompletionIndex;
		event.result = CompletionResultName(completion.result);
		if (!mRuntimeEventDispatcher.PublishPayload(event, "VideoBackend"))
			OutputDebugStringA("OutputFrameCompleted event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("OutputFrameCompleted event publish threw.\n");
	}
}

void VideoBackend::PublishTimingSample(const std::string& subsystem, const std::string& metric, double value, const std::string& unit)
{
	try
	{
		TimingSampleRecordedEvent event;
		event.subsystem = subsystem;
		event.metric = metric;
		event.value = value;
		event.unit = unit;
		if (!mRuntimeEventDispatcher.PublishPayload(event, "HealthTelemetry"))
			OutputDebugStringA("TimingSampleRecorded event publish failed.\n");
	}
	catch (...)
	{
		OutputDebugStringA("TimingSampleRecorded event publish threw.\n");
	}
}

std::string VideoBackend::CompletionResultName(VideoIOCompletionResult result)
{
	switch (result)
	{
	case VideoIOCompletionResult::Completed:
		return "Completed";
	case VideoIOCompletionResult::DisplayedLate:
		return "DisplayedLate";
	case VideoIOCompletionResult::Dropped:
		return "Dropped";
	case VideoIOCompletionResult::Flushed:
		return "Flushed";
	case VideoIOCompletionResult::Unknown:
	default:
		return "Unknown";
	}
}

std::string VideoBackend::PixelFormatName(VideoIOPixelFormat pixelFormat)
{
	return std::string(VideoIOPixelFormatName(pixelFormat));
}