video-shader-toys/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp

#include "DeckLinkDisplayMode.h"
#include "DeckLinkSession.h"
#include "OpenGLComposite.h"
#include "GLExtensions.h"
#include "GlRenderConstants.h"
#include "OpenGLRenderPass.h"
#include "OpenGLRenderPipeline.h"
#include "OpenGLShaderPrograms.h"
#include "OpenGLVideoIOBridge.h"
#include "PngScreenshotWriter.h"
#include "RuntimeServices.h"
#include "ShaderBuildQueue.h"

#include <algorithm>
#include <chrono>
#include <ctime>
#include <filesystem>
#include <iomanip>
#include <memory>
#include <sstream>
#include <string>
#include <vector>

OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) :
	hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC),
	mVideoIO(std::make_unique<DeckLinkSession>()),
	mRenderer(std::make_unique<OpenGLRenderer>()),
	mUseCommittedLayerStates(false),
	mScreenshotRequested(false)
{
	InitializeCriticalSection(&pMutex);
	mRuntimeHost = std::make_unique<RuntimeHost>();
	mRenderPipeline = std::make_unique<OpenGLRenderPipeline>(
		*mRenderer,
		*mRuntimeHost,
		[this]() { renderEffect(); },
		[this]() { ProcessScreenshotRequest(); },
		[this]() { paintGL(); });
	mVideoIOBridge = std::make_unique<OpenGLVideoIOBridge>(
		*mVideoIO,
		*mRenderer,
		*mRenderPipeline,
		*mRuntimeHost,
		pMutex,
		hGLDC,
		hGLRC);
	mRenderPass = std::make_unique<OpenGLRenderPass>(*mRenderer);
	mShaderPrograms = std::make_unique<OpenGLShaderPrograms>(*mRenderer, *mRuntimeHost);
	mShaderBuildQueue = std::make_unique<ShaderBuildQueue>(*mRuntimeHost);
	mRuntimeServices = std::make_unique<RuntimeServices>();
}

OpenGLComposite::~OpenGLComposite()
{
	if (mRuntimeServices)
		mRuntimeServices->Stop();
	if (mShaderBuildQueue)
		mShaderBuildQueue->Stop();
	mVideoIO->ReleaseResources();
	mRenderer->DestroyResources();

	DeleteCriticalSection(&pMutex);
}

bool OpenGLComposite::InitDeckLink()
{
	return InitVideoIO();
}

bool OpenGLComposite::InitVideoIO()
{
	VideoFormatSelection		videoModes;
	std::string						initFailureReason;

	if (mRuntimeHost && mRuntimeHost->GetRepoRoot().empty())
	{
		std::string runtimeError;
		if (!mRuntimeHost->Initialize(runtimeError))
		{
			MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK);
			return false;
		}
	}

	if (mRuntimeHost)
	{
		if (!ResolveConfiguredVideoFormats(
			mRuntimeHost->GetInputVideoFormat(),
			mRuntimeHost->GetInputFrameRate(),
			mRuntimeHost->GetOutputVideoFormat(),
			mRuntimeHost->GetOutputFrameRate(),
			videoModes,
			initFailureReason))
		{
			MessageBoxA(NULL, initFailureReason.c_str(), "DeckLink mode configuration error", MB_OK);
			return false;
		}
	}

	if (!mVideoIO->DiscoverDevicesAndModes(videoModes, initFailureReason))
	{
		const char* title = initFailureReason == "Please install the Blackmagic DeckLink drivers to use the features of this application."
			? "This application requires the DeckLink drivers installed."
			: "DeckLink initialization failed";
		MessageBoxA(NULL, initFailureReason.c_str(), title, MB_OK | MB_ICONERROR);
		return false;
	}
	const bool outputAlphaRequired = mRuntimeHost && mRuntimeHost->ExternalKeyingEnabled();
	if (!mVideoIO->SelectPreferredFormats(videoModes, outputAlphaRequired, initFailureReason))
		goto error;

	if (! CheckOpenGLExtensions())
	{
		initFailureReason = "OpenGL extension checks failed.";
		goto error;
	}

	if (! InitOpenGLState())
	{
		initFailureReason = "OpenGL state initialization failed.";
		goto error;
	}

	PublishVideoIOStatus(mVideoIO->OutputModelName().empty()
		? "DeckLink output device selected."
		: ("Selected output device: " + mVideoIO->OutputModelName()));

	// Resize window to match output video frame, but scale large formats down by half for viewing.
	if (mVideoIO->OutputFrameWidth() < 1920)
		resizeWindow(mVideoIO->OutputFrameWidth(), mVideoIO->OutputFrameHeight());
	else
		resizeWindow(mVideoIO->OutputFrameWidth() / 2, mVideoIO->OutputFrameHeight() / 2);

	if (!mVideoIO->ConfigureInput([this](const VideoIOFrame& frame) { mVideoIOBridge->VideoFrameArrived(frame); }, videoModes.input, initFailureReason))
	{
		goto error;
	}
	if (!mVideoIO->HasInputDevice() && mRuntimeHost)
	{
		mRuntimeHost->SetSignalStatus(false, mVideoIO->InputFrameWidth(), mVideoIO->InputFrameHeight(), mVideoIO->InputDisplayModeName());
	}

	if (!mVideoIO->ConfigureOutput([this](const VideoIOCompletion& completion) { mVideoIOBridge->PlayoutFrameCompleted(completion); }, videoModes.output, mRuntimeHost && mRuntimeHost->ExternalKeyingEnabled(), initFailureReason))
	{
		goto error;
	}

	PublishVideoIOStatus(mVideoIO->StatusMessage());

	return true;

error:
	if (!initFailureReason.empty())
		MessageBoxA(NULL, initFailureReason.c_str(), "DeckLink initialization failed", MB_OK | MB_ICONERROR);
	mVideoIO->ReleaseResources();
	return false;
}

void OpenGLComposite::paintGL()
{
	if (!TryEnterCriticalSection(&pMutex))
	{
		ValidateRect(hGLWnd, NULL);
		return;
	}

	mRenderer->PresentToWindow(hGLDC, mVideoIO->OutputFrameWidth(), mVideoIO->OutputFrameHeight());
	ValidateRect(hGLWnd, NULL);
	LeaveCriticalSection(&pMutex);
}

void OpenGLComposite::resizeGL(WORD width, WORD height)
{
	// We don't set the project or model matrices here since the window data is copied directly from
	// an off-screen FBO in paintGL().  Just save the width and height for use in paintGL().
	mRenderer->ResizeView(width, height);
}

void OpenGLComposite::resizeWindow(int width, int height)
{
	RECT r;
	if (GetWindowRect(hGLWnd, &r))
	{
		SetWindowPos(hGLWnd, HWND_TOP, r.left, r.top, r.left + width, r.top + height, 0);
	}
}

void OpenGLComposite::PublishVideoIOStatus(const std::string& statusMessage)
{
	if (!mRuntimeHost)
		return;

	if (!statusMessage.empty())
		mVideoIO->SetStatusMessage(statusMessage);

	mRuntimeHost->SetVideoIOStatus(
		"decklink",
		mVideoIO->OutputModelName(),
		mVideoIO->SupportsInternalKeying(),
		mVideoIO->SupportsExternalKeying(),
		mVideoIO->KeyerInterfaceAvailable(),
		mRuntimeHost->ExternalKeyingEnabled(),
		mVideoIO->ExternalKeyingActive(),
		mVideoIO->StatusMessage());
}

bool OpenGLComposite::InitOpenGLState()
{
	if (! ResolveGLExtensions())
		return false;

	std::string runtimeError;
	if (mRuntimeHost->GetRepoRoot().empty() && !mRuntimeHost->Initialize(runtimeError))
	{
		MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK);
		return false;
	}

	if (!mRuntimeServices->Start(*this, *mRuntimeHost, runtimeError))
	{
		MessageBoxA(NULL, runtimeError.c_str(), "Runtime control services failed to start", MB_OK);
		return false;
	}

	// Prepare the runtime shader program generated from the active shader package.
	char compilerErrorMessage[1024];
	if (!mShaderPrograms->CompileDecodeShader(sizeof(compilerErrorMessage), compilerErrorMessage))
	{
		MessageBoxA(NULL, compilerErrorMessage, "OpenGL decode shader failed to load or compile", MB_OK);
		return false;
	}
	if (!mShaderPrograms->CompileOutputPackShader(sizeof(compilerErrorMessage), compilerErrorMessage))
	{
		MessageBoxA(NULL, compilerErrorMessage, "OpenGL output pack shader failed to load or compile", MB_OK);
		return false;
	}

	if (!mShaderPrograms->CompileLayerPrograms(mVideoIO->InputFrameWidth(), mVideoIO->InputFrameHeight(), sizeof(compilerErrorMessage), compilerErrorMessage))
	{
		MessageBoxA(NULL, compilerErrorMessage, "OpenGL shader failed to load or compile", MB_OK);
		return false;
	}
	mCachedLayerRenderStates = mShaderPrograms->CommittedLayerStates();
	mUseCommittedLayerStates = false;
	mShaderPrograms->ResetTemporalHistoryState();

	std::string rendererError;
	if (!mRenderer->InitializeResources(
		mVideoIO->InputFrameWidth(),
		mVideoIO->InputFrameHeight(),
		mVideoIO->CaptureTextureWidth(),
		mVideoIO->OutputFrameWidth(),
		mVideoIO->OutputFrameHeight(),
		mVideoIO->OutputPackTextureWidth(),
		rendererError))
	{
		MessageBoxA(NULL, rendererError.c_str(), "OpenGL initialization error.", MB_OK);
		return false;
	}

	broadcastRuntimeState();
	mRuntimeServices->BeginPolling(*mRuntimeHost);
	return true;
}

bool OpenGLComposite::Start()
{
	return mVideoIO->Start();
}

bool OpenGLComposite::Stop()
{
	if (mRuntimeServices)
		mRuntimeServices->Stop();

	const bool wasExternalKeyingActive = mVideoIO->ExternalKeyingActive();
	mVideoIO->Stop();
	if (wasExternalKeyingActive)
		PublishVideoIOStatus("External keying has been disabled.");

	return true;
}

bool OpenGLComposite::ReloadShader()
{
	if (mRuntimeHost)
	{
		mRuntimeHost->SetCompileStatus(true, "Shader rebuild queued.");
		mRuntimeHost->ClearReloadRequest();
	}
	RequestShaderBuild();
	broadcastRuntimeState();
	return true;
}

bool OpenGLComposite::RequestScreenshot(std::string& error)
{
	(void)error;
	mScreenshotRequested.store(true);
	return true;
}

void OpenGLComposite::renderEffect()
{
	ProcessRuntimePollResults();

	const bool hasInputSource = mVideoIO->HasInputSource();
	std::vector<RuntimeRenderState> layerStates;
	if (mUseCommittedLayerStates)
	{
		layerStates = mShaderPrograms->CommittedLayerStates();
		if (mRuntimeHost)
			mRuntimeHost->RefreshDynamicRenderStateFields(layerStates);
	}
	else if (mRuntimeHost)
	{
		const unsigned renderWidth = mVideoIO->InputFrameWidth();
		const unsigned renderHeight = mVideoIO->InputFrameHeight();
		const uint64_t renderStateVersion = mRuntimeHost->GetRenderStateVersion();
		const bool renderStateCacheValid =
			!mCachedLayerRenderStates.empty() &&
			mCachedRenderStateVersion == renderStateVersion &&
			mCachedRenderStateWidth == renderWidth &&
			mCachedRenderStateHeight == renderHeight;

		if (renderStateCacheValid)
		{
			layerStates = mCachedLayerRenderStates;
			mRuntimeHost->RefreshDynamicRenderStateFields(layerStates);
		}
		else
		{
			if (mRuntimeHost->TryGetLayerRenderStates(renderWidth, renderHeight, layerStates))
			{
				mCachedLayerRenderStates = layerStates;
				mCachedRenderStateVersion = renderStateVersion;
				mCachedRenderStateWidth = renderWidth;
				mCachedRenderStateHeight = renderHeight;
			}
			else
			{
				layerStates = mCachedLayerRenderStates;
				mRuntimeHost->RefreshDynamicRenderStateFields(layerStates);
			}
		}
	}
	const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0;
	mRenderPass->Render(
		hasInputSource,
		layerStates,
		mVideoIO->InputFrameWidth(),
		mVideoIO->InputFrameHeight(),
		mVideoIO->CaptureTextureWidth(),
		mVideoIO->InputPixelFormat(),
		historyCap,
		[this](const RuntimeRenderState& state, LayerProgram::TextBinding& textBinding, std::string& error) {
			return mShaderPrograms->UpdateTextBindingTexture(state, textBinding, error);
		},
		[this](const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength) {
			return mShaderPrograms->UpdateGlobalParamsBuffer(state, availableSourceHistoryLength, availableTemporalHistoryLength);
		});
}

void OpenGLComposite::ProcessScreenshotRequest()
{
	if (!mScreenshotRequested.exchange(false))
		return;

	const unsigned width = mVideoIO ? mVideoIO->OutputFrameWidth() : 0;
	const unsigned height = mVideoIO ? mVideoIO->OutputFrameHeight() : 0;
	if (width == 0 || height == 0)
		return;

	std::vector<unsigned char> bottomUpPixels(static_cast<std::size_t>(width) * height * 4);
	std::vector<unsigned char> topDownPixels(bottomUpPixels.size());

	glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->OutputFramebuffer());
	glReadBuffer(GL_COLOR_ATTACHMENT0);
	glPixelStorei(GL_PACK_ALIGNMENT, 1);
	glPixelStorei(GL_PACK_ROW_LENGTH, 0);
	glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, bottomUpPixels.data());
	glPixelStorei(GL_PACK_ALIGNMENT, 4);

	const std::size_t rowBytes = static_cast<std::size_t>(width) * 4;
	for (unsigned y = 0; y < height; ++y)
	{
		const unsigned sourceY = height - 1 - y;
		std::copy(
			bottomUpPixels.begin() + static_cast<std::ptrdiff_t>(sourceY * rowBytes),
			bottomUpPixels.begin() + static_cast<std::ptrdiff_t>((sourceY + 1) * rowBytes),
			topDownPixels.begin() + static_cast<std::ptrdiff_t>(y * rowBytes));
	}

	try
	{
		const std::filesystem::path outputPath = BuildScreenshotPath();
		std::filesystem::create_directories(outputPath.parent_path());
		WritePngFileAsync(outputPath, width, height, std::move(topDownPixels));
	}
	catch (const std::exception& exception)
	{
		OutputDebugStringA((std::string("Screenshot request failed: ") + exception.what() + "\n").c_str());
	}
}

std::filesystem::path OpenGLComposite::BuildScreenshotPath() const
{
	const std::filesystem::path root = mRuntimeHost && !mRuntimeHost->GetRuntimeRoot().empty()
		? mRuntimeHost->GetRuntimeRoot()
		: std::filesystem::current_path();

	const auto now = std::chrono::system_clock::now();
	const auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()) % 1000;
	const std::time_t nowTime = std::chrono::system_clock::to_time_t(now);
	std::tm localTime = {};
	localtime_s(&localTime, &nowTime);

	std::ostringstream filename;
	filename << "video-shader-toys-"
		<< std::put_time(&localTime, "%Y%m%d-%H%M%S")
		<< "-" << std::setw(3) << std::setfill('0') << milliseconds.count()
		<< ".png";

	return root / "screenshots" / filename.str();
}

bool OpenGLComposite::ProcessRuntimePollResults()
{
	if (!mRuntimeHost || !mRuntimeServices)
		return true;

	const RuntimePollEvents events = mRuntimeServices->ConsumePollEvents();
	if (events.failed)
	{
		mRuntimeHost->SetCompileStatus(false, events.error);
		broadcastRuntimeState();
		return false;
	}

	if (events.registryChanged)
		broadcastRuntimeState();

	if (!events.reloadRequested)
	{
		PreparedShaderBuild readyBuild;
		if (!mShaderBuildQueue || !mShaderBuildQueue->TryConsumeReadyBuild(readyBuild))
			return true;

		char compilerErrorMessage[1024] = {};
		if (!mShaderPrograms->CommitPreparedLayerPrograms(readyBuild, mVideoIO->InputFrameWidth(), mVideoIO->InputFrameHeight(), sizeof(compilerErrorMessage), compilerErrorMessage))
		{
			mRuntimeHost->SetCompileStatus(false, compilerErrorMessage);
			mUseCommittedLayerStates = true;
			broadcastRuntimeState();
			return false;
		}

		mUseCommittedLayerStates = false;
		mCachedLayerRenderStates = mShaderPrograms->CommittedLayerStates();
		mShaderPrograms->ResetTemporalHistoryState();
		broadcastRuntimeState();
		return true;
	}

	mRuntimeHost->SetCompileStatus(true, "Shader rebuild queued.");
	RequestShaderBuild();
	broadcastRuntimeState();
	return true;
}

void OpenGLComposite::RequestShaderBuild()
{
	if (!mShaderBuildQueue || !mVideoIO)
		return;

	mUseCommittedLayerStates = true;
	if (mRuntimeHost)
		mRuntimeHost->ClearReloadRequest();
	mShaderBuildQueue->RequestBuild(mVideoIO->InputFrameWidth(), mVideoIO->InputFrameHeight());
}

void OpenGLComposite::broadcastRuntimeState()
{
	if (mRuntimeServices)
		mRuntimeServices->BroadcastState();
}

void OpenGLComposite::resetTemporalHistoryState()
{
	mShaderPrograms->ResetTemporalHistoryState();
}

bool OpenGLComposite::CheckOpenGLExtensions()
{
	return true;
}

////////////////////////////////////////////