video-shader-toys/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h

#pragma once

#include "OpenGLRenderPass.h"
#include "OpenGLRenderPipeline.h"
#include "OpenGLRenderer.h"
#include "OpenGLShaderPrograms.h"
#include "RenderCommandQueue.h"
#include "RenderFrameState.h"
#include "RenderFrameStateResolver.h"
#include "HealthTelemetry.h"
#include "RuntimeCoordinator.h"
#include "RuntimeSnapshotProvider.h"

#include <windows.h>

#include <atomic>
#include <cstdint>
#include <chrono>
#include <condition_variable>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <utility>
#include <vector>

class RenderEngine
{
public:
	using RenderEffectCallback = std::function<void()>;
	using ScreenshotCallback = std::function<void()>;
	using ScreenshotCaptureCallback = std::function<void(unsigned, unsigned, std::vector<unsigned char>)>;
	using PreviewPaintCallback = std::function<void()>;

	struct OscOverlayUpdate
	{
		std::string routeKey;
		std::string layerKey;
		std::string parameterKey;
		JsonValue targetValue;
	};

	struct OscOverlayCommitCompletion
	{
		std::string routeKey;
		uint64_t generation = 0;
	};

	struct OscOverlayCommitRequest
	{
		std::string routeKey;
		std::string layerKey;
		std::string parameterKey;
		JsonValue value;
		uint64_t generation = 0;
	};

	RenderEngine(
		RuntimeSnapshotProvider& runtimeSnapshotProvider,
		HealthTelemetry& healthTelemetry,
		HDC hdc,
		HGLRC hglrc,
		RenderEffectCallback renderEffect,
		ScreenshotCallback screenshotReady,
		PreviewPaintCallback previewPaint);
	~RenderEngine();

	bool StartRenderThread();
	void StopRenderThread();

	bool CompileDecodeShader(int errorMessageSize, char* errorMessage);
	bool CompileOutputPackShader(int errorMessageSize, char* errorMessage);
	bool InitializeResources(
		unsigned inputFrameWidth,
		unsigned inputFrameHeight,
		unsigned captureTextureWidth,
		unsigned outputFrameWidth,
		unsigned outputFrameHeight,
		unsigned outputPackTextureWidth,
		std::string& error);
	bool CompileLayerPrograms(unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage);
	bool ApplyPreparedShaderBuild(
		const PreparedShaderBuild& preparedBuild,
		unsigned inputFrameWidth,
		unsigned inputFrameHeight,
		bool preserveFeedbackState,
		int errorMessageSize,
		char* errorMessage);

	void ResetTemporalHistoryState();
	void ResetShaderFeedbackState();
	void ApplyRuntimeCoordinatorRenderReset(RuntimeCoordinatorRenderResetScope resetScope);
	void ClearOscOverlayState();
	void ClearOscOverlayStateForLayerKey(const std::string& layerKey);
	void UpdateOscOverlayState(
		const std::vector<OscOverlayUpdate>& updates,
		const std::vector<OscOverlayCommitCompletion>& completedCommits);
	void ResizeView(int width, int height);
	bool TryPresentPreview(bool force, unsigned previewFps, unsigned outputFrameWidth, unsigned outputFrameHeight);
	bool RequestScreenshotCapture(unsigned width, unsigned height, ScreenshotCaptureCallback completion);
	bool QueueInputFrame(const VideoIOFrame& inputFrame, const VideoIOState& videoState);
	bool RequestOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame);
	bool ResolveRenderFrameState(
		const RenderFrameInput& input,
		std::vector<OscOverlayCommitRequest>* commitRequests,
		RenderFrameState& frameState);
	void RenderPreparedFrame(const RenderFrameState& frameState);

private:
	static constexpr std::chrono::milliseconds kRenderThreadRequestTimeout{ 250 };

	struct RenderThreadTaskState
	{
		std::atomic<bool> started = false;
		std::atomic<bool> cancelled = false;
	};

	template<typename Func>
	auto InvokeOnRenderThread(Func&& func) -> decltype(func())
	{
		using Result = decltype(func());
		if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId)
			return func();

		auto task = std::make_shared<std::packaged_task<Result()>>(std::forward<Func>(func));
		std::future<Result> result = task->get_future();
		{
			std::lock_guard<std::mutex> lock(mRenderThreadMutex);
			mRenderThreadTasks.push([task]() { (*task)(); });
		}
		mRenderThreadCondition.notify_one();
		return result.get();
	}

	template<typename Func>
	bool TryInvokeOnRenderThread(const char* operationName, Func&& func)
	{
		if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId)
			return func();

		auto state = std::make_shared<RenderThreadTaskState>();
		auto task = std::make_shared<std::packaged_task<bool()>>(
			[state, func = std::forward<Func>(func)]() mutable {
				state->started = true;
				if (state->cancelled)
					return false;

				return func();
			});
		std::future<bool> result = task->get_future();
		{
			std::lock_guard<std::mutex> lock(mRenderThreadMutex);
			if (mRenderThreadStopping)
			{
				ReportRenderThreadRequestFailure(operationName, "render thread is stopping");
				return false;
			}
			mRenderThreadTasks.push([task]() { (*task)(); });
		}
		mRenderThreadCondition.notify_one();

		if (result.wait_for(kRenderThreadRequestTimeout) == std::future_status::ready)
			return result.get();

		if (!state->started)
		{
			state->cancelled = true;
			ReportRenderThreadRequestFailure(operationName, "timed out before execution");
			return false;
		}

		ReportRenderThreadRequestFailure(operationName, "exceeded timeout while executing; waiting for safe completion");
		return result.get();
	}

	void RenderThreadMain(std::promise<bool> ready);
	void ReportRenderThreadRequestFailure(const char* operationName, const char* reason);
	bool IsRenderThreadAccessExpected() const;
	void ReportWrongThreadRenderAccess(const char* operationName) const;
	bool CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage);
	void RenderLayerStack(
		bool hasInputSource,
		const std::vector<RuntimeRenderState>& layerStates,
		unsigned inputFrameWidth,
		unsigned inputFrameHeight,
		unsigned captureTextureWidth,
		VideoIOPixelFormat inputPixelFormat,
		unsigned historyCap);
	void ResetTemporalHistoryStateOnRenderThread();
	void ResetShaderFeedbackStateOnRenderThread();
	void ApplyRenderResetOnRenderThread(RenderCommandResetScope resetScope);
	void ProcessRenderResetCommandsOnRenderThread();
	void EnqueuePreviewPresentWake();
	void ProcessPreviewPresentCommandsOnRenderThread();
	void EnqueueInputUploadWake();
	void ProcessInputUploadCommandsOnRenderThread();
	void EnqueueScreenshotCaptureWake();
	void ProcessScreenshotCaptureCommandsOnRenderThread();
	bool PresentPreviewOnRenderThread(unsigned outputFrameWidth, unsigned outputFrameHeight);
	bool UploadInputFrameOnRenderThread(const VideoIOFrame& inputFrame, const VideoIOState& videoState);
	bool RenderOutputFrameOnRenderThread(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame);
	bool ReadOutputFrameRgbaOnRenderThread(unsigned width, unsigned height, std::vector<unsigned char>& bottomUpPixels);
	bool CaptureOutputFrameRgbaTopDownOnRenderThread(unsigned width, unsigned height, std::vector<unsigned char>& topDownPixels);

	OpenGLRenderer mRenderer;
	OpenGLRenderPass mRenderPass;
	OpenGLRenderPipeline mRenderPipeline;
	OpenGLShaderPrograms mShaderPrograms;
	HealthTelemetry& mHealthTelemetry;
	HDC mHdc;
	HGLRC mHglrc;

	std::chrono::steady_clock::time_point mLastPreviewPresentTime;
	RenderCommandQueue mRenderCommandQueue;
	RenderFrameStateResolver mFrameStateResolver;
	RuntimeLiveState mRuntimeLiveState;
	std::thread mRenderThread;
	std::atomic<DWORD> mRenderThreadId = 0;
	std::mutex mRenderThreadMutex;
	std::condition_variable mRenderThreadCondition;
	std::queue<std::function<void()>> mRenderThreadTasks;
	std::atomic<bool> mRenderThreadRunning = false;
	bool mRenderThreadStopping = false;
	bool mPreviewPresentWakePending = false;
	bool mInputUploadWakePending = false;
	bool mScreenshotCaptureWakePending = false;
	ScreenshotCaptureCallback mScreenshotCaptureCompletion;
	bool mResourcesDestroyed = false;
};