Step 3

CMakeLists.txt

@@ -378,6 +378,8 @@ add_executable(RenderCommandQueueTests
 target_include_directories(RenderCommandQueueTests PRIVATE
 	"${APP_DIR}"
 	"${APP_DIR}/gl"
+	"${APP_DIR}/videoio"
+	"${APP_DIR}/videoio/decklink"
 )
 
 if(MSVC)

apps/LoopThroughWithOpenGLCompositing/LoopThroughWithOpenGLCompositing.cpp

@@ -434,7 +434,7 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
 			}
 
 			// Deselect the current rendering context and delete it
-			wglMakeCurrent(hDC, NULL);
+			wglMakeCurrent(NULL, NULL);
 			wglDeleteContext(hRC);
 
 			// Tell the application to terminate after the window is gone
@@ -486,15 +486,12 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
 
 				if (!sInteractiveResize && pOpenGLComposite)
 				{
-					wglMakeCurrent(hDC, hRC);
 					pOpenGLComposite->paintGL(true);
-					wglMakeCurrent( NULL, NULL );
 					RaiseStatusControls(sStatusStrip);
 				}
 			}
 			catch (...)
 			{
-				wglMakeCurrent( NULL, NULL );
 				ShowUnhandledExceptionMessage("Paint failed inside the OpenGL runtime.");
 			}
 			break;

apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp

@@ -257,7 +257,14 @@ bool OpenGLComposite::InitOpenGLState()
 
 bool OpenGLComposite::Start()
 {
-	return mVideoBackend->Start();
+	if (!mRenderEngine->StartRenderThread())
+		return false;
+
+	if (mVideoBackend->Start())
+		return true;
+
+	mRenderEngine->StopRenderThread();
+	return false;
 }
 
 bool OpenGLComposite::Stop()
@@ -272,6 +279,9 @@ bool OpenGLComposite::Stop()
 			mRuntimeStore && mRuntimeStore->IsExternalKeyingConfigured(),
 			"External keying has been disabled.");
 
+	if (mRenderEngine)
+		mRenderEngine->StopRenderThread();
+
 	return true;
 }
 

apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.cpp

@@ -48,6 +48,48 @@ bool RenderCommandQueue::TryTakeScreenshotCapture(RenderScreenshotCaptureRequest
 	return true;
 }
 
+void RenderCommandQueue::RequestInputUpload(const RenderInputUploadRequest& request)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (mHasInputUploadRequest)
+		++mCoalescedCount;
+	else
+		++mEnqueuedCount;
+
+	mInputUploadRequest = request;
+	mHasInputUploadRequest = true;
+}
+
+bool RenderCommandQueue::TryTakeInputUpload(RenderInputUploadRequest& request)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (!mHasInputUploadRequest)
+		return false;
+
+	request = mInputUploadRequest;
+	mInputUploadRequest = {};
+	mHasInputUploadRequest = false;
+	return true;
+}
+
+void RenderCommandQueue::RequestOutputFrame(const RenderOutputFrameRequest& request)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mOutputFrameRequests.push_back(request);
+	++mEnqueuedCount;
+}
+
+bool RenderCommandQueue::TryTakeOutputFrame(RenderOutputFrameRequest& request)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (mOutputFrameRequests.empty())
+		return false;
+
+	request = mOutputFrameRequests.front();
+	mOutputFrameRequests.pop_front();
+	return true;
+}
+
 void RenderCommandQueue::RequestRenderReset(RenderCommandResetScope scope)
 {
 	if (scope == RenderCommandResetScope::None)
@@ -80,6 +122,8 @@ RenderCommandQueueMetrics RenderCommandQueue::GetMetrics() const
 	metrics.depth =
 		(mHasPreviewPresentRequest ? 1u : 0u) +
 		(mHasScreenshotCaptureRequest ? 1u : 0u) +
+		(mHasInputUploadRequest ? 1u : 0u) +
+		mOutputFrameRequests.size() +
 		(mRenderResetScope != RenderCommandResetScope::None ? 1u : 0u);
 	metrics.enqueuedCount = mEnqueuedCount;
 	metrics.coalescedCount = mCoalescedCount;

apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h

@@ -1,7 +1,10 @@
 #pragma once
 
+#include "VideoIOTypes.h"
+
 #include <cstddef>
 #include <cstdint>
+#include <deque>
 #include <mutex>
 
 enum class RenderCommandResetScope
@@ -24,6 +27,18 @@ struct RenderScreenshotCaptureRequest
 	unsigned height = 0;
 };
 
+struct RenderInputUploadRequest
+{
+	VideoIOFrame inputFrame;
+	VideoIOState videoState;
+};
+
+struct RenderOutputFrameRequest
+{
+	VideoIOState videoState;
+	VideoIOCompletion completion;
+};
+
 struct RenderCommandQueueMetrics
 {
 	std::size_t depth = 0;
@@ -40,6 +55,12 @@ public:
 	void RequestScreenshotCapture(const RenderScreenshotCaptureRequest& request);
 	bool TryTakeScreenshotCapture(RenderScreenshotCaptureRequest& request);
 
+	void RequestInputUpload(const RenderInputUploadRequest& request);
+	bool TryTakeInputUpload(RenderInputUploadRequest& request);
+
+	void RequestOutputFrame(const RenderOutputFrameRequest& request);
+	bool TryTakeOutputFrame(RenderOutputFrameRequest& request);
+
 	void RequestRenderReset(RenderCommandResetScope scope);
 	bool TryTakeRenderReset(RenderCommandResetScope& scope);
 
@@ -53,6 +74,9 @@ private:
 	RenderPreviewPresentRequest mPreviewPresentRequest;
 	bool mHasScreenshotCaptureRequest = false;
 	RenderScreenshotCaptureRequest mScreenshotCaptureRequest;
+	bool mHasInputUploadRequest = false;
+	RenderInputUploadRequest mInputUploadRequest;
+	std::deque<RenderOutputFrameRequest> mOutputFrameRequests;
 	RenderCommandResetScope mRenderResetScope = RenderCommandResetScope::None;
 	uint64_t mEnqueuedCount = 0;
 	uint64_t mCoalescedCount = 0;

apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp

@@ -26,17 +26,128 @@ RenderEngine::RenderEngine(
 
 RenderEngine::~RenderEngine()
 {
-	mRenderer.DestroyResources();
+	StopRenderThread();
+	if (!mResourcesDestroyed)
+	{
+		mRenderer.DestroyResources();
+		mResourcesDestroyed = true;
+	}
+}
+
+bool RenderEngine::StartRenderThread()
+{
+	if (mRenderThreadRunning)
+		return true;
+
+	{
+		std::lock_guard<std::mutex> lock(mRenderThreadMutex);
+		mRenderThreadStopping = false;
+	}
+
+	std::promise<bool> ready;
+	std::future<bool> readyResult = ready.get_future();
+	mRenderThread = std::thread(&RenderEngine::RenderThreadMain, this, std::move(ready));
+	if (!readyResult.get())
+	{
+		if (mRenderThread.joinable())
+			mRenderThread.join();
+		return false;
+	}
+
+	return true;
+}
+
+void RenderEngine::StopRenderThread()
+{
+	if (mRenderThreadRunning)
+	{
+		InvokeOnRenderThread([this]() {
+			if (!mResourcesDestroyed)
+			{
+				mRenderer.DestroyResources();
+				mResourcesDestroyed = true;
+			}
+		});
+	}
+
+	{
+		std::lock_guard<std::mutex> lock(mRenderThreadMutex);
+		mRenderThreadStopping = true;
+	}
+	mRenderThreadCondition.notify_one();
+
+	if (mRenderThread.joinable())
+		mRenderThread.join();
+}
+
+void RenderEngine::RenderThreadMain(std::promise<bool> ready)
+{
+	mRenderThreadId = GetCurrentThreadId();
+	if (!wglMakeCurrent(mHdc, mHglrc))
+	{
+		mRenderThreadId = 0;
+		ready.set_value(false);
+		return;
+	}
+
+	mRenderThreadRunning = true;
+	ready.set_value(true);
+
+	for (;;)
+	{
+		std::function<void()> task;
+		{
+			std::unique_lock<std::mutex> lock(mRenderThreadMutex);
+			mRenderThreadCondition.wait(lock, [this]() {
+				return mRenderThreadStopping || !mRenderThreadTasks.empty();
+			});
+
+			if (mRenderThreadStopping && mRenderThreadTasks.empty())
+				break;
+
+			task = std::move(mRenderThreadTasks.front());
+			mRenderThreadTasks.pop();
+		}
+
+		try
+		{
+			task();
+		}
+		catch (...)
+		{
+			OutputDebugStringA("Render thread task failed with an unhandled exception.\n");
+		}
+	}
+
+	wglMakeCurrent(NULL, NULL);
+	mRenderThreadRunning = false;
+	mRenderThreadId = 0;
+}
+
+void RenderEngine::ReportRenderThreadRequestFailure(const char* operationName, const char* reason)
+{
+	std::ostringstream message;
+	message << "Render thread request failed";
+	if (operationName && operationName[0] != '\0')
+		message << " [" << operationName << "]";
+	if (reason && reason[0] != '\0')
+		message << ": " << reason;
+	message << ".\n";
+	OutputDebugStringA(message.str().c_str());
 }
 
 bool RenderEngine::CompileDecodeShader(int errorMessageSize, char* errorMessage)
 {
-	return mShaderPrograms.CompileDecodeShader(errorMessageSize, errorMessage);
+	return InvokeOnRenderThread([this, errorMessageSize, errorMessage]() {
+		return mShaderPrograms.CompileDecodeShader(errorMessageSize, errorMessage);
+	});
 }
 
 bool RenderEngine::CompileOutputPackShader(int errorMessageSize, char* errorMessage)
 {
-	return mShaderPrograms.CompileOutputPackShader(errorMessageSize, errorMessage);
+	return InvokeOnRenderThread([this, errorMessageSize, errorMessage]() {
+		return mShaderPrograms.CompileOutputPackShader(errorMessageSize, errorMessage);
+	});
 }
 
 bool RenderEngine::InitializeResources(
@@ -48,24 +159,30 @@ bool RenderEngine::InitializeResources(
 	unsigned outputPackTextureWidth,
 	std::string& error)
 {
-	return mRenderer.InitializeResources(
-		inputFrameWidth,
-		inputFrameHeight,
-		captureTextureWidth,
-		outputFrameWidth,
-		outputFrameHeight,
-		outputPackTextureWidth,
-		error);
+	return InvokeOnRenderThread([this, inputFrameWidth, inputFrameHeight, captureTextureWidth, outputFrameWidth, outputFrameHeight, outputPackTextureWidth, &error]() {
+		return mRenderer.InitializeResources(
+			inputFrameWidth,
+			inputFrameHeight,
+			captureTextureWidth,
+			outputFrameWidth,
+			outputFrameHeight,
+			outputPackTextureWidth,
+			error);
+	});
 }
 
 bool RenderEngine::CompileLayerPrograms(unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage)
 {
-	return mShaderPrograms.CompileLayerPrograms(inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
+	return InvokeOnRenderThread([this, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage]() {
+		return mShaderPrograms.CompileLayerPrograms(inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
+	});
 }
 
 bool RenderEngine::CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage)
 {
-	return mShaderPrograms.CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
+	return InvokeOnRenderThread([this, &preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage]() {
+		return mShaderPrograms.CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
+	});
 }
 
 bool RenderEngine::ApplyPreparedShaderBuild(
@@ -88,32 +205,38 @@ bool RenderEngine::ApplyPreparedShaderBuild(
 
 void RenderEngine::ResetTemporalHistoryState()
 {
-	mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly);
-	ProcessRenderResetCommandsOnRenderThread();
+	InvokeOnRenderThread([this]() {
+		mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly);
+		ProcessRenderResetCommandsOnRenderThread();
+	});
 }
 
 void RenderEngine::ResetShaderFeedbackState()
 {
-	mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::ShaderFeedbackOnly);
-	ProcessRenderResetCommandsOnRenderThread();
+	InvokeOnRenderThread([this]() {
+		mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::ShaderFeedbackOnly);
+		ProcessRenderResetCommandsOnRenderThread();
+	});
 }
 
 void RenderEngine::ApplyRuntimeCoordinatorRenderReset(RuntimeCoordinatorRenderResetScope resetScope)
 {
-	switch (resetScope)
-	{
-	case RuntimeCoordinatorRenderResetScope::TemporalHistoryOnly:
-		mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly);
-		ProcessRenderResetCommandsOnRenderThread();
-		break;
-	case RuntimeCoordinatorRenderResetScope::TemporalHistoryAndFeedback:
-		mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryAndFeedback);
-		ProcessRenderResetCommandsOnRenderThread();
-		break;
-	case RuntimeCoordinatorRenderResetScope::None:
-	default:
-		break;
-	}
+	InvokeOnRenderThread([this, resetScope]() {
+		switch (resetScope)
+		{
+		case RuntimeCoordinatorRenderResetScope::TemporalHistoryOnly:
+			mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly);
+			ProcessRenderResetCommandsOnRenderThread();
+			break;
+		case RuntimeCoordinatorRenderResetScope::TemporalHistoryAndFeedback:
+			mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryAndFeedback);
+			ProcessRenderResetCommandsOnRenderThread();
+			break;
+		case RuntimeCoordinatorRenderResetScope::None:
+		default:
+			break;
+		}
+	});
 }
 
 void RenderEngine::ResetTemporalHistoryStateOnRenderThread()
@@ -177,7 +300,9 @@ void RenderEngine::UpdateOscOverlayState(
 
 void RenderEngine::ResizeView(int width, int height)
 {
-	mRenderer.ResizeView(width, height);
+	InvokeOnRenderThread([this, width, height]() {
+		mRenderer.ResizeView(width, height);
+	});
 }
 
 bool RenderEngine::TryPresentPreview(bool force, unsigned previewFps, unsigned outputFrameWidth, unsigned outputFrameHeight)
@@ -196,6 +321,16 @@ bool RenderEngine::TryPresentPreview(bool force, unsigned previewFps, unsigned o
 		}
 	}
 
+	if (mRenderThreadRunning)
+	{
+		return TryInvokeOnRenderThread("preview-present", [this, outputFrameWidth, outputFrameHeight]() {
+			mRenderCommandQueue.RequestPreviewPresent({ outputFrameWidth, outputFrameHeight });
+			RenderPreviewPresentRequest request;
+			return mRenderCommandQueue.TryTakePreviewPresent(request) &&
+				PresentPreviewOnRenderThread(request.outputFrameWidth, request.outputFrameHeight);
+		});
+	}
+
 	if (!TryEnterCriticalSection(&mMutex))
 		return false;
 
@@ -219,11 +354,24 @@ bool RenderEngine::TryUploadInputFrame(const VideoIOFrame& inputFrame, const Vid
 	if (inputFrame.hasNoInputSource || inputFrame.bytes == nullptr)
 		return true;
 
+	if (mRenderThreadRunning)
+	{
+		return TryInvokeOnRenderThread("input-upload", [this, inputFrame, videoState]() {
+			mRenderCommandQueue.RequestInputUpload({ inputFrame, videoState });
+			RenderInputUploadRequest request;
+			return mRenderCommandQueue.TryTakeInputUpload(request) &&
+				UploadInputFrameOnRenderThread(request.inputFrame, request.videoState);
+		});
+	}
+
 	if (!TryEnterCriticalSection(&mMutex))
 		return false;
 
 	wglMakeCurrent(mHdc, mHglrc);
-	const bool uploaded = UploadInputFrameOnRenderThread(inputFrame, videoState);
+	mRenderCommandQueue.RequestInputUpload({ inputFrame, videoState });
+	RenderInputUploadRequest request;
+	const bool uploaded = mRenderCommandQueue.TryTakeInputUpload(request) &&
+		UploadInputFrameOnRenderThread(request.inputFrame, request.videoState);
 	wglMakeCurrent(NULL, NULL);
 	LeaveCriticalSection(&mMutex);
 	return uploaded;
@@ -249,9 +397,22 @@ bool RenderEngine::UploadInputFrameOnRenderThread(const VideoIOFrame& inputFrame
 
 bool RenderEngine::RenderOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame)
 {
+	if (mRenderThreadRunning)
+	{
+		return TryInvokeOnRenderThread("output-render", [this, &context, &outputFrame]() {
+			mRenderCommandQueue.RequestOutputFrame({ context.videoState, context.completion });
+			RenderOutputFrameRequest request;
+			return mRenderCommandQueue.TryTakeOutputFrame(request) &&
+				RenderOutputFrameOnRenderThread({ request.videoState, request.completion }, outputFrame);
+		});
+	}
+
 	EnterCriticalSection(&mMutex);
 	wglMakeCurrent(mHdc, mHglrc);
-	const bool rendered = RenderOutputFrameOnRenderThread(context, outputFrame);
+	mRenderCommandQueue.RequestOutputFrame({ context.videoState, context.completion });
+	RenderOutputFrameRequest request;
+	const bool rendered = mRenderCommandQueue.TryTakeOutputFrame(request) &&
+		RenderOutputFrameOnRenderThread({ request.videoState, request.completion }, outputFrame);
 	wglMakeCurrent(NULL, NULL);
 	LeaveCriticalSection(&mMutex);
 	return rendered;
@@ -339,6 +500,16 @@ bool RenderEngine::ReadOutputFrameRgbaOnRenderThread(unsigned width, unsigned he
 
 bool RenderEngine::CaptureOutputFrameRgbaTopDown(unsigned width, unsigned height, std::vector<unsigned char>& topDownPixels)
 {
+	if (mRenderThreadRunning)
+	{
+		return TryInvokeOnRenderThread("screenshot-capture", [this, width, height, &topDownPixels]() {
+			mRenderCommandQueue.RequestScreenshotCapture({ width, height });
+			RenderScreenshotCaptureRequest request;
+			return mRenderCommandQueue.TryTakeScreenshotCapture(request) &&
+				CaptureOutputFrameRgbaTopDownOnRenderThread(request.width, request.height, topDownPixels);
+		});
+	}
+
 	EnterCriticalSection(&mMutex);
 	wglMakeCurrent(mHdc, mHglrc);
 	mRenderCommandQueue.RequestScreenshotCapture({ width, height });

apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h

@@ -13,10 +13,19 @@
 
 #include <windows.h>
 
+#include <atomic>
 #include <cstdint>
 #include <chrono>
+#include <condition_variable>
 #include <functional>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <sstream>
 #include <string>
+#include <thread>
+#include <utility>
 #include <vector>
 
 class RenderEngine
@@ -60,6 +69,9 @@ public:
 		PreviewPaintCallback previewPaint);
 	~RenderEngine();
 
+	bool StartRenderThread();
+	void StopRenderThread();
+
 	bool CompileDecodeShader(int errorMessageSize, char* errorMessage);
 	bool CompileOutputPackShader(int errorMessageSize, char* errorMessage);
 	bool InitializeResources(
@@ -98,6 +110,74 @@ public:
 	bool CaptureOutputFrameRgbaTopDown(unsigned width, unsigned height, std::vector<unsigned char>& topDownPixels);
 
 private:
+	static constexpr std::chrono::milliseconds kRenderThreadRequestTimeout{ 250 };
+
+	struct RenderThreadTaskState
+	{
+		std::atomic<bool> started = false;
+		std::atomic<bool> cancelled = false;
+	};
+
+	template<typename Func>
+	auto InvokeOnRenderThread(Func&& func) -> decltype(func())
+	{
+		using Result = decltype(func());
+		if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId)
+			return func();
+
+		auto task = std::make_shared<std::packaged_task<Result()>>(std::forward<Func>(func));
+		std::future<Result> result = task->get_future();
+		{
+			std::lock_guard<std::mutex> lock(mRenderThreadMutex);
+			mRenderThreadTasks.push([task]() { (*task)(); });
+		}
+		mRenderThreadCondition.notify_one();
+		return result.get();
+	}
+
+	template<typename Func>
+	bool TryInvokeOnRenderThread(const char* operationName, Func&& func)
+	{
+		if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId)
+			return func();
+
+		auto state = std::make_shared<RenderThreadTaskState>();
+		auto task = std::make_shared<std::packaged_task<bool()>>(
+			[state, func = std::forward<Func>(func)]() mutable {
+				state->started = true;
+				if (state->cancelled)
+					return false;
+
+				return func();
+			});
+		std::future<bool> result = task->get_future();
+		{
+			std::lock_guard<std::mutex> lock(mRenderThreadMutex);
+			if (mRenderThreadStopping)
+			{
+				ReportRenderThreadRequestFailure(operationName, "render thread is stopping");
+				return false;
+			}
+			mRenderThreadTasks.push([task]() { (*task)(); });
+		}
+		mRenderThreadCondition.notify_one();
+
+		if (result.wait_for(kRenderThreadRequestTimeout) == std::future_status::ready)
+			return result.get();
+
+		if (!state->started)
+		{
+			state->cancelled = true;
+			ReportRenderThreadRequestFailure(operationName, "timed out before execution");
+			return false;
+		}
+
+		ReportRenderThreadRequestFailure(operationName, "exceeded timeout while executing; waiting for safe completion");
+		return result.get();
+	}
+
+	void RenderThreadMain(std::promise<bool> ready);
+	void ReportRenderThreadRequestFailure(const char* operationName, const char* reason);
 	bool CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage);
 	void RenderLayerStack(
 		bool hasInputSource,
@@ -129,4 +209,12 @@ private:
 	RenderCommandQueue mRenderCommandQueue;
 	RenderFrameStateResolver mFrameStateResolver;
 	RuntimeLiveState mRuntimeLiveState;
+	std::thread mRenderThread;
+	std::atomic<DWORD> mRenderThreadId = 0;
+	std::mutex mRenderThreadMutex;
+	std::condition_variable mRenderThreadCondition;
+	std::queue<std::function<void()>> mRenderThreadTasks;
+	std::atomic<bool> mRenderThreadRunning = false;
+	bool mRenderThreadStopping = false;
+	bool mResourcesDestroyed = false;
 };

docs/PHASE_3_LIVE_STATE_SERVICE_COORDINATION_DESIGN.md

@@ -286,8 +286,8 @@ Target shape:
 void OpenGLComposite::renderEffect()
 {
     mRuntimeUpdateController->ProcessRuntimeWork();
-    RenderFrameState frameState = mRenderFrameCoordinator->BuildFrameState(...);
-    mRenderEngine->RenderLayerStack(frameState);
+    const RenderFrameInput frameInput = BuildRenderFrameInput();
+    RenderFrame(frameInput);
 }
 ```
 

docs/PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md

@@ -7,16 +7,16 @@ Phase 1 named the subsystems. Phase 2 added the typed event substrate. Phase 3 m
 ## Status
 
 - Phase 4 design package: proposed.
-- Phase 4 implementation: Step 2 started. The existing synchronous `RenderEngine` entrypoints delegate their GL bodies to named `...OnRenderThread(...)` helpers, and preview/screenshot/render-reset requests now pass through a small `RenderCommandQueue` compatibility mailbox. No dedicated render thread exists yet.
-- Current alignment: the repo has a named frame-state contract and cleaner render-state preparation, but GL work is still entered through multiple paths protected by one shared `CRITICAL_SECTION`.
+- Phase 4 implementation: Step 3 started. The existing synchronous `RenderEngine` entrypoints delegate their GL bodies to named `...OnRenderThread(...)` helpers, preview/screenshot/render-reset/input-upload/output-render requests pass through a small `RenderCommandQueue` compatibility mailbox, and `RenderEngine` now starts a dedicated render thread for normal runtime GL work.
+- Current alignment: the repo has a named frame-state contract and cleaner render-state preparation. Normal runtime GL work is routed through the render thread after startup, while startup initialization still runs before the render thread is started.
 
 Current GL ownership footholds:
 
-- `RenderEngine` owns GL resources, the current context-binding compatibility shims, a small render command mailbox, and named render-thread helper methods.
+- `RenderEngine` owns GL resources, a dedicated render thread, the current synchronous compatibility shims, a small render command mailbox, and named render-thread helper methods.
 - `RenderFrameInput` / `RenderFrameState` provide the frame-state contract that a render thread can consume.
 - `RenderFrameStateResolver` prepares the render-facing layer state before drawing.
 - `OpenGLVideoIOBridge` still calls `RenderEngine::TryUploadInputFrame(...)` from the input path and `RenderEngine::RenderOutputFrame(...)` from the output path.
-- `OpenGLComposite::paintGL(...)`, screenshot capture, input upload, and output rendering still reach GL through `RenderEngine` methods that bind the shared context under `pMutex`.
+- `OpenGLComposite::paintGL(...)`, screenshot capture, input upload, and output rendering still call synchronous `RenderEngine` methods, but those methods now invoke render-thread work once `OpenGLComposite::Start()` has started the render thread.
 
 ## Why Phase 4 Exists
 
@@ -64,10 +64,10 @@ The current code paths that matter most are:
 
 | Entry point | Current behavior | Phase 4 direction |
 | --- | --- | --- |
-| `RenderEngine::TryUploadInputFrame(...)` | attempts to take the GL lock, binds the context, delegates upload to `UploadInputFrameOnRenderThread(...)` | enqueue latest input frame; render thread uploads |
-| `RenderEngine::RenderOutputFrame(...)` | takes the GL lock, binds the context, delegates render/readback to `RenderOutputFrameOnRenderThread(...)` | render thread executes output frame production |
-| `RenderEngine::TryPresentPreview(...)` | attempts to take the GL lock and delegates presentation to `PresentPreviewOnRenderThread(...)` | render thread or preview presenter consumes latest completed frame |
-| `RenderEngine::CaptureOutputFrameRgbaTopDown(...)` | takes the GL lock, binds the context, delegates readback to `CaptureOutputFrameRgbaTopDownOnRenderThread(...)` | screenshot request becomes render-thread command |
+| `RenderEngine::TryUploadInputFrame(...)` | synchronous compatibility shim; after render-thread startup it queues input upload work and waits for render-thread completion | enqueue latest input frame; render thread uploads without callback-owned GL |
+| `RenderEngine::RenderOutputFrame(...)` | synchronous compatibility shim; after render-thread startup it queues output render work and waits for render-thread completion | render thread executes output frame production |
+| `RenderEngine::TryPresentPreview(...)` | synchronous compatibility shim; after render-thread startup it queues preview presentation and waits for render-thread completion | render thread or preview presenter consumes latest completed frame |
+| `RenderEngine::CaptureOutputFrameRgbaTopDown(...)` | synchronous compatibility shim; after render-thread startup it queues screenshot readback and waits for render-thread completion | screenshot request becomes render-thread command |
 | `OpenGLVideoIOBridge::UploadInputFrame(...)` | calls render upload directly | push input frame into render queue/mailbox |
 | `OpenGLVideoIOBridge::RenderScheduledFrame(...)` | calls render output directly from backend path | request/consume render-produced output without callback-owned GL |
 
@@ -133,8 +133,10 @@ Current implementation:
 
 - `RenderCommandQueue` exists as a pure C++ mailbox helper.
 - Preview present and screenshot capture requests use latest-value coalescing.
+- Input upload requests use latest-value coalescing. During the compatibility phase the input frame memory is still drained immediately; a real render thread will need copied or otherwise owned frame storage.
+- Output frame requests use FIFO semantics so scheduled output demand is not collapsed.
 - Render-local reset requests coalesce to the strongest pending reset scope.
-- `RenderEngine` drains these commands synchronously as compatibility shims until a dedicated render thread is introduced.
+- The synchronous compatibility shims submit queued work to the render thread and wait for completion once the render thread is running.
 
 Possible commands:
 
@@ -234,14 +236,23 @@ Start with low-risk commands:
 - [x] preview present request
 - [x] screenshot request
 - [x] render-local reset requests
+- [x] input upload request
+- [x] output render request
 
-Then move input upload and output render requests once the queue and wakeup behavior are proven.
+The queue and wakeup behavior still need the dedicated render thread before the callbacks stop borrowing the GL context.
 
 ### Step 3. Start A Dedicated Render Thread
 
 Create the render thread and make it own context binding.
 
-Transitional behavior may still allow synchronous request/response for output frames. The important change is that the caller waits for render-thread completion rather than taking the GL context itself.
+- [x] create a dedicated render thread owned by `RenderEngine`
+- [x] bind the existing GL context on the render thread for normal runtime work
+- [x] stop the render thread before GL context destruction
+- [x] keep transitional synchronous request/response for output frames
+- [x] remove normal runtime dependence on the shared GL `CRITICAL_SECTION`
+- [x] add timeout/failure behavior for render-thread requests
+
+Transitional behavior still allows synchronous request/response for output frames. Render-thread requests now fail fast if they cannot begin within the request timeout, and log over-budget tasks that have already started before waiting for safe completion. The important change is that the caller waits for render-thread completion rather than taking the GL context itself.
 
 ### Step 4. Move Input Upload To The Render Thread
 

tests/RenderCommandQueueTests.cpp

@@ -60,14 +60,72 @@ void TestRenderResetScopesCoalesceToStrongestRequest()
 	Expect(queue.GetMetrics().depth == 0, "none reset request is ignored");
 }
 
+void TestInputUploadRequestUsesLatestValue()
+{
+	int firstPixel = 1;
+	int secondPixel = 2;
+	RenderCommandQueue queue;
+
+	RenderInputUploadRequest firstRequest;
+	firstRequest.inputFrame.bytes = &firstPixel;
+	firstRequest.inputFrame.width = 1920;
+	firstRequest.videoState.captureTextureWidth = 1920;
+	queue.RequestInputUpload(firstRequest);
+
+	RenderInputUploadRequest secondRequest;
+	secondRequest.inputFrame.bytes = &secondPixel;
+	secondRequest.inputFrame.width = 1280;
+	secondRequest.videoState.captureTextureWidth = 1280;
+	queue.RequestInputUpload(secondRequest);
+
+	const RenderCommandQueueMetrics metrics = queue.GetMetrics();
+	Expect(metrics.depth == 1, "input upload requests coalesce to one pending command");
+	Expect(metrics.enqueuedCount == 1, "first input upload request is counted as enqueued");
+	Expect(metrics.coalescedCount == 1, "second input upload request is counted as coalesced");
+
+	RenderInputUploadRequest request;
+	Expect(queue.TryTakeInputUpload(request), "input upload request can be consumed");
+	Expect(request.inputFrame.bytes == &secondPixel, "latest input upload bytes pointer wins");
+	Expect(request.inputFrame.width == 1280, "latest input upload frame wins");
+	Expect(request.videoState.captureTextureWidth == 1280, "latest input upload state wins");
+	Expect(!queue.TryTakeInputUpload(request), "input upload request is removed after consume");
+}
+
+void TestOutputFrameRequestsAreFifo()
+{
+	RenderCommandQueue queue;
+	RenderOutputFrameRequest firstRequest;
+	firstRequest.videoState.outputFrameSize.width = 1920;
+	firstRequest.completion.result = VideoIOCompletionResult::Completed;
+	queue.RequestOutputFrame(firstRequest);
+
+	RenderOutputFrameRequest secondRequest;
+	secondRequest.videoState.outputFrameSize.width = 1280;
+	secondRequest.completion.result = VideoIOCompletionResult::Dropped;
+	queue.RequestOutputFrame(secondRequest);
+
+	Expect(queue.GetMetrics().depth == 2, "output frame requests are queued independently");
+
+	RenderOutputFrameRequest request;
+	Expect(queue.TryTakeOutputFrame(request), "first output request can be consumed");
+	Expect(request.videoState.outputFrameSize.width == 1920, "first output request is consumed first");
+	Expect(request.completion.result == VideoIOCompletionResult::Completed, "first output completion is preserved");
+	Expect(queue.TryTakeOutputFrame(request), "second output request can be consumed");
+	Expect(request.videoState.outputFrameSize.width == 1280, "second output request is consumed second");
+	Expect(request.completion.result == VideoIOCompletionResult::Dropped, "second output completion is preserved");
+	Expect(!queue.TryTakeOutputFrame(request), "output queue is empty after consuming all requests");
+}
+
 void TestIndependentCommandKindsShareDepth()
 {
 	RenderCommandQueue queue;
 	queue.RequestPreviewPresent({ 1, 2 });
 	queue.RequestScreenshotCapture({ 3, 4 });
+	queue.RequestInputUpload({});
+	queue.RequestOutputFrame({});
 	queue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly);
 
-	Expect(queue.GetMetrics().depth == 3, "independent command kinds each contribute to depth");
+	Expect(queue.GetMetrics().depth == 5, "independent command kinds each contribute to depth");
 }
 }
 
@@ -76,6 +134,8 @@ int main()
 	TestPreviewRequestUsesLatestValue();
 	TestScreenshotRequestUsesLatestValue();
 	TestRenderResetScopesCoalesceToStrongestRequest();
+	TestInputUploadRequestUsesLatestValue();
+	TestOutputFrameRequestsAreFifo();
 	TestIndependentCommandKindsShareDepth();
 
 	if (gFailures != 0)