Step 3

apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp

@@ -257,7 +257,14 @@ bool OpenGLComposite::InitOpenGLState()
 
 bool OpenGLComposite::Start()
 {
-	return mVideoBackend->Start();
+	if (!mRenderEngine->StartRenderThread())
+		return false;
+
+	if (mVideoBackend->Start())
+		return true;
+
+	mRenderEngine->StopRenderThread();
+	return false;
 }
 
 bool OpenGLComposite::Stop()
@@ -272,6 +279,9 @@ bool OpenGLComposite::Stop()
 			mRuntimeStore && mRuntimeStore->IsExternalKeyingConfigured(),
 			"External keying has been disabled.");
 
+	if (mRenderEngine)
+		mRenderEngine->StopRenderThread();
+
 	return true;
 }
 

apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.cpp

@@ -48,6 +48,48 @@ bool RenderCommandQueue::TryTakeScreenshotCapture(RenderScreenshotCaptureRequest
 	return true;
 }
 
+void RenderCommandQueue::RequestInputUpload(const RenderInputUploadRequest& request)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (mHasInputUploadRequest)
+		++mCoalescedCount;
+	else
+		++mEnqueuedCount;
+
+	mInputUploadRequest = request;
+	mHasInputUploadRequest = true;
+}
+
+bool RenderCommandQueue::TryTakeInputUpload(RenderInputUploadRequest& request)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (!mHasInputUploadRequest)
+		return false;
+
+	request = mInputUploadRequest;
+	mInputUploadRequest = {};
+	mHasInputUploadRequest = false;
+	return true;
+}
+
+void RenderCommandQueue::RequestOutputFrame(const RenderOutputFrameRequest& request)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mOutputFrameRequests.push_back(request);
+	++mEnqueuedCount;
+}
+
+bool RenderCommandQueue::TryTakeOutputFrame(RenderOutputFrameRequest& request)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (mOutputFrameRequests.empty())
+		return false;
+
+	request = mOutputFrameRequests.front();
+	mOutputFrameRequests.pop_front();
+	return true;
+}
+
 void RenderCommandQueue::RequestRenderReset(RenderCommandResetScope scope)
 {
 	if (scope == RenderCommandResetScope::None)
@@ -80,6 +122,8 @@ RenderCommandQueueMetrics RenderCommandQueue::GetMetrics() const
 	metrics.depth =
 		(mHasPreviewPresentRequest ? 1u : 0u) +
 		(mHasScreenshotCaptureRequest ? 1u : 0u) +
+		(mHasInputUploadRequest ? 1u : 0u) +
+		mOutputFrameRequests.size() +
 		(mRenderResetScope != RenderCommandResetScope::None ? 1u : 0u);
 	metrics.enqueuedCount = mEnqueuedCount;
 	metrics.coalescedCount = mCoalescedCount;

apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h

@@ -1,7 +1,10 @@
 #pragma once
 
+#include "VideoIOTypes.h"
+
 #include <cstddef>
 #include <cstdint>
+#include <deque>
 #include <mutex>
 
 enum class RenderCommandResetScope
@@ -24,6 +27,18 @@ struct RenderScreenshotCaptureRequest
 	unsigned height = 0;
 };
 
+struct RenderInputUploadRequest
+{
+	VideoIOFrame inputFrame;
+	VideoIOState videoState;
+};
+
+struct RenderOutputFrameRequest
+{
+	VideoIOState videoState;
+	VideoIOCompletion completion;
+};
+
 struct RenderCommandQueueMetrics
 {
 	std::size_t depth = 0;
@@ -40,6 +55,12 @@ public:
 	void RequestScreenshotCapture(const RenderScreenshotCaptureRequest& request);
 	bool TryTakeScreenshotCapture(RenderScreenshotCaptureRequest& request);
 
+	void RequestInputUpload(const RenderInputUploadRequest& request);
+	bool TryTakeInputUpload(RenderInputUploadRequest& request);
+
+	void RequestOutputFrame(const RenderOutputFrameRequest& request);
+	bool TryTakeOutputFrame(RenderOutputFrameRequest& request);
+
 	void RequestRenderReset(RenderCommandResetScope scope);
 	bool TryTakeRenderReset(RenderCommandResetScope& scope);
 
@@ -53,6 +74,9 @@ private:
 	RenderPreviewPresentRequest mPreviewPresentRequest;
 	bool mHasScreenshotCaptureRequest = false;
 	RenderScreenshotCaptureRequest mScreenshotCaptureRequest;
+	bool mHasInputUploadRequest = false;
+	RenderInputUploadRequest mInputUploadRequest;
+	std::deque<RenderOutputFrameRequest> mOutputFrameRequests;
 	RenderCommandResetScope mRenderResetScope = RenderCommandResetScope::None;
 	uint64_t mEnqueuedCount = 0;
 	uint64_t mCoalescedCount = 0;

apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp

@@ -26,17 +26,128 @@ RenderEngine::RenderEngine(
 
 RenderEngine::~RenderEngine()
 {
-	mRenderer.DestroyResources();
+	StopRenderThread();
+	if (!mResourcesDestroyed)
+	{
+		mRenderer.DestroyResources();
+		mResourcesDestroyed = true;
+	}
+}
+
+bool RenderEngine::StartRenderThread()
+{
+	if (mRenderThreadRunning)
+		return true;
+
+	{
+		std::lock_guard<std::mutex> lock(mRenderThreadMutex);
+		mRenderThreadStopping = false;
+	}
+
+	std::promise<bool> ready;
+	std::future<bool> readyResult = ready.get_future();
+	mRenderThread = std::thread(&RenderEngine::RenderThreadMain, this, std::move(ready));
+	if (!readyResult.get())
+	{
+		if (mRenderThread.joinable())
+			mRenderThread.join();
+		return false;
+	}
+
+	return true;
+}
+
+void RenderEngine::StopRenderThread()
+{
+	if (mRenderThreadRunning)
+	{
+		InvokeOnRenderThread([this]() {
+			if (!mResourcesDestroyed)
+			{
+				mRenderer.DestroyResources();
+				mResourcesDestroyed = true;
+			}
+		});
+	}
+
+	{
+		std::lock_guard<std::mutex> lock(mRenderThreadMutex);
+		mRenderThreadStopping = true;
+	}
+	mRenderThreadCondition.notify_one();
+
+	if (mRenderThread.joinable())
+		mRenderThread.join();
+}
+
+void RenderEngine::RenderThreadMain(std::promise<bool> ready)
+{
+	mRenderThreadId = GetCurrentThreadId();
+	if (!wglMakeCurrent(mHdc, mHglrc))
+	{
+		mRenderThreadId = 0;
+		ready.set_value(false);
+		return;
+	}
+
+	mRenderThreadRunning = true;
+	ready.set_value(true);
+
+	for (;;)
+	{
+		std::function<void()> task;
+		{
+			std::unique_lock<std::mutex> lock(mRenderThreadMutex);
+			mRenderThreadCondition.wait(lock, [this]() {
+				return mRenderThreadStopping || !mRenderThreadTasks.empty();
+			});
+
+			if (mRenderThreadStopping && mRenderThreadTasks.empty())
+				break;
+
+			task = std::move(mRenderThreadTasks.front());
+			mRenderThreadTasks.pop();
+		}
+
+		try
+		{
+			task();
+		}
+		catch (...)
+		{
+			OutputDebugStringA("Render thread task failed with an unhandled exception.\n");
+		}
+	}
+
+	wglMakeCurrent(NULL, NULL);
+	mRenderThreadRunning = false;
+	mRenderThreadId = 0;
+}
+
+void RenderEngine::ReportRenderThreadRequestFailure(const char* operationName, const char* reason)
+{
+	std::ostringstream message;
+	message << "Render thread request failed";
+	if (operationName && operationName[0] != '\0')
+		message << " [" << operationName << "]";
+	if (reason && reason[0] != '\0')
+		message << ": " << reason;
+	message << ".\n";
+	OutputDebugStringA(message.str().c_str());
 }
 
 bool RenderEngine::CompileDecodeShader(int errorMessageSize, char* errorMessage)
 {
-	return mShaderPrograms.CompileDecodeShader(errorMessageSize, errorMessage);
+	return InvokeOnRenderThread([this, errorMessageSize, errorMessage]() {
+		return mShaderPrograms.CompileDecodeShader(errorMessageSize, errorMessage);
+	});
 }
 
 bool RenderEngine::CompileOutputPackShader(int errorMessageSize, char* errorMessage)
 {
-	return mShaderPrograms.CompileOutputPackShader(errorMessageSize, errorMessage);
+	return InvokeOnRenderThread([this, errorMessageSize, errorMessage]() {
+		return mShaderPrograms.CompileOutputPackShader(errorMessageSize, errorMessage);
+	});
 }
 
 bool RenderEngine::InitializeResources(
@@ -48,24 +159,30 @@ bool RenderEngine::InitializeResources(
 	unsigned outputPackTextureWidth,
 	std::string& error)
 {
-	return mRenderer.InitializeResources(
-		inputFrameWidth,
-		inputFrameHeight,
-		captureTextureWidth,
-		outputFrameWidth,
-		outputFrameHeight,
-		outputPackTextureWidth,
-		error);
+	return InvokeOnRenderThread([this, inputFrameWidth, inputFrameHeight, captureTextureWidth, outputFrameWidth, outputFrameHeight, outputPackTextureWidth, &error]() {
+		return mRenderer.InitializeResources(
+			inputFrameWidth,
+			inputFrameHeight,
+			captureTextureWidth,
+			outputFrameWidth,
+			outputFrameHeight,
+			outputPackTextureWidth,
+			error);
+	});
 }
 
 bool RenderEngine::CompileLayerPrograms(unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage)
 {
-	return mShaderPrograms.CompileLayerPrograms(inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
+	return InvokeOnRenderThread([this, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage]() {
+		return mShaderPrograms.CompileLayerPrograms(inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
+	});
 }
 
 bool RenderEngine::CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage)
 {
-	return mShaderPrograms.CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
+	return InvokeOnRenderThread([this, &preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage]() {
+		return mShaderPrograms.CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
+	});
 }
 
 bool RenderEngine::ApplyPreparedShaderBuild(
@@ -88,32 +205,38 @@ bool RenderEngine::ApplyPreparedShaderBuild(
 
 void RenderEngine::ResetTemporalHistoryState()
 {
-	mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly);
-	ProcessRenderResetCommandsOnRenderThread();
+	InvokeOnRenderThread([this]() {
+		mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly);
+		ProcessRenderResetCommandsOnRenderThread();
+	});
 }
 
 void RenderEngine::ResetShaderFeedbackState()
 {
-	mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::ShaderFeedbackOnly);
-	ProcessRenderResetCommandsOnRenderThread();
+	InvokeOnRenderThread([this]() {
+		mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::ShaderFeedbackOnly);
+		ProcessRenderResetCommandsOnRenderThread();
+	});
 }
 
 void RenderEngine::ApplyRuntimeCoordinatorRenderReset(RuntimeCoordinatorRenderResetScope resetScope)
 {
-	switch (resetScope)
-	{
-	case RuntimeCoordinatorRenderResetScope::TemporalHistoryOnly:
-		mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly);
-		ProcessRenderResetCommandsOnRenderThread();
-		break;
-	case RuntimeCoordinatorRenderResetScope::TemporalHistoryAndFeedback:
-		mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryAndFeedback);
-		ProcessRenderResetCommandsOnRenderThread();
-		break;
-	case RuntimeCoordinatorRenderResetScope::None:
-	default:
-		break;
-	}
+	InvokeOnRenderThread([this, resetScope]() {
+		switch (resetScope)
+		{
+		case RuntimeCoordinatorRenderResetScope::TemporalHistoryOnly:
+			mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly);
+			ProcessRenderResetCommandsOnRenderThread();
+			break;
+		case RuntimeCoordinatorRenderResetScope::TemporalHistoryAndFeedback:
+			mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryAndFeedback);
+			ProcessRenderResetCommandsOnRenderThread();
+			break;
+		case RuntimeCoordinatorRenderResetScope::None:
+		default:
+			break;
+		}
+	});
 }
 
 void RenderEngine::ResetTemporalHistoryStateOnRenderThread()
@@ -177,7 +300,9 @@ void RenderEngine::UpdateOscOverlayState(
 
 void RenderEngine::ResizeView(int width, int height)
 {
-	mRenderer.ResizeView(width, height);
+	InvokeOnRenderThread([this, width, height]() {
+		mRenderer.ResizeView(width, height);
+	});
 }
 
 bool RenderEngine::TryPresentPreview(bool force, unsigned previewFps, unsigned outputFrameWidth, unsigned outputFrameHeight)
@@ -196,6 +321,16 @@ bool RenderEngine::TryPresentPreview(bool force, unsigned previewFps, unsigned o
 		}
 	}
 
+	if (mRenderThreadRunning)
+	{
+		return TryInvokeOnRenderThread("preview-present", [this, outputFrameWidth, outputFrameHeight]() {
+			mRenderCommandQueue.RequestPreviewPresent({ outputFrameWidth, outputFrameHeight });
+			RenderPreviewPresentRequest request;
+			return mRenderCommandQueue.TryTakePreviewPresent(request) &&
+				PresentPreviewOnRenderThread(request.outputFrameWidth, request.outputFrameHeight);
+		});
+	}
+
 	if (!TryEnterCriticalSection(&mMutex))
 		return false;
 
@@ -219,11 +354,24 @@ bool RenderEngine::TryUploadInputFrame(const VideoIOFrame& inputFrame, const Vid
 	if (inputFrame.hasNoInputSource || inputFrame.bytes == nullptr)
 		return true;
 
+	if (mRenderThreadRunning)
+	{
+		return TryInvokeOnRenderThread("input-upload", [this, inputFrame, videoState]() {
+			mRenderCommandQueue.RequestInputUpload({ inputFrame, videoState });
+			RenderInputUploadRequest request;
+			return mRenderCommandQueue.TryTakeInputUpload(request) &&
+				UploadInputFrameOnRenderThread(request.inputFrame, request.videoState);
+		});
+	}
+
 	if (!TryEnterCriticalSection(&mMutex))
 		return false;
 
 	wglMakeCurrent(mHdc, mHglrc);
-	const bool uploaded = UploadInputFrameOnRenderThread(inputFrame, videoState);
+	mRenderCommandQueue.RequestInputUpload({ inputFrame, videoState });
+	RenderInputUploadRequest request;
+	const bool uploaded = mRenderCommandQueue.TryTakeInputUpload(request) &&
+		UploadInputFrameOnRenderThread(request.inputFrame, request.videoState);
 	wglMakeCurrent(NULL, NULL);
 	LeaveCriticalSection(&mMutex);
 	return uploaded;
@@ -249,9 +397,22 @@ bool RenderEngine::UploadInputFrameOnRenderThread(const VideoIOFrame& inputFrame
 
 bool RenderEngine::RenderOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame)
 {
+	if (mRenderThreadRunning)
+	{
+		return TryInvokeOnRenderThread("output-render", [this, &context, &outputFrame]() {
+			mRenderCommandQueue.RequestOutputFrame({ context.videoState, context.completion });
+			RenderOutputFrameRequest request;
+			return mRenderCommandQueue.TryTakeOutputFrame(request) &&
+				RenderOutputFrameOnRenderThread({ request.videoState, request.completion }, outputFrame);
+		});
+	}
+
 	EnterCriticalSection(&mMutex);
 	wglMakeCurrent(mHdc, mHglrc);
-	const bool rendered = RenderOutputFrameOnRenderThread(context, outputFrame);
+	mRenderCommandQueue.RequestOutputFrame({ context.videoState, context.completion });
+	RenderOutputFrameRequest request;
+	const bool rendered = mRenderCommandQueue.TryTakeOutputFrame(request) &&
+		RenderOutputFrameOnRenderThread({ request.videoState, request.completion }, outputFrame);
 	wglMakeCurrent(NULL, NULL);
 	LeaveCriticalSection(&mMutex);
 	return rendered;
@@ -339,6 +500,16 @@ bool RenderEngine::ReadOutputFrameRgbaOnRenderThread(unsigned width, unsigned he
 
 bool RenderEngine::CaptureOutputFrameRgbaTopDown(unsigned width, unsigned height, std::vector<unsigned char>& topDownPixels)
 {
+	if (mRenderThreadRunning)
+	{
+		return TryInvokeOnRenderThread("screenshot-capture", [this, width, height, &topDownPixels]() {
+			mRenderCommandQueue.RequestScreenshotCapture({ width, height });
+			RenderScreenshotCaptureRequest request;
+			return mRenderCommandQueue.TryTakeScreenshotCapture(request) &&
+				CaptureOutputFrameRgbaTopDownOnRenderThread(request.width, request.height, topDownPixels);
+		});
+	}
+
 	EnterCriticalSection(&mMutex);
 	wglMakeCurrent(mHdc, mHglrc);
 	mRenderCommandQueue.RequestScreenshotCapture({ width, height });

apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h

@@ -13,10 +13,19 @@
 
 #include <windows.h>
 
+#include <atomic>
 #include <cstdint>
 #include <chrono>
+#include <condition_variable>
 #include <functional>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <sstream>
 #include <string>
+#include <thread>
+#include <utility>
 #include <vector>
 
 class RenderEngine
@@ -60,6 +69,9 @@ public:
 		PreviewPaintCallback previewPaint);
 	~RenderEngine();
 
+	bool StartRenderThread();
+	void StopRenderThread();
+
 	bool CompileDecodeShader(int errorMessageSize, char* errorMessage);
 	bool CompileOutputPackShader(int errorMessageSize, char* errorMessage);
 	bool InitializeResources(
@@ -98,6 +110,74 @@ public:
 	bool CaptureOutputFrameRgbaTopDown(unsigned width, unsigned height, std::vector<unsigned char>& topDownPixels);
 
 private:
+	static constexpr std::chrono::milliseconds kRenderThreadRequestTimeout{ 250 };
+
+	struct RenderThreadTaskState
+	{
+		std::atomic<bool> started = false;
+		std::atomic<bool> cancelled = false;
+	};
+
+	template<typename Func>
+	auto InvokeOnRenderThread(Func&& func) -> decltype(func())
+	{
+		using Result = decltype(func());
+		if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId)
+			return func();
+
+		auto task = std::make_shared<std::packaged_task<Result()>>(std::forward<Func>(func));
+		std::future<Result> result = task->get_future();
+		{
+			std::lock_guard<std::mutex> lock(mRenderThreadMutex);
+			mRenderThreadTasks.push([task]() { (*task)(); });
+		}
+		mRenderThreadCondition.notify_one();
+		return result.get();
+	}
+
+	template<typename Func>
+	bool TryInvokeOnRenderThread(const char* operationName, Func&& func)
+	{
+		if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId)
+			return func();
+
+		auto state = std::make_shared<RenderThreadTaskState>();
+		auto task = std::make_shared<std::packaged_task<bool()>>(
+			[state, func = std::forward<Func>(func)]() mutable {
+				state->started = true;
+				if (state->cancelled)
+					return false;
+
+				return func();
+			});
+		std::future<bool> result = task->get_future();
+		{
+			std::lock_guard<std::mutex> lock(mRenderThreadMutex);
+			if (mRenderThreadStopping)
+			{
+				ReportRenderThreadRequestFailure(operationName, "render thread is stopping");
+				return false;
+			}
+			mRenderThreadTasks.push([task]() { (*task)(); });
+		}
+		mRenderThreadCondition.notify_one();
+
+		if (result.wait_for(kRenderThreadRequestTimeout) == std::future_status::ready)
+			return result.get();
+
+		if (!state->started)
+		{
+			state->cancelled = true;
+			ReportRenderThreadRequestFailure(operationName, "timed out before execution");
+			return false;
+		}
+
+		ReportRenderThreadRequestFailure(operationName, "exceeded timeout while executing; waiting for safe completion");
+		return result.get();
+	}
+
+	void RenderThreadMain(std::promise<bool> ready);
+	void ReportRenderThreadRequestFailure(const char* operationName, const char* reason);
 	bool CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage);
 	void RenderLayerStack(
 		bool hasInputSource,
@@ -129,4 +209,12 @@ private:
 	RenderCommandQueue mRenderCommandQueue;
 	RenderFrameStateResolver mFrameStateResolver;
 	RuntimeLiveState mRuntimeLiveState;
+	std::thread mRenderThread;
+	std::atomic<DWORD> mRenderThreadId = 0;
+	std::mutex mRenderThreadMutex;
+	std::condition_variable mRenderThreadCondition;
+	std::queue<std::function<void()>> mRenderThreadTasks;
+	std::atomic<bool> mRenderThreadRunning = false;
+	bool mRenderThreadStopping = false;
+	bool mResourcesDestroyed = false;
 };