From 20476bdf63d2f4040730bbb54336bec747022d07 Mon Sep 17 00:00:00 2001 From: Aiden <68633820+awils27@users.noreply.github.com> Date: Mon, 11 May 2026 17:41:59 +1000 Subject: [PATCH] Step 3 --- CMakeLists.txt | 2 + .../LoopThroughWithOpenGLCompositing.cpp | 5 +- .../gl/OpenGLComposite.cpp | 12 +- .../gl/RenderCommandQueue.cpp | 44 ++++ .../gl/RenderCommandQueue.h | 24 ++ .../gl/RenderEngine.cpp | 239 +++++++++++++++--- .../gl/RenderEngine.h | 88 +++++++ ..._LIVE_STATE_SERVICE_COORDINATION_DESIGN.md | 4 +- .../PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md | 33 ++- tests/RenderCommandQueueTests.cpp | 62 ++++- 10 files changed, 460 insertions(+), 53 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d8ea0f4..eb90633 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -378,6 +378,8 @@ add_executable(RenderCommandQueueTests target_include_directories(RenderCommandQueueTests PRIVATE "${APP_DIR}" "${APP_DIR}/gl" + "${APP_DIR}/videoio" + "${APP_DIR}/videoio/decklink" ) if(MSVC) diff --git a/apps/LoopThroughWithOpenGLCompositing/LoopThroughWithOpenGLCompositing.cpp b/apps/LoopThroughWithOpenGLCompositing/LoopThroughWithOpenGLCompositing.cpp index bf2e273..fa9484c 100644 --- a/apps/LoopThroughWithOpenGLCompositing/LoopThroughWithOpenGLCompositing.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/LoopThroughWithOpenGLCompositing.cpp @@ -434,7 +434,7 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) } // Deselect the current rendering context and delete it - wglMakeCurrent(hDC, NULL); + wglMakeCurrent(NULL, NULL); wglDeleteContext(hRC); // Tell the application to terminate after the window is gone @@ -486,15 +486,12 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) if (!sInteractiveResize && pOpenGLComposite) { - wglMakeCurrent(hDC, hRC); pOpenGLComposite->paintGL(true); - wglMakeCurrent( NULL, NULL ); RaiseStatusControls(sStatusStrip); } } catch (...) { - wglMakeCurrent( NULL, NULL ); ShowUnhandledExceptionMessage("Paint failed inside the OpenGL runtime."); } break; diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp index d81b427..a810947 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp @@ -257,7 +257,14 @@ bool OpenGLComposite::InitOpenGLState() bool OpenGLComposite::Start() { - return mVideoBackend->Start(); + if (!mRenderEngine->StartRenderThread()) + return false; + + if (mVideoBackend->Start()) + return true; + + mRenderEngine->StopRenderThread(); + return false; } bool OpenGLComposite::Stop() @@ -272,6 +279,9 @@ bool OpenGLComposite::Stop() mRuntimeStore && mRuntimeStore->IsExternalKeyingConfigured(), "External keying has been disabled."); + if (mRenderEngine) + mRenderEngine->StopRenderThread(); + return true; } diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.cpp b/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.cpp index e0547ca..68ad021 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.cpp @@ -48,6 +48,48 @@ bool RenderCommandQueue::TryTakeScreenshotCapture(RenderScreenshotCaptureRequest return true; } +void RenderCommandQueue::RequestInputUpload(const RenderInputUploadRequest& request) +{ + std::lock_guard lock(mMutex); + if (mHasInputUploadRequest) + ++mCoalescedCount; + else + ++mEnqueuedCount; + + mInputUploadRequest = request; + mHasInputUploadRequest = true; +} + +bool RenderCommandQueue::TryTakeInputUpload(RenderInputUploadRequest& request) +{ + std::lock_guard lock(mMutex); + if (!mHasInputUploadRequest) + return false; + + request = mInputUploadRequest; + mInputUploadRequest = {}; + mHasInputUploadRequest = false; + return true; +} + +void RenderCommandQueue::RequestOutputFrame(const RenderOutputFrameRequest& request) +{ + std::lock_guard lock(mMutex); + mOutputFrameRequests.push_back(request); + ++mEnqueuedCount; +} + +bool RenderCommandQueue::TryTakeOutputFrame(RenderOutputFrameRequest& request) +{ + std::lock_guard lock(mMutex); + if (mOutputFrameRequests.empty()) + return false; + + request = mOutputFrameRequests.front(); + mOutputFrameRequests.pop_front(); + return true; +} + void RenderCommandQueue::RequestRenderReset(RenderCommandResetScope scope) { if (scope == RenderCommandResetScope::None) @@ -80,6 +122,8 @@ RenderCommandQueueMetrics RenderCommandQueue::GetMetrics() const metrics.depth = (mHasPreviewPresentRequest ? 1u : 0u) + (mHasScreenshotCaptureRequest ? 1u : 0u) + + (mHasInputUploadRequest ? 1u : 0u) + + mOutputFrameRequests.size() + (mRenderResetScope != RenderCommandResetScope::None ? 1u : 0u); metrics.enqueuedCount = mEnqueuedCount; metrics.coalescedCount = mCoalescedCount; diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h b/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h index b068f20..bc55646 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h +++ b/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h @@ -1,7 +1,10 @@ #pragma once +#include "VideoIOTypes.h" + #include #include +#include #include enum class RenderCommandResetScope @@ -24,6 +27,18 @@ struct RenderScreenshotCaptureRequest unsigned height = 0; }; +struct RenderInputUploadRequest +{ + VideoIOFrame inputFrame; + VideoIOState videoState; +}; + +struct RenderOutputFrameRequest +{ + VideoIOState videoState; + VideoIOCompletion completion; +}; + struct RenderCommandQueueMetrics { std::size_t depth = 0; @@ -40,6 +55,12 @@ public: void RequestScreenshotCapture(const RenderScreenshotCaptureRequest& request); bool TryTakeScreenshotCapture(RenderScreenshotCaptureRequest& request); + void RequestInputUpload(const RenderInputUploadRequest& request); + bool TryTakeInputUpload(RenderInputUploadRequest& request); + + void RequestOutputFrame(const RenderOutputFrameRequest& request); + bool TryTakeOutputFrame(RenderOutputFrameRequest& request); + void RequestRenderReset(RenderCommandResetScope scope); bool TryTakeRenderReset(RenderCommandResetScope& scope); @@ -53,6 +74,9 @@ private: RenderPreviewPresentRequest mPreviewPresentRequest; bool mHasScreenshotCaptureRequest = false; RenderScreenshotCaptureRequest mScreenshotCaptureRequest; + bool mHasInputUploadRequest = false; + RenderInputUploadRequest mInputUploadRequest; + std::deque mOutputFrameRequests; RenderCommandResetScope mRenderResetScope = RenderCommandResetScope::None; uint64_t mEnqueuedCount = 0; uint64_t mCoalescedCount = 0; diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp b/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp index 73976a2..af7acb1 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp @@ -26,17 +26,128 @@ RenderEngine::RenderEngine( RenderEngine::~RenderEngine() { - mRenderer.DestroyResources(); + StopRenderThread(); + if (!mResourcesDestroyed) + { + mRenderer.DestroyResources(); + mResourcesDestroyed = true; + } +} + +bool RenderEngine::StartRenderThread() +{ + if (mRenderThreadRunning) + return true; + + { + std::lock_guard lock(mRenderThreadMutex); + mRenderThreadStopping = false; + } + + std::promise ready; + std::future readyResult = ready.get_future(); + mRenderThread = std::thread(&RenderEngine::RenderThreadMain, this, std::move(ready)); + if (!readyResult.get()) + { + if (mRenderThread.joinable()) + mRenderThread.join(); + return false; + } + + return true; +} + +void RenderEngine::StopRenderThread() +{ + if (mRenderThreadRunning) + { + InvokeOnRenderThread([this]() { + if (!mResourcesDestroyed) + { + mRenderer.DestroyResources(); + mResourcesDestroyed = true; + } + }); + } + + { + std::lock_guard lock(mRenderThreadMutex); + mRenderThreadStopping = true; + } + mRenderThreadCondition.notify_one(); + + if (mRenderThread.joinable()) + mRenderThread.join(); +} + +void RenderEngine::RenderThreadMain(std::promise ready) +{ + mRenderThreadId = GetCurrentThreadId(); + if (!wglMakeCurrent(mHdc, mHglrc)) + { + mRenderThreadId = 0; + ready.set_value(false); + return; + } + + mRenderThreadRunning = true; + ready.set_value(true); + + for (;;) + { + std::function task; + { + std::unique_lock lock(mRenderThreadMutex); + mRenderThreadCondition.wait(lock, [this]() { + return mRenderThreadStopping || !mRenderThreadTasks.empty(); + }); + + if (mRenderThreadStopping && mRenderThreadTasks.empty()) + break; + + task = std::move(mRenderThreadTasks.front()); + mRenderThreadTasks.pop(); + } + + try + { + task(); + } + catch (...) + { + OutputDebugStringA("Render thread task failed with an unhandled exception.\n"); + } + } + + wglMakeCurrent(NULL, NULL); + mRenderThreadRunning = false; + mRenderThreadId = 0; +} + +void RenderEngine::ReportRenderThreadRequestFailure(const char* operationName, const char* reason) +{ + std::ostringstream message; + message << "Render thread request failed"; + if (operationName && operationName[0] != '\0') + message << " [" << operationName << "]"; + if (reason && reason[0] != '\0') + message << ": " << reason; + message << ".\n"; + OutputDebugStringA(message.str().c_str()); } bool RenderEngine::CompileDecodeShader(int errorMessageSize, char* errorMessage) { - return mShaderPrograms.CompileDecodeShader(errorMessageSize, errorMessage); + return InvokeOnRenderThread([this, errorMessageSize, errorMessage]() { + return mShaderPrograms.CompileDecodeShader(errorMessageSize, errorMessage); + }); } bool RenderEngine::CompileOutputPackShader(int errorMessageSize, char* errorMessage) { - return mShaderPrograms.CompileOutputPackShader(errorMessageSize, errorMessage); + return InvokeOnRenderThread([this, errorMessageSize, errorMessage]() { + return mShaderPrograms.CompileOutputPackShader(errorMessageSize, errorMessage); + }); } bool RenderEngine::InitializeResources( @@ -48,24 +159,30 @@ bool RenderEngine::InitializeResources( unsigned outputPackTextureWidth, std::string& error) { - return mRenderer.InitializeResources( - inputFrameWidth, - inputFrameHeight, - captureTextureWidth, - outputFrameWidth, - outputFrameHeight, - outputPackTextureWidth, - error); + return InvokeOnRenderThread([this, inputFrameWidth, inputFrameHeight, captureTextureWidth, outputFrameWidth, outputFrameHeight, outputPackTextureWidth, &error]() { + return mRenderer.InitializeResources( + inputFrameWidth, + inputFrameHeight, + captureTextureWidth, + outputFrameWidth, + outputFrameHeight, + outputPackTextureWidth, + error); + }); } bool RenderEngine::CompileLayerPrograms(unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage) { - return mShaderPrograms.CompileLayerPrograms(inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage); + return InvokeOnRenderThread([this, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage]() { + return mShaderPrograms.CompileLayerPrograms(inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage); + }); } bool RenderEngine::CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage) { - return mShaderPrograms.CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage); + return InvokeOnRenderThread([this, &preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage]() { + return mShaderPrograms.CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage); + }); } bool RenderEngine::ApplyPreparedShaderBuild( @@ -88,32 +205,38 @@ bool RenderEngine::ApplyPreparedShaderBuild( void RenderEngine::ResetTemporalHistoryState() { - mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly); - ProcessRenderResetCommandsOnRenderThread(); + InvokeOnRenderThread([this]() { + mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly); + ProcessRenderResetCommandsOnRenderThread(); + }); } void RenderEngine::ResetShaderFeedbackState() { - mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::ShaderFeedbackOnly); - ProcessRenderResetCommandsOnRenderThread(); + InvokeOnRenderThread([this]() { + mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::ShaderFeedbackOnly); + ProcessRenderResetCommandsOnRenderThread(); + }); } void RenderEngine::ApplyRuntimeCoordinatorRenderReset(RuntimeCoordinatorRenderResetScope resetScope) { - switch (resetScope) - { - case RuntimeCoordinatorRenderResetScope::TemporalHistoryOnly: - mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly); - ProcessRenderResetCommandsOnRenderThread(); - break; - case RuntimeCoordinatorRenderResetScope::TemporalHistoryAndFeedback: - mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryAndFeedback); - ProcessRenderResetCommandsOnRenderThread(); - break; - case RuntimeCoordinatorRenderResetScope::None: - default: - break; - } + InvokeOnRenderThread([this, resetScope]() { + switch (resetScope) + { + case RuntimeCoordinatorRenderResetScope::TemporalHistoryOnly: + mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly); + ProcessRenderResetCommandsOnRenderThread(); + break; + case RuntimeCoordinatorRenderResetScope::TemporalHistoryAndFeedback: + mRenderCommandQueue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryAndFeedback); + ProcessRenderResetCommandsOnRenderThread(); + break; + case RuntimeCoordinatorRenderResetScope::None: + default: + break; + } + }); } void RenderEngine::ResetTemporalHistoryStateOnRenderThread() @@ -177,7 +300,9 @@ void RenderEngine::UpdateOscOverlayState( void RenderEngine::ResizeView(int width, int height) { - mRenderer.ResizeView(width, height); + InvokeOnRenderThread([this, width, height]() { + mRenderer.ResizeView(width, height); + }); } bool RenderEngine::TryPresentPreview(bool force, unsigned previewFps, unsigned outputFrameWidth, unsigned outputFrameHeight) @@ -196,6 +321,16 @@ bool RenderEngine::TryPresentPreview(bool force, unsigned previewFps, unsigned o } } + if (mRenderThreadRunning) + { + return TryInvokeOnRenderThread("preview-present", [this, outputFrameWidth, outputFrameHeight]() { + mRenderCommandQueue.RequestPreviewPresent({ outputFrameWidth, outputFrameHeight }); + RenderPreviewPresentRequest request; + return mRenderCommandQueue.TryTakePreviewPresent(request) && + PresentPreviewOnRenderThread(request.outputFrameWidth, request.outputFrameHeight); + }); + } + if (!TryEnterCriticalSection(&mMutex)) return false; @@ -219,11 +354,24 @@ bool RenderEngine::TryUploadInputFrame(const VideoIOFrame& inputFrame, const Vid if (inputFrame.hasNoInputSource || inputFrame.bytes == nullptr) return true; + if (mRenderThreadRunning) + { + return TryInvokeOnRenderThread("input-upload", [this, inputFrame, videoState]() { + mRenderCommandQueue.RequestInputUpload({ inputFrame, videoState }); + RenderInputUploadRequest request; + return mRenderCommandQueue.TryTakeInputUpload(request) && + UploadInputFrameOnRenderThread(request.inputFrame, request.videoState); + }); + } + if (!TryEnterCriticalSection(&mMutex)) return false; wglMakeCurrent(mHdc, mHglrc); - const bool uploaded = UploadInputFrameOnRenderThread(inputFrame, videoState); + mRenderCommandQueue.RequestInputUpload({ inputFrame, videoState }); + RenderInputUploadRequest request; + const bool uploaded = mRenderCommandQueue.TryTakeInputUpload(request) && + UploadInputFrameOnRenderThread(request.inputFrame, request.videoState); wglMakeCurrent(NULL, NULL); LeaveCriticalSection(&mMutex); return uploaded; @@ -249,9 +397,22 @@ bool RenderEngine::UploadInputFrameOnRenderThread(const VideoIOFrame& inputFrame bool RenderEngine::RenderOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame) { + if (mRenderThreadRunning) + { + return TryInvokeOnRenderThread("output-render", [this, &context, &outputFrame]() { + mRenderCommandQueue.RequestOutputFrame({ context.videoState, context.completion }); + RenderOutputFrameRequest request; + return mRenderCommandQueue.TryTakeOutputFrame(request) && + RenderOutputFrameOnRenderThread({ request.videoState, request.completion }, outputFrame); + }); + } + EnterCriticalSection(&mMutex); wglMakeCurrent(mHdc, mHglrc); - const bool rendered = RenderOutputFrameOnRenderThread(context, outputFrame); + mRenderCommandQueue.RequestOutputFrame({ context.videoState, context.completion }); + RenderOutputFrameRequest request; + const bool rendered = mRenderCommandQueue.TryTakeOutputFrame(request) && + RenderOutputFrameOnRenderThread({ request.videoState, request.completion }, outputFrame); wglMakeCurrent(NULL, NULL); LeaveCriticalSection(&mMutex); return rendered; @@ -339,6 +500,16 @@ bool RenderEngine::ReadOutputFrameRgbaOnRenderThread(unsigned width, unsigned he bool RenderEngine::CaptureOutputFrameRgbaTopDown(unsigned width, unsigned height, std::vector& topDownPixels) { + if (mRenderThreadRunning) + { + return TryInvokeOnRenderThread("screenshot-capture", [this, width, height, &topDownPixels]() { + mRenderCommandQueue.RequestScreenshotCapture({ width, height }); + RenderScreenshotCaptureRequest request; + return mRenderCommandQueue.TryTakeScreenshotCapture(request) && + CaptureOutputFrameRgbaTopDownOnRenderThread(request.width, request.height, topDownPixels); + }); + } + EnterCriticalSection(&mMutex); wglMakeCurrent(mHdc, mHglrc); mRenderCommandQueue.RequestScreenshotCapture({ width, height }); diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h b/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h index 94f52a7..47048fd 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h +++ b/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h @@ -13,10 +13,19 @@ #include +#include #include #include +#include #include +#include +#include +#include +#include +#include #include +#include +#include #include class RenderEngine @@ -60,6 +69,9 @@ public: PreviewPaintCallback previewPaint); ~RenderEngine(); + bool StartRenderThread(); + void StopRenderThread(); + bool CompileDecodeShader(int errorMessageSize, char* errorMessage); bool CompileOutputPackShader(int errorMessageSize, char* errorMessage); bool InitializeResources( @@ -98,6 +110,74 @@ public: bool CaptureOutputFrameRgbaTopDown(unsigned width, unsigned height, std::vector& topDownPixels); private: + static constexpr std::chrono::milliseconds kRenderThreadRequestTimeout{ 250 }; + + struct RenderThreadTaskState + { + std::atomic started = false; + std::atomic cancelled = false; + }; + + template + auto InvokeOnRenderThread(Func&& func) -> decltype(func()) + { + using Result = decltype(func()); + if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId) + return func(); + + auto task = std::make_shared>(std::forward(func)); + std::future result = task->get_future(); + { + std::lock_guard lock(mRenderThreadMutex); + mRenderThreadTasks.push([task]() { (*task)(); }); + } + mRenderThreadCondition.notify_one(); + return result.get(); + } + + template + bool TryInvokeOnRenderThread(const char* operationName, Func&& func) + { + if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId) + return func(); + + auto state = std::make_shared(); + auto task = std::make_shared>( + [state, func = std::forward(func)]() mutable { + state->started = true; + if (state->cancelled) + return false; + + return func(); + }); + std::future result = task->get_future(); + { + std::lock_guard lock(mRenderThreadMutex); + if (mRenderThreadStopping) + { + ReportRenderThreadRequestFailure(operationName, "render thread is stopping"); + return false; + } + mRenderThreadTasks.push([task]() { (*task)(); }); + } + mRenderThreadCondition.notify_one(); + + if (result.wait_for(kRenderThreadRequestTimeout) == std::future_status::ready) + return result.get(); + + if (!state->started) + { + state->cancelled = true; + ReportRenderThreadRequestFailure(operationName, "timed out before execution"); + return false; + } + + ReportRenderThreadRequestFailure(operationName, "exceeded timeout while executing; waiting for safe completion"); + return result.get(); + } + + void RenderThreadMain(std::promise ready); + void ReportRenderThreadRequestFailure(const char* operationName, const char* reason); bool CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage); void RenderLayerStack( bool hasInputSource, @@ -129,4 +209,12 @@ private: RenderCommandQueue mRenderCommandQueue; RenderFrameStateResolver mFrameStateResolver; RuntimeLiveState mRuntimeLiveState; + std::thread mRenderThread; + std::atomic mRenderThreadId = 0; + std::mutex mRenderThreadMutex; + std::condition_variable mRenderThreadCondition; + std::queue> mRenderThreadTasks; + std::atomic mRenderThreadRunning = false; + bool mRenderThreadStopping = false; + bool mResourcesDestroyed = false; }; diff --git a/docs/PHASE_3_LIVE_STATE_SERVICE_COORDINATION_DESIGN.md b/docs/PHASE_3_LIVE_STATE_SERVICE_COORDINATION_DESIGN.md index caa5d97..aeb6e97 100644 --- a/docs/PHASE_3_LIVE_STATE_SERVICE_COORDINATION_DESIGN.md +++ b/docs/PHASE_3_LIVE_STATE_SERVICE_COORDINATION_DESIGN.md @@ -286,8 +286,8 @@ Target shape: void OpenGLComposite::renderEffect() { mRuntimeUpdateController->ProcessRuntimeWork(); - RenderFrameState frameState = mRenderFrameCoordinator->BuildFrameState(...); - mRenderEngine->RenderLayerStack(frameState); + const RenderFrameInput frameInput = BuildRenderFrameInput(); + RenderFrame(frameInput); } ``` diff --git a/docs/PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md b/docs/PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md index 3414b5c..4258134 100644 --- a/docs/PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md +++ b/docs/PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md @@ -7,16 +7,16 @@ Phase 1 named the subsystems. Phase 2 added the typed event substrate. Phase 3 m ## Status - Phase 4 design package: proposed. -- Phase 4 implementation: Step 2 started. The existing synchronous `RenderEngine` entrypoints delegate their GL bodies to named `...OnRenderThread(...)` helpers, and preview/screenshot/render-reset requests now pass through a small `RenderCommandQueue` compatibility mailbox. No dedicated render thread exists yet. -- Current alignment: the repo has a named frame-state contract and cleaner render-state preparation, but GL work is still entered through multiple paths protected by one shared `CRITICAL_SECTION`. +- Phase 4 implementation: Step 3 started. The existing synchronous `RenderEngine` entrypoints delegate their GL bodies to named `...OnRenderThread(...)` helpers, preview/screenshot/render-reset/input-upload/output-render requests pass through a small `RenderCommandQueue` compatibility mailbox, and `RenderEngine` now starts a dedicated render thread for normal runtime GL work. +- Current alignment: the repo has a named frame-state contract and cleaner render-state preparation. Normal runtime GL work is routed through the render thread after startup, while startup initialization still runs before the render thread is started. Current GL ownership footholds: -- `RenderEngine` owns GL resources, the current context-binding compatibility shims, a small render command mailbox, and named render-thread helper methods. +- `RenderEngine` owns GL resources, a dedicated render thread, the current synchronous compatibility shims, a small render command mailbox, and named render-thread helper methods. - `RenderFrameInput` / `RenderFrameState` provide the frame-state contract that a render thread can consume. - `RenderFrameStateResolver` prepares the render-facing layer state before drawing. - `OpenGLVideoIOBridge` still calls `RenderEngine::TryUploadInputFrame(...)` from the input path and `RenderEngine::RenderOutputFrame(...)` from the output path. -- `OpenGLComposite::paintGL(...)`, screenshot capture, input upload, and output rendering still reach GL through `RenderEngine` methods that bind the shared context under `pMutex`. +- `OpenGLComposite::paintGL(...)`, screenshot capture, input upload, and output rendering still call synchronous `RenderEngine` methods, but those methods now invoke render-thread work once `OpenGLComposite::Start()` has started the render thread. ## Why Phase 4 Exists @@ -64,10 +64,10 @@ The current code paths that matter most are: | Entry point | Current behavior | Phase 4 direction | | --- | --- | --- | -| `RenderEngine::TryUploadInputFrame(...)` | attempts to take the GL lock, binds the context, delegates upload to `UploadInputFrameOnRenderThread(...)` | enqueue latest input frame; render thread uploads | -| `RenderEngine::RenderOutputFrame(...)` | takes the GL lock, binds the context, delegates render/readback to `RenderOutputFrameOnRenderThread(...)` | render thread executes output frame production | -| `RenderEngine::TryPresentPreview(...)` | attempts to take the GL lock and delegates presentation to `PresentPreviewOnRenderThread(...)` | render thread or preview presenter consumes latest completed frame | -| `RenderEngine::CaptureOutputFrameRgbaTopDown(...)` | takes the GL lock, binds the context, delegates readback to `CaptureOutputFrameRgbaTopDownOnRenderThread(...)` | screenshot request becomes render-thread command | +| `RenderEngine::TryUploadInputFrame(...)` | synchronous compatibility shim; after render-thread startup it queues input upload work and waits for render-thread completion | enqueue latest input frame; render thread uploads without callback-owned GL | +| `RenderEngine::RenderOutputFrame(...)` | synchronous compatibility shim; after render-thread startup it queues output render work and waits for render-thread completion | render thread executes output frame production | +| `RenderEngine::TryPresentPreview(...)` | synchronous compatibility shim; after render-thread startup it queues preview presentation and waits for render-thread completion | render thread or preview presenter consumes latest completed frame | +| `RenderEngine::CaptureOutputFrameRgbaTopDown(...)` | synchronous compatibility shim; after render-thread startup it queues screenshot readback and waits for render-thread completion | screenshot request becomes render-thread command | | `OpenGLVideoIOBridge::UploadInputFrame(...)` | calls render upload directly | push input frame into render queue/mailbox | | `OpenGLVideoIOBridge::RenderScheduledFrame(...)` | calls render output directly from backend path | request/consume render-produced output without callback-owned GL | @@ -133,8 +133,10 @@ Current implementation: - `RenderCommandQueue` exists as a pure C++ mailbox helper. - Preview present and screenshot capture requests use latest-value coalescing. +- Input upload requests use latest-value coalescing. During the compatibility phase the input frame memory is still drained immediately; a real render thread will need copied or otherwise owned frame storage. +- Output frame requests use FIFO semantics so scheduled output demand is not collapsed. - Render-local reset requests coalesce to the strongest pending reset scope. -- `RenderEngine` drains these commands synchronously as compatibility shims until a dedicated render thread is introduced. +- The synchronous compatibility shims submit queued work to the render thread and wait for completion once the render thread is running. Possible commands: @@ -234,14 +236,23 @@ Start with low-risk commands: - [x] preview present request - [x] screenshot request - [x] render-local reset requests +- [x] input upload request +- [x] output render request -Then move input upload and output render requests once the queue and wakeup behavior are proven. +The queue and wakeup behavior still need the dedicated render thread before the callbacks stop borrowing the GL context. ### Step 3. Start A Dedicated Render Thread Create the render thread and make it own context binding. -Transitional behavior may still allow synchronous request/response for output frames. The important change is that the caller waits for render-thread completion rather than taking the GL context itself. +- [x] create a dedicated render thread owned by `RenderEngine` +- [x] bind the existing GL context on the render thread for normal runtime work +- [x] stop the render thread before GL context destruction +- [x] keep transitional synchronous request/response for output frames +- [x] remove normal runtime dependence on the shared GL `CRITICAL_SECTION` +- [x] add timeout/failure behavior for render-thread requests + +Transitional behavior still allows synchronous request/response for output frames. Render-thread requests now fail fast if they cannot begin within the request timeout, and log over-budget tasks that have already started before waiting for safe completion. The important change is that the caller waits for render-thread completion rather than taking the GL context itself. ### Step 4. Move Input Upload To The Render Thread diff --git a/tests/RenderCommandQueueTests.cpp b/tests/RenderCommandQueueTests.cpp index 523d677..e3ff285 100644 --- a/tests/RenderCommandQueueTests.cpp +++ b/tests/RenderCommandQueueTests.cpp @@ -60,14 +60,72 @@ void TestRenderResetScopesCoalesceToStrongestRequest() Expect(queue.GetMetrics().depth == 0, "none reset request is ignored"); } +void TestInputUploadRequestUsesLatestValue() +{ + int firstPixel = 1; + int secondPixel = 2; + RenderCommandQueue queue; + + RenderInputUploadRequest firstRequest; + firstRequest.inputFrame.bytes = &firstPixel; + firstRequest.inputFrame.width = 1920; + firstRequest.videoState.captureTextureWidth = 1920; + queue.RequestInputUpload(firstRequest); + + RenderInputUploadRequest secondRequest; + secondRequest.inputFrame.bytes = &secondPixel; + secondRequest.inputFrame.width = 1280; + secondRequest.videoState.captureTextureWidth = 1280; + queue.RequestInputUpload(secondRequest); + + const RenderCommandQueueMetrics metrics = queue.GetMetrics(); + Expect(metrics.depth == 1, "input upload requests coalesce to one pending command"); + Expect(metrics.enqueuedCount == 1, "first input upload request is counted as enqueued"); + Expect(metrics.coalescedCount == 1, "second input upload request is counted as coalesced"); + + RenderInputUploadRequest request; + Expect(queue.TryTakeInputUpload(request), "input upload request can be consumed"); + Expect(request.inputFrame.bytes == &secondPixel, "latest input upload bytes pointer wins"); + Expect(request.inputFrame.width == 1280, "latest input upload frame wins"); + Expect(request.videoState.captureTextureWidth == 1280, "latest input upload state wins"); + Expect(!queue.TryTakeInputUpload(request), "input upload request is removed after consume"); +} + +void TestOutputFrameRequestsAreFifo() +{ + RenderCommandQueue queue; + RenderOutputFrameRequest firstRequest; + firstRequest.videoState.outputFrameSize.width = 1920; + firstRequest.completion.result = VideoIOCompletionResult::Completed; + queue.RequestOutputFrame(firstRequest); + + RenderOutputFrameRequest secondRequest; + secondRequest.videoState.outputFrameSize.width = 1280; + secondRequest.completion.result = VideoIOCompletionResult::Dropped; + queue.RequestOutputFrame(secondRequest); + + Expect(queue.GetMetrics().depth == 2, "output frame requests are queued independently"); + + RenderOutputFrameRequest request; + Expect(queue.TryTakeOutputFrame(request), "first output request can be consumed"); + Expect(request.videoState.outputFrameSize.width == 1920, "first output request is consumed first"); + Expect(request.completion.result == VideoIOCompletionResult::Completed, "first output completion is preserved"); + Expect(queue.TryTakeOutputFrame(request), "second output request can be consumed"); + Expect(request.videoState.outputFrameSize.width == 1280, "second output request is consumed second"); + Expect(request.completion.result == VideoIOCompletionResult::Dropped, "second output completion is preserved"); + Expect(!queue.TryTakeOutputFrame(request), "output queue is empty after consuming all requests"); +} + void TestIndependentCommandKindsShareDepth() { RenderCommandQueue queue; queue.RequestPreviewPresent({ 1, 2 }); queue.RequestScreenshotCapture({ 3, 4 }); + queue.RequestInputUpload({}); + queue.RequestOutputFrame({}); queue.RequestRenderReset(RenderCommandResetScope::TemporalHistoryOnly); - Expect(queue.GetMetrics().depth == 3, "independent command kinds each contribute to depth"); + Expect(queue.GetMetrics().depth == 5, "independent command kinds each contribute to depth"); } } @@ -76,6 +134,8 @@ int main() TestPreviewRequestUsesLatestValue(); TestScreenshotRequestUsesLatestValue(); TestRenderResetScopesCoalesceToStrongestRequest(); + TestInputUploadRequestUsesLatestValue(); + TestOutputFrameRequestsAreFifo(); TestIndependentCommandKindsShareDepth(); if (gFailures != 0)