diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp index a810947..c328467 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp @@ -23,10 +23,8 @@ #include OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) : - hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC), - mScreenshotRequested(false) + hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC) { - InitializeCriticalSection(&pMutex); mRuntimeStore = std::make_unique(); mRuntimeEventDispatcher = std::make_unique(); mRuntimeSnapshotProvider = std::make_unique(mRuntimeStore->GetRenderSnapshotBuilder(), *mRuntimeEventDispatcher); @@ -34,11 +32,10 @@ OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) : mRenderEngine = std::make_unique( *mRuntimeSnapshotProvider, mRuntimeStore->GetHealthTelemetry(), - pMutex, hGLDC, hGLRC, [this]() { renderEffect(); }, - [this]() { ProcessScreenshotRequest(); }, + []() {}, [this]() { paintGL(false); }); mVideoBackend = std::make_unique(*mRenderEngine, mRuntimeStore->GetHealthTelemetry(), *mRuntimeEventDispatcher); mShaderBuildQueue = std::make_unique(*mRuntimeSnapshotProvider, *mRuntimeEventDispatcher); @@ -61,8 +58,6 @@ OpenGLComposite::~OpenGLComposite() mShaderBuildQueue->Stop(); if (mVideoBackend) mVideoBackend->ReleaseResources(); - - DeleteCriticalSection(&pMutex); } bool OpenGLComposite::InitDeckLink() @@ -294,8 +289,50 @@ bool OpenGLComposite::ReloadShader(bool preserveFeedbackState) bool OpenGLComposite::RequestScreenshot(std::string& error) { - (void)error; - mScreenshotRequested.store(true); + if (!mRenderEngine || !mVideoBackend) + { + error = "The render engine is not ready."; + return false; + } + + const unsigned width = mVideoBackend->OutputFrameWidth(); + const unsigned height = mVideoBackend->OutputFrameHeight(); + if (width == 0 || height == 0) + { + error = "The output frame size is not available."; + return false; + } + + std::filesystem::path outputPath; + try + { + outputPath = BuildScreenshotPath(); + std::filesystem::create_directories(outputPath.parent_path()); + } + catch (const std::exception& exception) + { + error = exception.what(); + return false; + } + + if (!mRenderEngine->RequestScreenshotCapture( + width, + height, + [outputPath](unsigned captureWidth, unsigned captureHeight, std::vector topDownPixels) { + try + { + WritePngFileAsync(outputPath, captureWidth, captureHeight, std::move(topDownPixels)); + } + catch (const std::exception& exception) + { + OutputDebugStringA((std::string("Screenshot request failed: ") + exception.what() + "\n").c_str()); + } + })) + { + error = "Screenshot capture request failed."; + return false; + } + return true; } @@ -342,32 +379,6 @@ void OpenGLComposite::RenderFrame(const RenderFrameInput& frameInput) mRenderEngine->RenderPreparedFrame(frameState); } -void OpenGLComposite::ProcessScreenshotRequest() -{ - if (!mScreenshotRequested.exchange(false)) - return; - - const unsigned width = mVideoBackend ? mVideoBackend->OutputFrameWidth() : 0; - const unsigned height = mVideoBackend ? mVideoBackend->OutputFrameHeight() : 0; - if (width == 0 || height == 0) - return; - - std::vector topDownPixels; - if (!mRenderEngine->CaptureOutputFrameRgbaTopDown(width, height, topDownPixels)) - return; - - try - { - const std::filesystem::path outputPath = BuildScreenshotPath(); - std::filesystem::create_directories(outputPath.parent_path()); - WritePngFileAsync(outputPath, width, height, std::move(topDownPixels)); - } - catch (const std::exception& exception) - { - OutputDebugStringA((std::string("Screenshot request failed: ") + exception.what() + "\n").c_str()); - } -} - std::filesystem::path OpenGLComposite::BuildScreenshotPath() const { const std::filesystem::path root = mRuntimeStore && !mRuntimeStore->GetRuntimeDataRoot().empty() diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h index 474fba5..ab35fda 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h +++ b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h @@ -15,7 +15,6 @@ #include "RenderFrameState.h" #include -#include #include #include #include @@ -73,7 +72,6 @@ private: HWND hGLWnd; HDC hGLDC; HGLRC hGLRC; - CRITICAL_SECTION pMutex; std::unique_ptr mRuntimeStore; std::unique_ptr mRuntimeCoordinator; @@ -84,13 +82,11 @@ private: std::unique_ptr mRuntimeServices; std::unique_ptr mRuntimeUpdateController; std::unique_ptr mVideoBackend; - std::atomic mScreenshotRequested; bool InitOpenGLState(); void renderEffect(); RenderFrameInput BuildRenderFrameInput() const; void RenderFrame(const RenderFrameInput& frameInput); - void ProcessScreenshotRequest(); std::filesystem::path BuildScreenshotPath() const; }; diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h b/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h index bc55646..14af141 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h +++ b/apps/LoopThroughWithOpenGLCompositing/gl/RenderCommandQueue.h @@ -6,6 +6,7 @@ #include #include #include +#include enum class RenderCommandResetScope { @@ -31,6 +32,7 @@ struct RenderInputUploadRequest { VideoIOFrame inputFrame; VideoIOState videoState; + std::vector ownedBytes; }; struct RenderOutputFrameRequest diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp b/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp index af7acb1..ba22b49 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp @@ -3,11 +3,11 @@ #include #include +#include RenderEngine::RenderEngine( RuntimeSnapshotProvider& runtimeSnapshotProvider, HealthTelemetry& healthTelemetry, - CRITICAL_SECTION& mutex, HDC hdc, HGLRC hglrc, RenderEffectCallback renderEffect, @@ -17,7 +17,6 @@ RenderEngine::RenderEngine( mRenderPass(mRenderer), mRenderPipeline(mRenderer, runtimeSnapshotProvider, healthTelemetry, std::move(renderEffect), std::move(screenshotReady), std::move(previewPaint)), mShaderPrograms(mRenderer, runtimeSnapshotProvider), - mMutex(mutex), mHdc(hdc), mHglrc(hglrc), mFrameStateResolver(runtimeSnapshotProvider) @@ -136,6 +135,24 @@ void RenderEngine::ReportRenderThreadRequestFailure(const char* operationName, c OutputDebugStringA(message.str().c_str()); } +bool RenderEngine::IsRenderThreadAccessExpected() const +{ + return !mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId; +} + +void RenderEngine::ReportWrongThreadRenderAccess(const char* operationName) const +{ + if (IsRenderThreadAccessExpected()) + return; + + std::ostringstream message; + message << "Wrong-thread render access detected"; + if (operationName && operationName[0] != '\0') + message << " [" << operationName << "]"; + message << ".\n"; + OutputDebugStringA(message.str().c_str()); +} + bool RenderEngine::CompileDecodeShader(int errorMessageSize, char* errorMessage) { return InvokeOnRenderThread([this, errorMessageSize, errorMessage]() { @@ -241,11 +258,13 @@ void RenderEngine::ApplyRuntimeCoordinatorRenderReset(RuntimeCoordinatorRenderRe void RenderEngine::ResetTemporalHistoryStateOnRenderThread() { + ReportWrongThreadRenderAccess("reset-temporal-history"); mShaderPrograms.ResetTemporalHistoryState(); } void RenderEngine::ResetShaderFeedbackStateOnRenderThread() { + ReportWrongThreadRenderAccess("reset-shader-feedback"); mShaderPrograms.ResetShaderFeedbackState(); } @@ -276,6 +295,124 @@ void RenderEngine::ProcessRenderResetCommandsOnRenderThread() ApplyRenderResetOnRenderThread(resetScope); } +void RenderEngine::EnqueuePreviewPresentWake() +{ + if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId) + return; + + bool shouldNotify = false; + { + std::lock_guard lock(mRenderThreadMutex); + if (!mRenderThreadStopping && !mPreviewPresentWakePending) + { + mPreviewPresentWakePending = true; + mRenderThreadTasks.push([this]() { + { + std::lock_guard lock(mRenderThreadMutex); + mPreviewPresentWakePending = false; + } + ProcessPreviewPresentCommandsOnRenderThread(); + }); + shouldNotify = true; + } + } + + if (shouldNotify) + mRenderThreadCondition.notify_one(); +} + +void RenderEngine::ProcessPreviewPresentCommandsOnRenderThread() +{ + RenderPreviewPresentRequest request; + if (mRenderCommandQueue.TryTakePreviewPresent(request)) + PresentPreviewOnRenderThread(request.outputFrameWidth, request.outputFrameHeight); +} + +void RenderEngine::EnqueueInputUploadWake() +{ + if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId) + return; + + bool shouldNotify = false; + { + std::lock_guard lock(mRenderThreadMutex); + if (!mRenderThreadStopping && !mInputUploadWakePending) + { + mInputUploadWakePending = true; + mRenderThreadTasks.push([this]() { + { + std::lock_guard lock(mRenderThreadMutex); + mInputUploadWakePending = false; + } + ProcessInputUploadCommandsOnRenderThread(); + }); + shouldNotify = true; + } + } + + if (shouldNotify) + mRenderThreadCondition.notify_one(); +} + +void RenderEngine::ProcessInputUploadCommandsOnRenderThread() +{ + RenderInputUploadRequest request; + while (mRenderCommandQueue.TryTakeInputUpload(request)) + { + if (request.ownedBytes.empty()) + continue; + + request.inputFrame.bytes = request.ownedBytes.data(); + UploadInputFrameOnRenderThread(request.inputFrame, request.videoState); + } +} + +void RenderEngine::EnqueueScreenshotCaptureWake() +{ + if (!mRenderThreadRunning || GetCurrentThreadId() == mRenderThreadId) + return; + + bool shouldNotify = false; + { + std::lock_guard lock(mRenderThreadMutex); + if (!mRenderThreadStopping && !mScreenshotCaptureWakePending) + { + mScreenshotCaptureWakePending = true; + mRenderThreadTasks.push([this]() { + { + std::lock_guard lock(mRenderThreadMutex); + mScreenshotCaptureWakePending = false; + } + ProcessScreenshotCaptureCommandsOnRenderThread(); + }); + shouldNotify = true; + } + } + + if (shouldNotify) + mRenderThreadCondition.notify_one(); +} + +void RenderEngine::ProcessScreenshotCaptureCommandsOnRenderThread() +{ + RenderScreenshotCaptureRequest request; + ScreenshotCaptureCallback completion; + { + std::lock_guard lock(mRenderThreadMutex); + completion = mScreenshotCaptureCompletion; + } + + while (mRenderCommandQueue.TryTakeScreenshotCapture(request)) + { + if (!completion) + continue; + + std::vector topDownPixels; + if (CaptureOutputFrameRgbaTopDownOnRenderThread(request.width, request.height, topDownPixels)) + completion(request.width, request.height, std::move(topDownPixels)); + } +} + void RenderEngine::ClearOscOverlayState() { mRuntimeLiveState.Clear(); @@ -323,32 +460,62 @@ bool RenderEngine::TryPresentPreview(bool force, unsigned previewFps, unsigned o if (mRenderThreadRunning) { - return TryInvokeOnRenderThread("preview-present", [this, outputFrameWidth, outputFrameHeight]() { - mRenderCommandQueue.RequestPreviewPresent({ outputFrameWidth, outputFrameHeight }); - RenderPreviewPresentRequest request; - return mRenderCommandQueue.TryTakePreviewPresent(request) && - PresentPreviewOnRenderThread(request.outputFrameWidth, request.outputFrameHeight); - }); + mRenderCommandQueue.RequestPreviewPresent({ outputFrameWidth, outputFrameHeight }); + if (GetCurrentThreadId() == mRenderThreadId) + ProcessPreviewPresentCommandsOnRenderThread(); + else + EnqueuePreviewPresentWake(); + return true; } - if (!TryEnterCriticalSection(&mMutex)) - return false; - - mRenderCommandQueue.RequestPreviewPresent({ outputFrameWidth, outputFrameHeight }); - RenderPreviewPresentRequest request; - const bool presented = mRenderCommandQueue.TryTakePreviewPresent(request) && - PresentPreviewOnRenderThread(request.outputFrameWidth, request.outputFrameHeight); - LeaveCriticalSection(&mMutex); - return presented; + ReportRenderThreadRequestFailure("preview-present", "render thread is not running"); + return false; } bool RenderEngine::PresentPreviewOnRenderThread(unsigned outputFrameWidth, unsigned outputFrameHeight) { + ReportWrongThreadRenderAccess("preview-present"); mRenderer.PresentToWindow(mHdc, outputFrameWidth, outputFrameHeight); mLastPreviewPresentTime = std::chrono::steady_clock::now(); return true; } +bool RenderEngine::RequestScreenshotCapture(unsigned width, unsigned height, ScreenshotCaptureCallback completion) +{ + if (width == 0 || height == 0 || !completion) + return false; + if (!mRenderThreadRunning) + return false; + + { + std::lock_guard lock(mRenderThreadMutex); + mScreenshotCaptureCompletion = std::move(completion); + } + mRenderCommandQueue.RequestScreenshotCapture({ width, height }); + EnqueueScreenshotCaptureWake(); + return true; +} + +bool RenderEngine::QueueInputFrame(const VideoIOFrame& inputFrame, const VideoIOState& videoState) +{ + if (inputFrame.hasNoInputSource || inputFrame.bytes == nullptr) + return true; + if (inputFrame.rowBytes <= 0 || inputFrame.height == 0) + return false; + + const std::size_t byteCount = static_cast(inputFrame.rowBytes) * inputFrame.height; + RenderInputUploadRequest request; + request.inputFrame = inputFrame; + request.videoState = videoState; + request.ownedBytes.resize(byteCount); + std::memcpy(request.ownedBytes.data(), inputFrame.bytes, byteCount); + request.inputFrame.bytes = nullptr; + + mRenderCommandQueue.RequestInputUpload(request); + EnqueueInputUploadWake(); + return true; +} + bool RenderEngine::TryUploadInputFrame(const VideoIOFrame& inputFrame, const VideoIOState& videoState) { if (inputFrame.hasNoInputSource || inputFrame.bytes == nullptr) @@ -364,21 +531,13 @@ bool RenderEngine::TryUploadInputFrame(const VideoIOFrame& inputFrame, const Vid }); } - if (!TryEnterCriticalSection(&mMutex)) - return false; - - wglMakeCurrent(mHdc, mHglrc); - mRenderCommandQueue.RequestInputUpload({ inputFrame, videoState }); - RenderInputUploadRequest request; - const bool uploaded = mRenderCommandQueue.TryTakeInputUpload(request) && - UploadInputFrameOnRenderThread(request.inputFrame, request.videoState); - wglMakeCurrent(NULL, NULL); - LeaveCriticalSection(&mMutex); - return uploaded; + ReportRenderThreadRequestFailure("input-upload", "render thread is not running"); + return false; } bool RenderEngine::UploadInputFrameOnRenderThread(const VideoIOFrame& inputFrame, const VideoIOState& videoState) { + ReportWrongThreadRenderAccess("input-upload"); const long textureSize = inputFrame.rowBytes * static_cast(inputFrame.height); glPixelStorei(GL_UNPACK_ALIGNMENT, 4); glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); @@ -395,7 +554,7 @@ bool RenderEngine::UploadInputFrameOnRenderThread(const VideoIOFrame& inputFrame return true; } -bool RenderEngine::RenderOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame) +bool RenderEngine::RequestOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame) { if (mRenderThreadRunning) { @@ -407,20 +566,20 @@ bool RenderEngine::RenderOutputFrame(const RenderPipelineFrameContext& context, }); } - EnterCriticalSection(&mMutex); - wglMakeCurrent(mHdc, mHglrc); - mRenderCommandQueue.RequestOutputFrame({ context.videoState, context.completion }); - RenderOutputFrameRequest request; - const bool rendered = mRenderCommandQueue.TryTakeOutputFrame(request) && - RenderOutputFrameOnRenderThread({ request.videoState, request.completion }, outputFrame); - wglMakeCurrent(NULL, NULL); - LeaveCriticalSection(&mMutex); - return rendered; + ReportRenderThreadRequestFailure("output-render", "render thread is not running"); + return false; +} + +bool RenderEngine::RenderOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame) +{ + return RequestOutputFrame(context, outputFrame); } bool RenderEngine::RenderOutputFrameOnRenderThread(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame) { + ReportWrongThreadRenderAccess("output-render"); ProcessRenderResetCommandsOnRenderThread(); + ProcessInputUploadCommandsOnRenderThread(); return mRenderPipeline.RenderFrame(context, outputFrame); } @@ -466,6 +625,7 @@ void RenderEngine::RenderLayerStack( VideoIOPixelFormat inputPixelFormat, unsigned historyCap) { + ReportWrongThreadRenderAccess("render-layer-stack"); mRenderPass.Render( hasInputSource, layerStates, @@ -484,6 +644,7 @@ void RenderEngine::RenderLayerStack( bool RenderEngine::ReadOutputFrameRgbaOnRenderThread(unsigned width, unsigned height, std::vector& bottomUpPixels) { + ReportWrongThreadRenderAccess("read-output-frame-rgba"); if (width == 0 || height == 0) return false; @@ -510,15 +671,8 @@ bool RenderEngine::CaptureOutputFrameRgbaTopDown(unsigned width, unsigned height }); } - EnterCriticalSection(&mMutex); - wglMakeCurrent(mHdc, mHglrc); - mRenderCommandQueue.RequestScreenshotCapture({ width, height }); - RenderScreenshotCaptureRequest request; - const bool captured = mRenderCommandQueue.TryTakeScreenshotCapture(request) && - CaptureOutputFrameRgbaTopDownOnRenderThread(request.width, request.height, topDownPixels); - wglMakeCurrent(NULL, NULL); - LeaveCriticalSection(&mMutex); - return captured; + ReportRenderThreadRequestFailure("screenshot-capture", "render thread is not running"); + return false; } bool RenderEngine::CaptureOutputFrameRgbaTopDownOnRenderThread(unsigned width, unsigned height, std::vector& topDownPixels) diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h b/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h index 47048fd..319a452 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h +++ b/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.h @@ -33,6 +33,7 @@ class RenderEngine public: using RenderEffectCallback = std::function; using ScreenshotCallback = std::function; + using ScreenshotCaptureCallback = std::function)>; using PreviewPaintCallback = std::function; struct OscOverlayUpdate @@ -61,7 +62,6 @@ public: RenderEngine( RuntimeSnapshotProvider& runtimeSnapshotProvider, HealthTelemetry& healthTelemetry, - CRITICAL_SECTION& mutex, HDC hdc, HGLRC hglrc, RenderEffectCallback renderEffect, @@ -100,7 +100,10 @@ public: const std::vector& completedCommits); void ResizeView(int width, int height); bool TryPresentPreview(bool force, unsigned previewFps, unsigned outputFrameWidth, unsigned outputFrameHeight); + bool RequestScreenshotCapture(unsigned width, unsigned height, ScreenshotCaptureCallback completion); + bool QueueInputFrame(const VideoIOFrame& inputFrame, const VideoIOState& videoState); bool TryUploadInputFrame(const VideoIOFrame& inputFrame, const VideoIOState& videoState); + bool RequestOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame); bool RenderOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame); bool ResolveRenderFrameState( const RenderFrameInput& input, @@ -178,6 +181,8 @@ private: void RenderThreadMain(std::promise ready); void ReportRenderThreadRequestFailure(const char* operationName, const char* reason); + bool IsRenderThreadAccessExpected() const; + void ReportWrongThreadRenderAccess(const char* operationName) const; bool CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage); void RenderLayerStack( bool hasInputSource, @@ -191,6 +196,12 @@ private: void ResetShaderFeedbackStateOnRenderThread(); void ApplyRenderResetOnRenderThread(RenderCommandResetScope resetScope); void ProcessRenderResetCommandsOnRenderThread(); + void EnqueuePreviewPresentWake(); + void ProcessPreviewPresentCommandsOnRenderThread(); + void EnqueueInputUploadWake(); + void ProcessInputUploadCommandsOnRenderThread(); + void EnqueueScreenshotCaptureWake(); + void ProcessScreenshotCaptureCommandsOnRenderThread(); bool PresentPreviewOnRenderThread(unsigned outputFrameWidth, unsigned outputFrameHeight); bool UploadInputFrameOnRenderThread(const VideoIOFrame& inputFrame, const VideoIOState& videoState); bool RenderOutputFrameOnRenderThread(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame); @@ -201,7 +212,6 @@ private: OpenGLRenderPass mRenderPass; OpenGLRenderPipeline mRenderPipeline; OpenGLShaderPrograms mShaderPrograms; - CRITICAL_SECTION& mMutex; HDC mHdc; HGLRC mHglrc; @@ -216,5 +226,9 @@ private: std::queue> mRenderThreadTasks; std::atomic mRenderThreadRunning = false; bool mRenderThreadStopping = false; + bool mPreviewPresentWakePending = false; + bool mInputUploadWakePending = false; + bool mScreenshotCaptureWakePending = false; + ScreenshotCaptureCallback mScreenshotCaptureCompletion; bool mResourcesDestroyed = false; }; diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.cpp b/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.cpp index ccba9a2..50b5ef9 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.cpp @@ -12,14 +12,14 @@ void OpenGLVideoIOBridge::UploadInputFrame(const VideoIOFrame& inputFrame, const if (inputFrame.hasNoInputSource || inputFrame.bytes == nullptr) return; // don't transfer texture when there's no input - mRenderEngine.TryUploadInputFrame(inputFrame, state); + mRenderEngine.QueueInputFrame(inputFrame, state); } -void OpenGLVideoIOBridge::RenderScheduledFrame(const VideoIOState& state, const VideoIOCompletion& completion, VideoIOOutputFrame& outputFrame) +bool OpenGLVideoIOBridge::RenderScheduledFrame(const VideoIOState& state, const VideoIOCompletion& completion, VideoIOOutputFrame& outputFrame) { RenderPipelineFrameContext frameContext; frameContext.videoState = state; frameContext.completion = completion; - mRenderEngine.RenderOutputFrame(frameContext, outputFrame); + return mRenderEngine.RequestOutputFrame(frameContext, outputFrame); } diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.h b/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.h index 94525c9..61b0523 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.h +++ b/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.h @@ -10,7 +10,7 @@ public: explicit OpenGLVideoIOBridge(RenderEngine& renderEngine); void UploadInputFrame(const VideoIOFrame& inputFrame, const VideoIOState& state); - void RenderScheduledFrame(const VideoIOState& state, const VideoIOCompletion& completion, VideoIOOutputFrame& outputFrame); + bool RenderScheduledFrame(const VideoIOState& state, const VideoIOCompletion& completion, VideoIOOutputFrame& outputFrame); private: RenderEngine& mRenderEngine; diff --git a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp index 7a75ab5..953d0cc 100644 --- a/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp @@ -232,11 +232,17 @@ void VideoBackend::HandleOutputFrameCompletion(const VideoIOCompletion& completi return; const VideoIOState& state = mVideoIODevice->State(); + bool rendered = true; if (mBridge) - mBridge->RenderScheduledFrame(state, completion, outputFrame); + rendered = mBridge->RenderScheduledFrame(state, completion, outputFrame); EndOutputFrame(outputFrame); AccountForCompletionResult(completion.result); + if (!rendered) + { + PublishBackendStateChanged("output-render-failed", "Output frame render request failed; skipping schedule for this frame."); + return; + } // Schedule the next frame after render work is complete so device-side // bookkeeping stays with the backend seam and the bridge stays render-only. diff --git a/docs/PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md b/docs/PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md index 4258134..823c1b0 100644 --- a/docs/PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md +++ b/docs/PHASE_4_RENDER_THREAD_OWNERSHIP_DESIGN.md @@ -7,16 +7,16 @@ Phase 1 named the subsystems. Phase 2 added the typed event substrate. Phase 3 m ## Status - Phase 4 design package: proposed. -- Phase 4 implementation: Step 3 started. The existing synchronous `RenderEngine` entrypoints delegate their GL bodies to named `...OnRenderThread(...)` helpers, preview/screenshot/render-reset/input-upload/output-render requests pass through a small `RenderCommandQueue` compatibility mailbox, and `RenderEngine` now starts a dedicated render thread for normal runtime GL work. +- Phase 4 implementation: Step 7 started. The existing synchronous `RenderEngine` entrypoints delegate their GL bodies to named `...OnRenderThread(...)` helpers, preview/screenshot/render-reset/input-upload/output-render requests pass through a small `RenderCommandQueue` compatibility mailbox, and `RenderEngine` now starts a dedicated render thread for normal runtime GL work. - Current alignment: the repo has a named frame-state contract and cleaner render-state preparation. Normal runtime GL work is routed through the render thread after startup, while startup initialization still runs before the render thread is started. Current GL ownership footholds: -- `RenderEngine` owns GL resources, a dedicated render thread, the current synchronous compatibility shims, a small render command mailbox, and named render-thread helper methods. +- `RenderEngine` owns GL resources, a dedicated render thread, the current synchronous compatibility shims, a small render command mailbox, named render-thread helper methods, and wrong-thread diagnostics for those helpers. - `RenderFrameInput` / `RenderFrameState` provide the frame-state contract that a render thread can consume. - `RenderFrameStateResolver` prepares the render-facing layer state before drawing. -- `OpenGLVideoIOBridge` still calls `RenderEngine::TryUploadInputFrame(...)` from the input path and `RenderEngine::RenderOutputFrame(...)` from the output path. -- `OpenGLComposite::paintGL(...)`, screenshot capture, input upload, and output rendering still call synchronous `RenderEngine` methods, but those methods now invoke render-thread work once `OpenGLComposite::Start()` has started the render thread. +- `OpenGLVideoIOBridge` calls `RenderEngine::QueueInputFrame(...)` from the input path and `RenderEngine::RequestOutputFrame(...)` from the output path. +- `OpenGLComposite::paintGL(...)`, screenshot capture, input upload, and output rendering enter render work through explicit `RenderEngine` requests. After `OpenGLComposite::Start()` starts the render thread, those requests do not bind the GL context on the caller thread. ## Why Phase 4 Exists @@ -65,11 +65,11 @@ The current code paths that matter most are: | Entry point | Current behavior | Phase 4 direction | | --- | --- | --- | | `RenderEngine::TryUploadInputFrame(...)` | synchronous compatibility shim; after render-thread startup it queues input upload work and waits for render-thread completion | enqueue latest input frame; render thread uploads without callback-owned GL | -| `RenderEngine::RenderOutputFrame(...)` | synchronous compatibility shim; after render-thread startup it queues output render work and waits for render-thread completion | render thread executes output frame production | -| `RenderEngine::TryPresentPreview(...)` | synchronous compatibility shim; after render-thread startup it queues preview presentation and waits for render-thread completion | render thread or preview presenter consumes latest completed frame | +| `RenderEngine::RequestOutputFrame(...)` | synchronous output request; after render-thread startup it queues output render work and waits for render-thread completion with timeout/failure reporting | render thread executes output frame production | +| `RenderEngine::TryPresentPreview(...)` | best-effort compatibility shim; after render-thread startup non-render callers queue preview presentation and return | render thread or preview presenter consumes latest completed frame | | `RenderEngine::CaptureOutputFrameRgbaTopDown(...)` | synchronous compatibility shim; after render-thread startup it queues screenshot readback and waits for render-thread completion | screenshot request becomes render-thread command | -| `OpenGLVideoIOBridge::UploadInputFrame(...)` | calls render upload directly | push input frame into render queue/mailbox | -| `OpenGLVideoIOBridge::RenderScheduledFrame(...)` | calls render output directly from backend path | request/consume render-produced output without callback-owned GL | +| `OpenGLVideoIOBridge::UploadInputFrame(...)` | copies the latest input frame into the render mailbox and returns without waiting for GL | render thread uploads the latest queued input frame | +| `OpenGLVideoIOBridge::RenderScheduledFrame(...)` | requests render-thread output production and reports success/failure to the backend | consume render-produced output without callback-owned GL | ## Target Ownership Model @@ -260,10 +260,12 @@ Change `OpenGLVideoIOBridge::UploadInputFrame(...)` so it enqueues or replaces t Policy targets: -- bounded memory -- latest-frame wins under load -- input upload skip count is observable -- input callback never waits for GL +- [x] bounded memory +- [x] latest-frame wins under load +- [x] input upload skip count is observable through render command coalescing metrics +- [x] input callback never waits for GL + +Current implementation: `OpenGLVideoIOBridge::UploadInputFrame(...)` calls `RenderEngine::QueueInputFrame(...)`, which copies the input bytes into the latest-value render mailbox and schedules one bounded render-thread wakeup to upload the newest pending frame. ### Step 5. Move Output Rendering To The Render Thread @@ -271,32 +273,38 @@ Change `OpenGLVideoIOBridge::RenderScheduledFrame(...)` so it requests render-th Transitional option: -- synchronous request/response through the render thread +- [x] synchronous request/response through the render thread Better follow-up: - render ahead into a bounded output queue and let backend callbacks consume ready frames +Current implementation: `OpenGLVideoIOBridge::RenderScheduledFrame(...)` calls `RenderEngine::RequestOutputFrame(...)` and returns whether the render-thread request produced an output frame. `VideoBackend` skips scheduling that frame when render production fails or times out. + ### Step 6. Decouple Preview And Screenshot Requests Preview should become best-effort: -- request preview presentation from the render thread -- skip when render is busy or output deadline pressure is high -- record preview skips +- [x] request preview presentation from the render thread +- [x] skip/coalesce when render is busy or output deadline pressure is high +- [x] record preview skips through render command coalescing metrics Screenshot should become: -- queued render-thread capture request -- async disk write remains outside render thread +- [x] queued render-thread capture request +- [x] async disk write remains outside render thread + +Current implementation: `OpenGLComposite::RequestScreenshot(...)` builds the output path, queues `RenderEngine::RequestScreenshotCapture(...)`, and the render thread captures pixels before handing them to the existing async PNG writer. Preview presentation is a latest-value best-effort render command; non-render callers enqueue and return, while render-thread callers drain the latest preview command inline. ### Step 7. Remove Shared GL Lock From Normal Paths Once all GL entrypoints are render-thread-owned: -- remove normal dependence on `pMutex` for render correctness -- keep assertions or diagnostics that detect wrong-thread GL calls -- leave only lifecycle synchronization where needed +- [x] remove normal dependence on `pMutex` for render correctness +- [x] keep diagnostics that detect wrong-thread render-thread helper calls +- [x] leave only lifecycle context binding where needed + +Current implementation: `OpenGLComposite` no longer owns or passes a shared `CRITICAL_SECTION`, and `RenderEngine` no longer has caller-thread GL fallback paths for preview, input upload, output render, or screenshot capture. Runtime callers must go through the render thread; pre-start direct GL fallback is limited to startup initialization while the app explicitly owns the context. ## Testing Strategy @@ -366,12 +374,12 @@ Preview can remain a hidden budget consumer if it stays in the output frame path Phase 4 can be considered complete once the project can say: -- [ ] one render thread owns the GL context during normal operation -- [ ] input callbacks do not bind GL or wait on GL upload -- [ ] output callbacks do not bind GL directly -- [ ] preview and screenshot requests enter render through explicit render-thread requests +- [x] one render thread owns the GL context during normal operation +- [x] input callbacks do not bind GL or wait on GL upload +- [x] output callbacks do not bind GL directly +- [x] preview and screenshot requests enter render through explicit render-thread requests - [ ] `RenderFrameInput` / `RenderFrameState` remain the frame-state contract -- [ ] normal frame production no longer depends on a shared GL `CRITICAL_SECTION` +- [x] normal frame production no longer depends on a shared GL `CRITICAL_SECTION` - [ ] render-thread queue/mailbox behavior has non-GL tests - [ ] shutdown order is explicit and tested or manually verified diff --git a/tests/RenderCommandQueueTests.cpp b/tests/RenderCommandQueueTests.cpp index e3ff285..0efafdb 100644 --- a/tests/RenderCommandQueueTests.cpp +++ b/tests/RenderCommandQueueTests.cpp @@ -70,12 +70,14 @@ void TestInputUploadRequestUsesLatestValue() firstRequest.inputFrame.bytes = &firstPixel; firstRequest.inputFrame.width = 1920; firstRequest.videoState.captureTextureWidth = 1920; + firstRequest.ownedBytes = { 1, 2, 3, 4 }; queue.RequestInputUpload(firstRequest); RenderInputUploadRequest secondRequest; secondRequest.inputFrame.bytes = &secondPixel; secondRequest.inputFrame.width = 1280; secondRequest.videoState.captureTextureWidth = 1280; + secondRequest.ownedBytes = { 5, 6 }; queue.RequestInputUpload(secondRequest); const RenderCommandQueueMetrics metrics = queue.GetMetrics(); @@ -88,6 +90,7 @@ void TestInputUploadRequestUsesLatestValue() Expect(request.inputFrame.bytes == &secondPixel, "latest input upload bytes pointer wins"); Expect(request.inputFrame.width == 1280, "latest input upload frame wins"); Expect(request.videoState.captureTextureWidth == 1280, "latest input upload state wins"); + Expect(request.ownedBytes.size() == 2 && request.ownedBytes[0] == 5 && request.ownedBytes[1] == 6, "latest input upload owned bytes win"); Expect(!queue.TryTakeInputUpload(request), "input upload request is removed after consume"); }