diff --git a/CMakeLists.txt b/CMakeLists.txt index e780b24..3ffb3b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -361,11 +361,12 @@ set(RENDER_CADENCE_APP_SOURCES "${RENDER_CADENCE_APP_DIR}/telemetry/CadenceTelemetryJson.h" "${RENDER_CADENCE_APP_DIR}/telemetry/CadenceTelemetry.h" "${RENDER_CADENCE_APP_DIR}/telemetry/TelemetryHealthMonitor.h" + "${RENDER_CADENCE_APP_DIR}/video/DeckLinkInput.cpp" + "${RENDER_CADENCE_APP_DIR}/video/DeckLinkInput.h" + "${RENDER_CADENCE_APP_DIR}/video/DeckLinkInputThread.h" "${RENDER_CADENCE_APP_DIR}/video/DeckLinkOutput.cpp" "${RENDER_CADENCE_APP_DIR}/video/DeckLinkOutput.h" "${RENDER_CADENCE_APP_DIR}/video/DeckLinkOutputThread.h" - "${RENDER_CADENCE_APP_DIR}/video/SyntheticInputProducer.cpp" - "${RENDER_CADENCE_APP_DIR}/video/SyntheticInputProducer.h" ) add_executable(RenderCadenceCompositor ${RENDER_CADENCE_APP_SOURCES}) diff --git a/apps/RenderCadenceCompositor/README.md b/apps/RenderCadenceCompositor/README.md index f56e710..9534b17 100644 --- a/apps/RenderCadenceCompositor/README.md +++ b/apps/RenderCadenceCompositor/README.md @@ -40,11 +40,11 @@ Startup warms up real rendered frames before DeckLink scheduled playback starts. Included now: - output-only DeckLink +- optional DeckLink input edge with BGRA8 capture or UYVY8-to-BGRA8 CPU conversion - non-blocking startup when DeckLink output is unavailable - hidden render-thread-owned OpenGL context - simple smooth-motion renderer - BGRA8-only output -- synthetic BGRA8 input producer - non-blocking latest-frame input mailbox - render-thread-owned input texture upload - async PBO readback @@ -74,7 +74,6 @@ Included now: Intentionally not included yet: -- real DeckLink input capture - input format conversion - temporal/history/feedback shader storage - texture/LUT asset upload @@ -118,12 +117,12 @@ This tracks parity with `apps/LoopThroughWithOpenGLCompositing`. - [x] Runtime parameter updates from HTTP controls - [x] Layer reorder/bypass/set-shader/update-parameter/reset-parameter HTTP controls - [x] Trigger parameter pulse count/time for latest trigger events -- [x] Synthetic BGRA8 frame input producer +- [x] Optional DeckLink input capture +- [x] UYVY8-to-BGRA8 input conversion - [x] Latest-frame CPU input mailbox - [x] Render-owned input texture upload - [x] Runtime shaders receive input through `gVideoInput` -- [ ] DeckLink input capture -- [ ] Live DeckLink input bound to `gVideoInput` +- [x] Live DeckLink input bound to `gVideoInput` - [ ] Input format conversion/scaling - [ ] Temporal history buffers - [ ] Feedback buffers @@ -240,6 +239,20 @@ If DeckLink discovery or output setup fails, the app logs a warning and continue `/api/state` reports the output status in `videoIO.statusMessage`. +## Optional DeckLink Input + +DeckLink input is an optional edge service in this app. + +Startup order is: + +1. create `InputFrameMailbox` +2. try to attach DeckLink input for the configured input mode +3. prefer BGRA8 capture, otherwise accept UYVY8 capture and convert to BGRA8 before the mailbox +4. start `DeckLinkInputThread` +5. leave input absent if discovery, setup, format support, or stream startup fails + +`DeckLinkInput` and `DeckLinkInputThread` are deliberately narrow. They capture BGRA8 frames directly or convert UYVY8 frames to BGRA8 before submitting to `InputFrameMailbox`; they do not call GL, render, preview, screenshot, shader, or output scheduling code. Unsupported input modes or formats outside BGRA8/UYVY8 are reported explicitly and treated as an unavailable edge rather than silently converted. + The app samples telemetry once per second. Normal cadence samples are available through `GET /api/state` and are not printed to the console. The telemetry monitor only logs health events: @@ -248,6 +261,24 @@ Normal cadence samples are available through `GET /api/state` and are not printe - warning when schedule failures increase - error when the app/DeckLink output buffer is starved +Input telemetry: + +- `inputFramesReceived`: frames accepted into `InputFrameMailbox` +- `inputFramesDropped`: ready input frames dropped or missed because the mailbox was full +- `inputLatestAgeMs`: age of the newest submitted input frame +- `inputUploadMs`: render-thread GL upload time for the latest uploaded input frame +- `inputFormatSupported`: whether the latest frame reaching the render upload path was BGRA8-compatible +- `inputSignalPresent`: whether any input frame has reached the mailbox +- `inputCaptureFps`: DeckLink input callback capture rate +- `inputConvertMs`: input-edge UYVY8-to-BGRA8 conversion time for the latest converted frame +- `inputSubmitMs`: time spent submitting the latest captured/converted input frame to `InputFrameMailbox` +- `inputCaptureFormat`: selected DeckLink input format (`BGRA8`, `UYVY8`, or `none`) +- `inputNoSignalFrames`: DeckLink callbacks reporting no input source +- `inputUnsupportedFrames`: input frames rejected before mailbox submission +- `inputSubmitMisses`: input frames that could not be submitted to the mailbox + +Runtime shaders continue rendering when input is missing. If no mailbox frame has been uploaded yet, shader samplers use the runtime fallback source texture; once DeckLink input is flowing, shaders such as CRT and trigger-ripple sample the real/latest input through `gVideoInput`. + Healthy first-run signs: - visible DeckLink output is smooth @@ -282,10 +313,18 @@ Current runtime shader support is deliberately limited to stateless full-frame p - HTTP controls can update runtime parameter values without rebuilding GL programs when the shader program is unchanged - trigger parameters are treated as latest-pulse controls: each press increments the trigger count and records the current runtime time - repeated trigger history is not stored yet, so effects such as `trigger-ripple` restart from the latest trigger rather than accumulating overlapping ripples -- the first layer receives a small fallback source texture until DeckLink input is added -- the first layer receives the latest synthetic input texture through both `gVideoInput` and `gLayerInput` when input frames are available +- the first layer receives a small fallback source texture until DeckLink input is available +- the first layer receives the latest DeckLink input texture through both `gVideoInput` and `gLayerInput` when input frames are available - stacked layers receive the original input through `gVideoInput` and the previous ready layer output through `gLayerInput` +Shader source semantics: + +- `gVideoInput` means the raw latest input frame for every layer. +- `gLayerInput` means the previous layer output. +- the first layer may receive `gLayerInput = gVideoInput`. +- later layers receive `gVideoInput = original input` and `gLayerInput = previous layer`. +- named intermediate pass inputs inside a multipass layer are still routed through the selected pass-source slot; layer stacking should use `gLayerInput`. + The `/api/state` shader list uses the same support rules as runtime shader compilation and reports only packages this app can run today. Unsupported manifest feature sets such as temporal, feedback, texture-backed, font-backed, or text-parameter shaders are hidden from the control UI for now. Runtime shaders are exposed through `RuntimeLayerModel` as display layers with manifest parameter definitions, current parameter values, build status, and render-ready artifacts. POST controls mutate this app-owned model and may start background shader builds when the selected shader changes. @@ -341,7 +380,6 @@ This app keeps the same core behavior but splits it into modules that can grow: - `platform/`: COM/Win32/hidden GL context support - `render/`: cadence thread, clock, and simple renderer - `frames/InputFrameMailbox`: non-blocking latest-frame CPU input handoff -- `video/SyntheticInputProducer`: temporary BGRA8 test-pattern producer for proving the frame-input path - `render/InputFrameTexture`: render-thread-owned upload of the latest CPU input frame into GL - `render/readback/`: PBO-backed BGRA8 readback and completed-frame publication - `render/runtime/RuntimeRenderScene`: render-thread-owned GL scene for ready runtime shader layers @@ -365,4 +403,4 @@ Only after this app matches the probe's smooth output: 3. port runtime snapshots/live state 4. add control services 5. add preview/screenshot from system-memory frames -6. replace synthetic input with DeckLink input capture into the existing CPU latest-frame mailbox +6. add scaling and additional input format support after the BGRA8/UYVY8 input edge is stable diff --git a/apps/RenderCadenceCompositor/RenderCadenceCompositor.cpp b/apps/RenderCadenceCompositor/RenderCadenceCompositor.cpp index cb4515b..322478d 100644 --- a/apps/RenderCadenceCompositor/RenderCadenceCompositor.cpp +++ b/apps/RenderCadenceCompositor/RenderCadenceCompositor.cpp @@ -5,7 +5,9 @@ #include "frames/SystemFrameExchange.h" #include "logging/Logger.h" #include "render/RenderThread.h" -#include "video/SyntheticInputProducer.h" +#include "video/DeckLinkInput.h" +#include "video/DeckLinkInputThread.h" +#include "DeckLinkDisplayMode.h" #include "VideoIOFormat.h" #include @@ -92,15 +94,40 @@ int main(int argc, char** argv) inputMailboxConfig.capacity = 4; InputFrameMailbox inputMailbox(inputMailboxConfig); - RenderCadenceCompositor::SyntheticInputProducerConfig inputProducerConfig; - inputProducerConfig.width = inputMailboxConfig.width; - inputProducerConfig.height = inputMailboxConfig.height; - inputProducerConfig.frameDurationMilliseconds = RenderCadenceCompositor::FrameDurationMillisecondsFromRateString(appConfig.inputFrameRate); - RenderCadenceCompositor::SyntheticInputProducer syntheticInput(inputMailbox, inputProducerConfig); - if (syntheticInput.Start()) - RenderCadenceCompositor::Log("app", "Synthetic BGRA8 input producer started."); + VideoFormat inputVideoMode; + std::string inputVideoModeError; + const bool inputVideoModeResolved = ResolveConfiguredVideoFormat(appConfig.inputVideoFormat, appConfig.inputFrameRate, inputVideoMode); + if (!inputVideoModeResolved) + { + inputVideoModeError = "Unsupported DeckLink inputVideoFormat/inputFrameRate in config/runtime-host.json: " + + appConfig.inputVideoFormat + " / " + appConfig.inputFrameRate; + RenderCadenceCompositor::LogWarning("app", inputVideoModeError); + } + + RenderCadenceCompositor::DeckLinkInput deckLinkInput(inputMailbox); + RenderCadenceCompositor::DeckLinkInputThread deckLinkInputThread(deckLinkInput); + bool deckLinkInputStarted = false; + if (inputVideoModeResolved) + { + RenderCadenceCompositor::DeckLinkInputConfig deckLinkInputConfig; + deckLinkInputConfig.videoFormat = inputVideoMode; + std::string deckLinkInputError; + if (deckLinkInput.Initialize(deckLinkInputConfig, deckLinkInputError) && + deckLinkInputThread.Start(deckLinkInputError)) + { + deckLinkInputStarted = true; + RenderCadenceCompositor::Log("app", "DeckLink input edge started for " + inputVideoMode.displayName + "."); + } + else + { + RenderCadenceCompositor::LogWarning("app", "DeckLink input edge unavailable; runtime shaders will use fallback input until a real input edge is available. " + deckLinkInputError); + deckLinkInput.ReleaseResources(); + } + } else - RenderCadenceCompositor::LogWarning("app", "Synthetic input producer did not start; shaders will use fallback input."); + { + RenderCadenceCompositor::LogWarning("app", "DeckLink input mode was not resolved; runtime shaders will use fallback input until a real input edge is available."); + } RenderThread::Config renderConfig; renderConfig.width = frameExchangeConfig.width; @@ -111,12 +138,16 @@ int main(int argc, char** argv) RenderThread renderThread(frameExchange, &inputMailbox, renderConfig); RenderCadenceCompositor::RenderCadenceApp app(renderThread, frameExchange, appConfig); + app.SetDeckLinkInputMetricsProvider([&deckLinkInput]() { + return deckLinkInput.Metrics(); + }); std::string error; if (!app.Start(error)) { RenderCadenceCompositor::LogError("app", "RenderCadenceCompositor start failed: " + error); - syntheticInput.Stop(); + if (deckLinkInputStarted) + deckLinkInputThread.Stop(); RenderCadenceCompositor::Logger::Instance().Stop(); return 1; } @@ -124,7 +155,8 @@ int main(int argc, char** argv) std::string line; std::getline(std::cin, line); app.Stop(); - syntheticInput.Stop(); + if (deckLinkInputStarted) + deckLinkInputThread.Stop(); RenderCadenceCompositor::Log("app", "RenderCadenceCompositor stopped."); RenderCadenceCompositor::Logger::Instance().Stop(); return 0; diff --git a/apps/RenderCadenceCompositor/app/RenderCadenceApp.h b/apps/RenderCadenceCompositor/app/RenderCadenceApp.h index 32629b5..19231af 100644 --- a/apps/RenderCadenceCompositor/app/RenderCadenceApp.h +++ b/apps/RenderCadenceCompositor/app/RenderCadenceApp.h @@ -6,11 +6,13 @@ #include "../logging/Logger.h" #include "../control/RuntimeStateJson.h" #include "../telemetry/TelemetryHealthMonitor.h" +#include "../video/DeckLinkInput.h" #include "../video/DeckLinkOutput.h" #include "../video/DeckLinkOutputThread.h" #include #include +#include #include #include #include @@ -118,6 +120,10 @@ public: bool Started() const { return mStarted; } const DeckLinkOutput& Output() const { return mOutput; } + void SetDeckLinkInputMetricsProvider(std::function provider) + { + mDeckLinkInputMetricsProvider = std::move(provider); + } private: void StartOptionalVideoOutput() @@ -208,6 +214,7 @@ private: std::string BuildStateJson() { CadenceTelemetrySnapshot telemetry = mHttpTelemetry.Sample(mFrameExchange, mOutput, mOutputThread, mRenderThread); + ApplyDeckLinkInputMetrics(telemetry); RuntimeLayerModelSnapshot layerSnapshot = mRuntimeLayers.Snapshot(telemetry); return RuntimeStateToJson(RuntimeStateJsonInput{ mConfig, @@ -220,6 +227,23 @@ private: }); } + void ApplyDeckLinkInputMetrics(CadenceTelemetrySnapshot& telemetry) + { + if (!mDeckLinkInputMetricsProvider) + return; + + const DeckLinkInputMetrics inputMetrics = mDeckLinkInputMetricsProvider(); + telemetry.inputConvertMilliseconds = inputMetrics.convertMilliseconds; + telemetry.inputSubmitMilliseconds = inputMetrics.submitMilliseconds; + telemetry.inputNoSignalFrames = inputMetrics.noInputSourceFrames; + telemetry.inputUnsupportedFrames = inputMetrics.unsupportedFrames; + telemetry.inputSubmitMisses = inputMetrics.submitMisses; + telemetry.inputCaptureFormat = inputMetrics.captureFormat ? inputMetrics.captureFormat : "none"; + if (telemetry.sampleSeconds > 0.0) + telemetry.inputCaptureFps = static_cast(inputMetrics.capturedFrames - mLastInputCapturedFrames) / telemetry.sampleSeconds; + mLastInputCapturedFrames = inputMetrics.capturedFrames; + } + bool WaitForPreroll() const { const auto deadline = std::chrono::steady_clock::now() + mConfig.prerollTimeout; @@ -241,6 +265,8 @@ private: CadenceTelemetry mHttpTelemetry; HttpControlServer mHttpServer; RuntimeLayerController mRuntimeLayers; + std::function mDeckLinkInputMetricsProvider; + uint64_t mLastInputCapturedFrames = 0; bool mStarted = false; bool mVideoOutputEnabled = false; std::string mVideoOutputStatus = "DeckLink output not started."; diff --git a/apps/RenderCadenceCompositor/frames/InputFrameMailbox.cpp b/apps/RenderCadenceCompositor/frames/InputFrameMailbox.cpp index e1482e5..52baa15 100644 --- a/apps/RenderCadenceCompositor/frames/InputFrameMailbox.cpp +++ b/apps/RenderCadenceCompositor/frames/InputFrameMailbox.cpp @@ -1,6 +1,7 @@ #include "InputFrameMailbox.h" #include +#include #include namespace @@ -102,6 +103,8 @@ bool InputFrameMailbox::SubmitFrame(const void* bytes, unsigned rowBytes, uint64 mReadyIndices.push_back(slotIndex); ++mCounters.submittedFrames; mCounters.latestFrameIndex = frameIndex; + mCounters.hasSubmittedFrame = true; + mLatestSubmitTime = std::chrono::steady_clock::now(); return true; } @@ -170,6 +173,11 @@ InputFrameMailboxMetrics InputFrameMailbox::Metrics() const std::lock_guard lock(mMutex); InputFrameMailboxMetrics metrics = mCounters; metrics.capacity = mSlots.size(); + if (metrics.hasSubmittedFrame) + { + metrics.latestFrameAgeMilliseconds = std::chrono::duration_cast>( + std::chrono::steady_clock::now() - mLatestSubmitTime).count(); + } for (const Slot& slot : mSlots) { diff --git a/apps/RenderCadenceCompositor/frames/InputFrameMailbox.h b/apps/RenderCadenceCompositor/frames/InputFrameMailbox.h index 48d90cb..b94831f 100644 --- a/apps/RenderCadenceCompositor/frames/InputFrameMailbox.h +++ b/apps/RenderCadenceCompositor/frames/InputFrameMailbox.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,8 @@ struct InputFrameMailboxMetrics uint64_t submitMisses = 0; uint64_t consumeMisses = 0; uint64_t latestFrameIndex = 0; + bool hasSubmittedFrame = false; + double latestFrameAgeMilliseconds = 0.0; }; class InputFrameMailbox @@ -84,4 +87,5 @@ private: std::vector mSlots; std::deque mReadyIndices; InputFrameMailboxMetrics mCounters; + std::chrono::steady_clock::time_point mLatestSubmitTime; }; diff --git a/apps/RenderCadenceCompositor/render/InputFrameTexture.cpp b/apps/RenderCadenceCompositor/render/InputFrameTexture.cpp index e99d183..b1de000 100644 --- a/apps/RenderCadenceCompositor/render/InputFrameTexture.cpp +++ b/apps/RenderCadenceCompositor/render/InputFrameTexture.cpp @@ -1,5 +1,7 @@ #include "InputFrameTexture.h" +#include + InputFrameTexture::~InputFrameTexture() { ShutdownGl(); @@ -14,28 +16,24 @@ GLuint InputFrameTexture::PollAndUpload(InputFrameMailbox* mailbox) if (!mailbox->TryAcquireLatest(frame)) { ++mUploadMisses; + mLastUploadMilliseconds = 0.0; return mTexture; } if (frame.bytes != nullptr && frame.pixelFormat == VideoIOPixelFormat::Bgra8 && EnsureTexture(frame)) { - glBindTexture(GL_TEXTURE_2D, mTexture); - glPixelStorei(GL_UNPACK_ALIGNMENT, 4); - glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.rowBytes > 0 ? static_cast(frame.rowBytes / 4) : 0); - glTexSubImage2D( - GL_TEXTURE_2D, - 0, - 0, - 0, - static_cast(frame.width), - static_cast(frame.height), - GL_BGRA, - GL_UNSIGNED_INT_8_8_8_8_REV, - frame.bytes); - glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); - glBindTexture(GL_TEXTURE_2D, 0); + mLastFrameFormatSupported = true; + const auto uploadStart = std::chrono::steady_clock::now(); + UploadBgra8FrameFlippedVertically(frame); + const auto uploadEnd = std::chrono::steady_clock::now(); + mLastUploadMilliseconds = std::chrono::duration_cast>(uploadEnd - uploadStart).count(); ++mUploadedFrames; } + else + { + mLastFrameFormatSupported = frame.pixelFormat == VideoIOPixelFormat::Bgra8; + mLastUploadMilliseconds = 0.0; + } mailbox->Release(frame); return mTexture; @@ -81,3 +79,30 @@ bool InputFrameTexture::EnsureTexture(const InputFrame& frame) mHeight = frame.height; return mTexture != 0; } + +void InputFrameTexture::UploadBgra8FrameFlippedVertically(const InputFrame& frame) +{ + glBindTexture(GL_TEXTURE_2D, mTexture); + glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.rowBytes > 0 ? static_cast(frame.rowBytes / 4) : 0); + + const unsigned char* sourceBytes = static_cast(frame.bytes); + for (unsigned destinationY = 0; destinationY < frame.height; ++destinationY) + { + const unsigned sourceY = frame.height - 1u - destinationY; + const unsigned char* sourceRow = sourceBytes + static_cast(sourceY) * static_cast(frame.rowBytes); + glTexSubImage2D( + GL_TEXTURE_2D, + 0, + 0, + static_cast(destinationY), + static_cast(frame.width), + 1, + GL_BGRA, + GL_UNSIGNED_INT_8_8_8_8_REV, + sourceRow); + } + + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + glBindTexture(GL_TEXTURE_2D, 0); +} diff --git a/apps/RenderCadenceCompositor/render/InputFrameTexture.h b/apps/RenderCadenceCompositor/render/InputFrameTexture.h index bda5497..02f2eda 100644 --- a/apps/RenderCadenceCompositor/render/InputFrameTexture.h +++ b/apps/RenderCadenceCompositor/render/InputFrameTexture.h @@ -17,14 +17,19 @@ public: GLuint Texture() const { return mTexture; } uint64_t UploadedFrames() const { return mUploadedFrames; } uint64_t UploadMisses() const { return mUploadMisses; } + double LastUploadMilliseconds() const { return mLastUploadMilliseconds; } + bool LastFrameFormatSupported() const { return mLastFrameFormatSupported; } void ShutdownGl(); private: bool EnsureTexture(const InputFrame& frame); + void UploadBgra8FrameFlippedVertically(const InputFrame& frame); GLuint mTexture = 0; unsigned mWidth = 0; unsigned mHeight = 0; uint64_t mUploadedFrames = 0; uint64_t mUploadMisses = 0; + double mLastUploadMilliseconds = 0.0; + bool mLastFrameFormatSupported = true; }; diff --git a/apps/RenderCadenceCompositor/render/RenderThread.cpp b/apps/RenderCadenceCompositor/render/RenderThread.cpp index 6dcd173..46f6efa 100644 --- a/apps/RenderCadenceCompositor/render/RenderThread.cpp +++ b/apps/RenderCadenceCompositor/render/RenderThread.cpp @@ -85,6 +85,12 @@ RenderThread::Metrics RenderThread::GetMetrics() const metrics.skippedFrames = mSkippedFrames.load(std::memory_order_relaxed); metrics.shaderBuildsCommitted = mShaderBuildsCommitted.load(std::memory_order_relaxed); metrics.shaderBuildFailures = mShaderBuildFailures.load(std::memory_order_relaxed); + metrics.inputFramesReceived = mInputFramesReceived.load(std::memory_order_relaxed); + metrics.inputFramesDropped = mInputFramesDropped.load(std::memory_order_relaxed); + metrics.inputLatestAgeMilliseconds = mInputLatestAgeMilliseconds.load(std::memory_order_relaxed); + metrics.inputUploadMilliseconds = mInputUploadMilliseconds.load(std::memory_order_relaxed); + metrics.inputFormatSupported = mInputFormatSupported.load(std::memory_order_relaxed); + metrics.inputSignalPresent = mInputSignalPresent.load(std::memory_order_relaxed); return metrics; } @@ -156,6 +162,7 @@ void RenderThread::ThreadMain() TryCommitReadyRuntimeShader(runtimeRenderScene); const GLuint videoInputTexture = inputTexture.PollAndUpload(mInputMailbox); + PublishInputMetrics(inputTexture); if (!readback.RenderAndQueue(frameIndex, [this, &renderer, &runtimeRenderScene, videoInputTexture](uint64_t index) { if (runtimeRenderScene.HasLayers()) runtimeRenderScene.RenderFrame(index, mConfig.width, mConfig.height, videoInputTexture); @@ -226,6 +233,28 @@ void RenderThread::CountAcquireMiss() mAcquireMisses.fetch_add(1, std::memory_order_relaxed); } +void RenderThread::PublishInputMetrics(const InputFrameTexture& inputTexture) +{ + if (mInputMailbox != nullptr) + { + const InputFrameMailboxMetrics mailboxMetrics = mInputMailbox->Metrics(); + mInputFramesReceived.store(mailboxMetrics.submittedFrames, std::memory_order_relaxed); + mInputFramesDropped.store(mailboxMetrics.droppedReadyFrames + mailboxMetrics.submitMisses, std::memory_order_relaxed); + mInputLatestAgeMilliseconds.store(mailboxMetrics.latestFrameAgeMilliseconds, std::memory_order_relaxed); + mInputSignalPresent.store(mailboxMetrics.hasSubmittedFrame, std::memory_order_relaxed); + } + else + { + mInputFramesReceived.store(0, std::memory_order_relaxed); + mInputFramesDropped.store(0, std::memory_order_relaxed); + mInputLatestAgeMilliseconds.store(0.0, std::memory_order_relaxed); + mInputSignalPresent.store(false, std::memory_order_relaxed); + } + + mInputUploadMilliseconds.store(inputTexture.LastUploadMilliseconds(), std::memory_order_relaxed); + mInputFormatSupported.store(inputTexture.LastFrameFormatSupported(), std::memory_order_relaxed); +} + void RenderThread::SubmitRuntimeShaderArtifact(const RuntimeShaderArtifact& artifact) { if (artifact.fragmentShaderSource.empty()) diff --git a/apps/RenderCadenceCompositor/render/RenderThread.h b/apps/RenderCadenceCompositor/render/RenderThread.h index fefc0b7..0a81c30 100644 --- a/apps/RenderCadenceCompositor/render/RenderThread.h +++ b/apps/RenderCadenceCompositor/render/RenderThread.h @@ -15,6 +15,7 @@ class SystemFrameExchange; class InputFrameMailbox; +class InputFrameTexture; class RenderThread { @@ -37,6 +38,12 @@ public: uint64_t skippedFrames = 0; uint64_t shaderBuildsCommitted = 0; uint64_t shaderBuildFailures = 0; + uint64_t inputFramesReceived = 0; + uint64_t inputFramesDropped = 0; + double inputLatestAgeMilliseconds = 0.0; + double inputUploadMilliseconds = 0.0; + bool inputFormatSupported = true; + bool inputSignalPresent = false; }; RenderThread(SystemFrameExchange& frameExchange, Config config); @@ -60,6 +67,7 @@ private: void CountRendered(); void CountCompleted(); void CountAcquireMiss(); + void PublishInputMetrics(const InputFrameTexture& inputTexture); void TryCommitReadyRuntimeShader(RuntimeRenderScene& runtimeRenderScene); bool TryTakePendingRuntimeShaderArtifact(RuntimeShaderArtifact& artifact); bool TryTakePendingRenderLayers(std::vector& layers); @@ -84,6 +92,12 @@ private: std::atomic mSkippedFrames{ 0 }; std::atomic mShaderBuildsCommitted{ 0 }; std::atomic mShaderBuildFailures{ 0 }; + std::atomic mInputFramesReceived{ 0 }; + std::atomic mInputFramesDropped{ 0 }; + std::atomic mInputLatestAgeMilliseconds{ 0.0 }; + std::atomic mInputUploadMilliseconds{ 0.0 }; + std::atomic mInputFormatSupported{ true }; + std::atomic mInputSignalPresent{ false }; std::mutex mShaderArtifactMutex; bool mHasPendingShaderArtifact = false; diff --git a/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.cpp b/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.cpp index 530bd4f..1b5e7e2 100644 --- a/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.cpp +++ b/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.cpp @@ -159,6 +159,9 @@ void RuntimeRenderScene::RenderFrame(uint64_t frameIndex, unsigned width, unsign return; } + // Shader source contract: + // - gVideoInput is the raw/latest input texture for every layer in the stack. + // - gLayerInput starts as gVideoInput for the first layer, then becomes the previous layer output. GLuint layerInputTexture = videoInputTexture; std::size_t nextTargetIndex = 0; for (std::size_t layerIndex = 0; layerIndex < readyLayers.size(); ++layerIndex) @@ -324,7 +327,7 @@ GLuint RuntimeRenderScene::RenderLayer( if (!pass.renderer || !pass.renderer->HasProgram()) continue; - GLuint sourceTexture = layerInputTexture; + GLuint sourceTexture = videoInputTexture; if (!pass.inputNames.empty()) { const std::string& inputName = pass.inputNames.front(); @@ -334,6 +337,9 @@ GLuint RuntimeRenderScene::RenderLayer( } else if (inputName != "layerInput") { + // Named intermediate pass inputs currently use the gVideoInput binding slot as the + // selected pass source. Layer stack shaders should use gLayerInput for previous-layer + // sampling and gVideoInput for the original input frame. for (std::size_t index = 0; index < 2; ++index) { if (namedOutputNames[index] == inputName) diff --git a/apps/RenderCadenceCompositor/telemetry/CadenceTelemetry.h b/apps/RenderCadenceCompositor/telemetry/CadenceTelemetry.h index 6ff08e4..aaf69b2 100644 --- a/apps/RenderCadenceCompositor/telemetry/CadenceTelemetry.h +++ b/apps/RenderCadenceCompositor/telemetry/CadenceTelemetry.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace RenderCadenceCompositor { @@ -23,6 +24,19 @@ struct CadenceTelemetrySnapshot uint64_t dropped = 0; uint64_t shaderBuildsCommitted = 0; uint64_t shaderBuildFailures = 0; + uint64_t inputFramesReceived = 0; + uint64_t inputFramesDropped = 0; + double inputLatestAgeMilliseconds = 0.0; + double inputUploadMilliseconds = 0.0; + bool inputFormatSupported = true; + bool inputSignalPresent = false; + double inputCaptureFps = 0.0; + double inputConvertMilliseconds = 0.0; + double inputSubmitMilliseconds = 0.0; + uint64_t inputNoSignalFrames = 0; + uint64_t inputUnsupportedFrames = 0; + uint64_t inputSubmitMisses = 0; + std::string inputCaptureFormat = "none"; bool deckLinkBufferedAvailable = false; uint64_t deckLinkBuffered = 0; double deckLinkScheduleCallMilliseconds = 0.0; @@ -88,6 +102,34 @@ public: const auto renderMetrics = renderThread.GetMetrics(); snapshot.shaderBuildsCommitted = renderMetrics.shaderBuildsCommitted; snapshot.shaderBuildFailures = renderMetrics.shaderBuildFailures; + snapshot.inputFramesReceived = renderMetrics.inputFramesReceived; + snapshot.inputFramesDropped = renderMetrics.inputFramesDropped; + snapshot.inputLatestAgeMilliseconds = renderMetrics.inputLatestAgeMilliseconds; + snapshot.inputUploadMilliseconds = renderMetrics.inputUploadMilliseconds; + snapshot.inputFormatSupported = renderMetrics.inputFormatSupported; + snapshot.inputSignalPresent = renderMetrics.inputSignalPresent; + return snapshot; + } + + template + CadenceTelemetrySnapshot Sample( + const SystemFrameExchange& exchange, + const Output& output, + const OutputThread& outputThread, + const RenderThread& renderThread, + const InputEdge& inputEdge) + { + CadenceTelemetrySnapshot snapshot = Sample(exchange, output, outputThread, renderThread); + const auto inputMetrics = inputEdge.Metrics(); + snapshot.inputConvertMilliseconds = inputMetrics.convertMilliseconds; + snapshot.inputSubmitMilliseconds = inputMetrics.submitMilliseconds; + snapshot.inputNoSignalFrames = inputMetrics.noInputSourceFrames; + snapshot.inputUnsupportedFrames = inputMetrics.unsupportedFrames; + snapshot.inputSubmitMisses = inputMetrics.submitMisses; + snapshot.inputCaptureFormat = inputMetrics.captureFormat ? inputMetrics.captureFormat : "none"; + if (snapshot.sampleSeconds > 0.0) + snapshot.inputCaptureFps = static_cast(inputMetrics.capturedFrames - mLastInputCapturedFrames) / snapshot.sampleSeconds; + mLastInputCapturedFrames = inputMetrics.capturedFrames; return snapshot; } @@ -97,6 +139,7 @@ private: Clock::time_point mLastSampleTime = Clock::now(); uint64_t mLastRenderedFrames = 0; uint64_t mLastScheduledFrames = 0; + uint64_t mLastInputCapturedFrames = 0; bool mHasLastSample = false; }; } diff --git a/apps/RenderCadenceCompositor/telemetry/CadenceTelemetryJson.h b/apps/RenderCadenceCompositor/telemetry/CadenceTelemetryJson.h index 66c3034..3618717 100644 --- a/apps/RenderCadenceCompositor/telemetry/CadenceTelemetryJson.h +++ b/apps/RenderCadenceCompositor/telemetry/CadenceTelemetryJson.h @@ -26,6 +26,19 @@ inline void WriteCadenceTelemetryJson(JsonWriter& writer, const CadenceTelemetry writer.KeyUInt("dropped", snapshot.dropped); writer.KeyUInt("shaderCommitted", snapshot.shaderBuildsCommitted); writer.KeyUInt("shaderFailures", snapshot.shaderBuildFailures); + writer.KeyUInt("inputFramesReceived", snapshot.inputFramesReceived); + writer.KeyUInt("inputFramesDropped", snapshot.inputFramesDropped); + writer.KeyDouble("inputLatestAgeMs", snapshot.inputLatestAgeMilliseconds); + writer.KeyDouble("inputUploadMs", snapshot.inputUploadMilliseconds); + writer.KeyBool("inputFormatSupported", snapshot.inputFormatSupported); + writer.KeyBool("inputSignalPresent", snapshot.inputSignalPresent); + writer.KeyDouble("inputCaptureFps", snapshot.inputCaptureFps); + writer.KeyDouble("inputConvertMs", snapshot.inputConvertMilliseconds); + writer.KeyDouble("inputSubmitMs", snapshot.inputSubmitMilliseconds); + writer.KeyUInt("inputNoSignalFrames", snapshot.inputNoSignalFrames); + writer.KeyUInt("inputUnsupportedFrames", snapshot.inputUnsupportedFrames); + writer.KeyUInt("inputSubmitMisses", snapshot.inputSubmitMisses); + writer.KeyString("inputCaptureFormat", snapshot.inputCaptureFormat); writer.KeyBool("deckLinkBufferedAvailable", snapshot.deckLinkBufferedAvailable); writer.Key("deckLinkBuffered"); if (snapshot.deckLinkBufferedAvailable) diff --git a/apps/RenderCadenceCompositor/video/DeckLinkInput.cpp b/apps/RenderCadenceCompositor/video/DeckLinkInput.cpp new file mode 100644 index 0000000..d3a05d2 --- /dev/null +++ b/apps/RenderCadenceCompositor/video/DeckLinkInput.cpp @@ -0,0 +1,408 @@ +#include "DeckLinkInput.h" + +#include "DeckLinkVideoIOFormat.h" +#include "../logging/Logger.h" + +#include +#include +#include + +namespace RenderCadenceCompositor +{ +namespace +{ +bool FindInputDisplayMode(IDeckLinkInput* input, BMDDisplayMode targetMode, IDeckLinkDisplayMode** foundMode) +{ + if (input == nullptr || foundMode == nullptr) + return false; + + *foundMode = nullptr; + CComPtr iterator; + if (input->GetDisplayModeIterator(&iterator) != S_OK) + return false; + + return FindDeckLinkDisplayMode(iterator, targetMode, foundMode); +} + +unsigned char ClampToByte(double value) +{ + if (value <= 0.0) + return 0; + if (value >= 255.0) + return 255; + return static_cast(value + 0.5); +} + +void StoreRec709UyvyAsBgra(unsigned char yByte, unsigned char uByte, unsigned char vByte, unsigned char* destination) +{ + const double y = (static_cast(yByte) - 16.0) / 219.0; + const double cb = (static_cast(uByte) - 16.0) / 224.0 - 0.5; + const double cr = (static_cast(vByte) - 16.0) / 224.0 - 0.5; + const double red = y + 1.5748 * cr; + const double green = y - 0.1873 * cb - 0.4681 * cr; + const double blue = y + 1.8556 * cb; + destination[0] = ClampToByte(blue * 255.0); + destination[1] = ClampToByte(green * 255.0); + destination[2] = ClampToByte(red * 255.0); + destination[3] = 255; +} +} + +DeckLinkInputCallback::DeckLinkInputCallback(DeckLinkInput& owner) : + mOwner(owner) +{ +} + +HRESULT STDMETHODCALLTYPE DeckLinkInputCallback::QueryInterface(REFIID iid, LPVOID* ppv) +{ + if (ppv == nullptr) + return E_POINTER; + if (iid == IID_IUnknown || iid == IID_IDeckLinkInputCallback) + { + *ppv = static_cast(this); + AddRef(); + return S_OK; + } + *ppv = nullptr; + return E_NOINTERFACE; +} + +ULONG STDMETHODCALLTYPE DeckLinkInputCallback::AddRef() +{ + return ++mRefCount; +} + +ULONG STDMETHODCALLTYPE DeckLinkInputCallback::Release() +{ + const ULONG refCount = --mRefCount; + if (refCount == 0) + delete this; + return refCount; +} + +HRESULT STDMETHODCALLTYPE DeckLinkInputCallback::VideoInputFrameArrived(IDeckLinkVideoInputFrame* videoFrame, IDeckLinkAudioInputPacket*) +{ + if (videoFrame != nullptr) + mOwner.HandleFrameArrived(videoFrame); + return S_OK; +} + +HRESULT STDMETHODCALLTYPE DeckLinkInputCallback::VideoInputFormatChanged(BMDVideoInputFormatChangedEvents, IDeckLinkDisplayMode*, BMDDetectedVideoInputFormatFlags) +{ + mOwner.HandleFormatChanged(); + return S_OK; +} + +DeckLinkInput::DeckLinkInput(InputFrameMailbox& mailbox) : + mMailbox(mailbox) +{ +} + +DeckLinkInput::~DeckLinkInput() +{ + ReleaseResources(); +} + +bool DeckLinkInput::Initialize(const DeckLinkInputConfig& config, std::string& error) +{ + ReleaseResources(); + mConfig = config; + Log("decklink-input", "Initializing DeckLink input for " + config.videoFormat.displayName + "."); + + if (!DiscoverInput(config, error)) + return false; + + if (mInput->EnableVideoInput(config.videoFormat.displayMode, mCapturePixelFormat, bmdVideoInputFlagDefault) != S_OK) + { + error = "DeckLink input setup failed while enabling " + + std::string(mCapturePixelFormat == bmdFormat8BitBGRA ? "BGRA8" : "UYVY8") + + " input for " + config.videoFormat.displayName + "."; + ReleaseResources(); + return false; + } + Log( + "decklink-input", + std::string("DeckLink input enabled in ") + (mCapturePixelFormat == bmdFormat8BitBGRA ? "BGRA8" : "UYVY8-to-BGRA8 conversion") + " mode."); + + mCallback.Attach(new (std::nothrow) DeckLinkInputCallback(*this)); + if (mCallback == nullptr) + { + error = "DeckLink input setup failed while creating the capture callback."; + ReleaseResources(); + return false; + } + + if (mInput->SetCallback(mCallback) != S_OK) + { + error = "DeckLink input setup failed while installing the capture callback."; + ReleaseResources(); + return false; + } + Log("decklink-input", "DeckLink input callback installed."); + + return true; +} + +bool DeckLinkInput::Start(std::string& error) +{ + if (mInput == nullptr) + { + error = "DeckLink input has not been initialized."; + return false; + } + if (mRunning.load(std::memory_order_acquire)) + return true; + + if (mInput->StartStreams() != S_OK) + { + error = "DeckLink input stream failed to start."; + return false; + } + mRunning.store(true, std::memory_order_release); + Log("decklink-input", "DeckLink input stream started."); + return true; +} + +void DeckLinkInput::Stop() +{ + if (mInput != nullptr && mRunning.exchange(false, std::memory_order_acq_rel)) + { + mInput->StopStreams(); + Log("decklink-input", "DeckLink input stream stopped."); + } +} + +void DeckLinkInput::ReleaseResources() +{ + Stop(); + if (mInput != nullptr) + { + mInput->SetCallback(nullptr); + mInput->DisableVideoInput(); + } + mCallback.Release(); + mInput.Release(); +} + +DeckLinkInputMetrics DeckLinkInput::Metrics() const +{ + DeckLinkInputMetrics metrics; + metrics.capturedFrames = mCapturedFrames.load(std::memory_order_relaxed); + metrics.noInputSourceFrames = mNoInputSourceFrames.load(std::memory_order_relaxed); + metrics.unsupportedFrames = mUnsupportedFrames.load(std::memory_order_relaxed); + metrics.submitMisses = mSubmitMisses.load(std::memory_order_relaxed); + metrics.convertMilliseconds = mConvertMilliseconds.load(std::memory_order_relaxed); + metrics.submitMilliseconds = mSubmitMilliseconds.load(std::memory_order_relaxed); + metrics.captureFormat = CaptureFormatName(); + return metrics; +} + +void DeckLinkInput::HandleFrameArrived(IDeckLinkVideoInputFrame* inputFrame) +{ + if (inputFrame == nullptr) + return; + + if ((inputFrame->GetFlags() & bmdFrameHasNoInputSource) == bmdFrameHasNoInputSource) + { + mNoInputSourceFrames.fetch_add(1, std::memory_order_relaxed); + bool expected = false; + if (mLoggedNoInputSource.compare_exchange_strong(expected, true, std::memory_order_relaxed)) + LogWarning("decklink-input", "DeckLink input callback reports no input source."); + return; + } + + if (inputFrame->GetWidth() != static_cast(mMailbox.Config().width) || + inputFrame->GetHeight() != static_cast(mMailbox.Config().height)) + { + mUnsupportedFrames.fetch_add(1, std::memory_order_relaxed); + bool expected = false; + if (mLoggedUnsupportedFrame.compare_exchange_strong(expected, true, std::memory_order_relaxed)) + LogWarning("decklink-input", "DeckLink input frame dimensions do not match the configured mailbox."); + return; + } + + CComPtr inputFrameBuffer; + if (inputFrame->QueryInterface(IID_IDeckLinkVideoBuffer, reinterpret_cast(&inputFrameBuffer)) != S_OK) + { + mUnsupportedFrames.fetch_add(1, std::memory_order_relaxed); + bool expected = false; + if (mLoggedUnsupportedFrame.compare_exchange_strong(expected, true, std::memory_order_relaxed)) + LogWarning("decklink-input", "DeckLink input frame does not expose IDeckLinkVideoBuffer."); + return; + } + + if (inputFrameBuffer->StartAccess(bmdBufferAccessRead) != S_OK) + { + mUnsupportedFrames.fetch_add(1, std::memory_order_relaxed); + bool expected = false; + if (mLoggedUnsupportedFrame.compare_exchange_strong(expected, true, std::memory_order_relaxed)) + LogWarning("decklink-input", "DeckLink input frame buffer could not be opened for read access."); + return; + } + + void* bytes = nullptr; + inputFrameBuffer->GetBytes(&bytes); + bool submitted = false; + if (mCapturePixelFormat == bmdFormat8BitBGRA) + submitted = SubmitBgra8Frame(inputFrame, bytes); + else if (mCapturePixelFormat == bmdFormat8BitYUV) + submitted = SubmitUyvy8Frame(inputFrame, bytes); + else + mUnsupportedFrames.fetch_add(1, std::memory_order_relaxed); + + if (!submitted) + { + mSubmitMisses.fetch_add(1, std::memory_order_relaxed); + bool expected = false; + if (mLoggedSubmitMiss.compare_exchange_strong(expected, true, std::memory_order_relaxed)) + LogWarning("decklink-input", "DeckLink input frame could not be submitted to InputFrameMailbox."); + } + mCapturedFrames.fetch_add(1, std::memory_order_relaxed); + bool expectedFirstFrame = false; + if (mLoggedFirstFrame.compare_exchange_strong(expectedFirstFrame, true, std::memory_order_relaxed)) + { + Log( + "decklink-input", + std::string("First DeckLink ") + (mCapturePixelFormat == bmdFormat8BitBGRA ? "BGRA8" : "UYVY8 converted BGRA8") + " input frame submitted to InputFrameMailbox."); + } + + inputFrameBuffer->EndAccess(bmdBufferAccessRead); +} + +void DeckLinkInput::HandleFormatChanged() +{ + mUnsupportedFrames.fetch_add(1, std::memory_order_relaxed); + LogWarning("decklink-input", "DeckLink input format changed; input edge does not auto-switch formats yet."); +} + +bool DeckLinkInput::DiscoverInput(const DeckLinkInputConfig& config, std::string& error) +{ + CComPtr iterator; + HRESULT result = CoCreateInstance(CLSID_CDeckLinkIterator, nullptr, CLSCTX_ALL, IID_IDeckLinkIterator, reinterpret_cast(&iterator)); + if (FAILED(result)) + { + error = "DeckLink input discovery failed. Blackmagic DeckLink drivers may not be installed."; + return false; + } + + CComPtr deckLink; + while (iterator->Next(&deckLink) == S_OK) + { + CComPtr candidateInput; + if (deckLink->QueryInterface(IID_IDeckLinkInput, reinterpret_cast(&candidateInput)) == S_OK && candidateInput != nullptr) + { + CComPtr displayMode; + if (FindInputDisplayMode(candidateInput, config.videoFormat.displayMode, &displayMode) && + SupportsInputFormat(candidateInput, config.videoFormat.displayMode, bmdFormat8BitBGRA)) + { + mInput = candidateInput; + mCapturePixelFormat = bmdFormat8BitBGRA; + Log("decklink-input", "DeckLink input device selected for BGRA8 capture."); + return true; + } + if (displayMode != nullptr && + SupportsInputFormat(candidateInput, config.videoFormat.displayMode, bmdFormat8BitYUV)) + { + mInput = candidateInput; + mCapturePixelFormat = bmdFormat8BitYUV; + Log("decklink-input", "DeckLink input device selected for UYVY8 capture with CPU BGRA8 conversion."); + return true; + } + } + deckLink.Release(); + } + + error = "No DeckLink input device supports BGRA8 or UYVY8 capture for " + config.videoFormat.displayName + "."; + return false; +} + +bool DeckLinkInput::SupportsInputFormat(IDeckLinkInput* input, BMDDisplayMode displayMode, BMDPixelFormat pixelFormat) const +{ + if (input == nullptr) + return false; + + BOOL supported = FALSE; + BMDDisplayMode actualMode = bmdModeUnknown; + const HRESULT result = input->DoesSupportVideoMode( + bmdVideoConnectionUnspecified, + displayMode, + pixelFormat, + bmdNoVideoInputConversion, + bmdSupportedVideoModeDefault, + &actualMode, + &supported); + return result == S_OK && supported != FALSE; +} + +bool DeckLinkInput::SubmitBgra8Frame(IDeckLinkVideoInputFrame* inputFrame, const void* bytes) +{ + if (inputFrame == nullptr || bytes == nullptr) + return false; + const uint64_t frameIndex = mCapturedFrames.load(std::memory_order_relaxed); + mConvertMilliseconds.store(0.0, std::memory_order_relaxed); + const auto submitStart = std::chrono::steady_clock::now(); + const bool submitted = mMailbox.SubmitFrame(bytes, static_cast(inputFrame->GetRowBytes()), frameIndex); + const auto submitEnd = std::chrono::steady_clock::now(); + mSubmitMilliseconds.store( + std::chrono::duration_cast>(submitEnd - submitStart).count(), + std::memory_order_relaxed); + return submitted; +} + +bool DeckLinkInput::SubmitUyvy8Frame(IDeckLinkVideoInputFrame* inputFrame, const void* bytes) +{ + if (inputFrame == nullptr || bytes == nullptr) + return false; + + const unsigned width = static_cast(inputFrame->GetWidth()); + const unsigned height = static_cast(inputFrame->GetHeight()); + const long sourceRowBytes = inputFrame->GetRowBytes(); + if (width == 0 || height == 0 || sourceRowBytes < static_cast(width * 2u)) + return false; + + const unsigned destinationRowBytes = VideoIORowBytes(VideoIOPixelFormat::Bgra8, width); + const auto convertStart = std::chrono::steady_clock::now(); + mConversionBuffer.resize(static_cast(destinationRowBytes) * static_cast(height)); + const unsigned char* sourceBytes = static_cast(bytes); + for (unsigned y = 0; y < height; ++y) + { + const unsigned char* sourceRow = sourceBytes + static_cast(y) * static_cast(sourceRowBytes); + unsigned char* destinationRow = mConversionBuffer.data() + static_cast(y) * static_cast(destinationRowBytes); + for (unsigned x = 0; x < width; x += 2) + { + const unsigned pairOffset = x * 2u; + const unsigned char u = sourceRow[pairOffset + 0]; + const unsigned char y0 = sourceRow[pairOffset + 1]; + const unsigned char v = sourceRow[pairOffset + 2]; + const unsigned char y1 = sourceRow[pairOffset + 3]; + StoreRec709UyvyAsBgra(y0, u, v, destinationRow + static_cast(x) * 4u); + if (x + 1u < width) + StoreRec709UyvyAsBgra(y1, u, v, destinationRow + static_cast(x + 1u) * 4u); + } + } + const auto convertEnd = std::chrono::steady_clock::now(); + mConvertMilliseconds.store( + std::chrono::duration_cast>(convertEnd - convertStart).count(), + std::memory_order_relaxed); + + const uint64_t frameIndex = mCapturedFrames.load(std::memory_order_relaxed); + const auto submitStart = std::chrono::steady_clock::now(); + const bool submitted = mMailbox.SubmitFrame(mConversionBuffer.data(), destinationRowBytes, frameIndex); + const auto submitEnd = std::chrono::steady_clock::now(); + mSubmitMilliseconds.store( + std::chrono::duration_cast>(submitEnd - submitStart).count(), + std::memory_order_relaxed); + return submitted; +} + +const char* DeckLinkInput::CaptureFormatName() const +{ + if (mInput == nullptr) + return "none"; + if (mCapturePixelFormat == bmdFormat8BitBGRA) + return "BGRA8"; + if (mCapturePixelFormat == bmdFormat8BitYUV) + return "UYVY8"; + return "unsupported"; +} +} diff --git a/apps/RenderCadenceCompositor/video/DeckLinkInput.h b/apps/RenderCadenceCompositor/video/DeckLinkInput.h new file mode 100644 index 0000000..5f23df5 --- /dev/null +++ b/apps/RenderCadenceCompositor/video/DeckLinkInput.h @@ -0,0 +1,96 @@ +#pragma once + +#include "../frames/InputFrameMailbox.h" +#include "DeckLinkAPI_h.h" +#include "DeckLinkDisplayMode.h" + +#include + +#include +#include +#include +#include +#include + +namespace RenderCadenceCompositor +{ +struct DeckLinkInputConfig +{ + VideoFormat videoFormat; +}; + +struct DeckLinkInputMetrics +{ + uint64_t capturedFrames = 0; + uint64_t noInputSourceFrames = 0; + uint64_t unsupportedFrames = 0; + uint64_t submitMisses = 0; + double convertMilliseconds = 0.0; + double submitMilliseconds = 0.0; + const char* captureFormat = "none"; +}; + +class DeckLinkInput; + +class DeckLinkInputCallback final : public IDeckLinkInputCallback +{ +public: + explicit DeckLinkInputCallback(DeckLinkInput& owner); + + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID* ppv) override; + ULONG STDMETHODCALLTYPE AddRef() override; + ULONG STDMETHODCALLTYPE Release() override; + HRESULT STDMETHODCALLTYPE VideoInputFrameArrived(IDeckLinkVideoInputFrame* videoFrame, IDeckLinkAudioInputPacket* audioPacket) override; + HRESULT STDMETHODCALLTYPE VideoInputFormatChanged(BMDVideoInputFormatChangedEvents notificationEvents, IDeckLinkDisplayMode* newDisplayMode, BMDDetectedVideoInputFormatFlags detectedSignalFlags) override; + +private: + DeckLinkInput& mOwner; + std::atomic mRefCount{ 1 }; +}; + +class DeckLinkInput +{ +public: + DeckLinkInput(InputFrameMailbox& mailbox); + DeckLinkInput(const DeckLinkInput&) = delete; + DeckLinkInput& operator=(const DeckLinkInput&) = delete; + ~DeckLinkInput(); + + bool Initialize(const DeckLinkInputConfig& config, std::string& error); + bool Start(std::string& error); + void Stop(); + void ReleaseResources(); + + bool IsInitialized() const { return mInput != nullptr; } + bool IsRunning() const { return mRunning.load(std::memory_order_acquire); } + DeckLinkInputMetrics Metrics() const; + + void HandleFrameArrived(IDeckLinkVideoInputFrame* inputFrame); + void HandleFormatChanged(); + +private: + bool DiscoverInput(const DeckLinkInputConfig& config, std::string& error); + bool SupportsInputFormat(IDeckLinkInput* input, BMDDisplayMode displayMode, BMDPixelFormat pixelFormat) const; + bool SubmitBgra8Frame(IDeckLinkVideoInputFrame* inputFrame, const void* bytes); + bool SubmitUyvy8Frame(IDeckLinkVideoInputFrame* inputFrame, const void* bytes); + const char* CaptureFormatName() const; + + InputFrameMailbox& mMailbox; + DeckLinkInputConfig mConfig; + BMDPixelFormat mCapturePixelFormat = bmdFormat8BitBGRA; + CComPtr mInput; + CComPtr mCallback; + std::vector mConversionBuffer; + std::atomic mRunning{ false }; + std::atomic mCapturedFrames{ 0 }; + std::atomic mNoInputSourceFrames{ 0 }; + std::atomic mUnsupportedFrames{ 0 }; + std::atomic mSubmitMisses{ 0 }; + std::atomic mConvertMilliseconds{ 0.0 }; + std::atomic mSubmitMilliseconds{ 0.0 }; + std::atomic mLoggedFirstFrame{ false }; + std::atomic mLoggedNoInputSource{ false }; + std::atomic mLoggedUnsupportedFrame{ false }; + std::atomic mLoggedSubmitMiss{ false }; +}; +} diff --git a/apps/RenderCadenceCompositor/video/DeckLinkInputThread.h b/apps/RenderCadenceCompositor/video/DeckLinkInputThread.h new file mode 100644 index 0000000..9985d28 --- /dev/null +++ b/apps/RenderCadenceCompositor/video/DeckLinkInputThread.h @@ -0,0 +1,87 @@ +#pragma once + +#include "DeckLinkInput.h" + +#include +#include +#include +#include + +namespace RenderCadenceCompositor +{ +struct DeckLinkInputThreadConfig +{ + std::chrono::milliseconds idleSleep = std::chrono::milliseconds(100); +}; + +class DeckLinkInputThread +{ +public: + DeckLinkInputThread(DeckLinkInput& input, DeckLinkInputThreadConfig config = DeckLinkInputThreadConfig()) : + mInput(input), + mConfig(config) + { + } + + DeckLinkInputThread(const DeckLinkInputThread&) = delete; + DeckLinkInputThread& operator=(const DeckLinkInputThread&) = delete; + + ~DeckLinkInputThread() + { + Stop(); + } + + bool Start(std::string& error) + { + if (mThread.joinable()) + return true; + mStartSucceeded.store(false, std::memory_order_release); + mStartCompleted.store(false, std::memory_order_release); + mStopping.store(false, std::memory_order_release); + mThread = std::thread([this]() { ThreadMain(); }); + + while (!mStartCompleted.load(std::memory_order_acquire)) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mStartSucceeded.load(std::memory_order_acquire)) + return true; + + error = mStartError; + Stop(); + return false; + } + + void Stop() + { + mStopping.store(true, std::memory_order_release); + if (mThread.joinable()) + mThread.join(); + } + +private: + void ThreadMain() + { + std::string error; + if (!mInput.Start(error)) + { + mStartError = error; + mStartCompleted.store(true, std::memory_order_release); + return; + } + + mStartSucceeded.store(true, std::memory_order_release); + mStartCompleted.store(true, std::memory_order_release); + while (!mStopping.load(std::memory_order_acquire)) + std::this_thread::sleep_for(mConfig.idleSleep); + mInput.Stop(); + } + + DeckLinkInput& mInput; + DeckLinkInputThreadConfig mConfig; + std::thread mThread; + std::atomic mStopping{ false }; + std::atomic mStartCompleted{ false }; + std::atomic mStartSucceeded{ false }; + std::string mStartError; +}; +} diff --git a/apps/RenderCadenceCompositor/video/SyntheticInputProducer.cpp b/apps/RenderCadenceCompositor/video/SyntheticInputProducer.cpp deleted file mode 100644 index c5f4e02..0000000 --- a/apps/RenderCadenceCompositor/video/SyntheticInputProducer.cpp +++ /dev/null @@ -1,111 +0,0 @@ -#include "SyntheticInputProducer.h" - -#include "VideoIOFormat.h" - -#include -#include - -namespace RenderCadenceCompositor -{ -SyntheticInputProducer::SyntheticInputProducer(InputFrameMailbox& mailbox, SyntheticInputProducerConfig config) : - mMailbox(mailbox), - mConfig(config) -{ -} - -SyntheticInputProducer::~SyntheticInputProducer() -{ - Stop(); -} - -bool SyntheticInputProducer::Start() -{ - if (mThread.joinable()) - return true; - if (mConfig.width == 0 || mConfig.height == 0) - return false; - - mStopping.store(false, std::memory_order_release); - mThread = std::thread([this]() { ThreadMain(); }); - return true; -} - -void SyntheticInputProducer::Stop() -{ - mStopping.store(true, std::memory_order_release); - if (mThread.joinable()) - mThread.join(); -} - -SyntheticInputProducerMetrics SyntheticInputProducer::Metrics() const -{ - SyntheticInputProducerMetrics metrics; - metrics.generatedFrames = mGeneratedFrames.load(std::memory_order_relaxed); - metrics.submitMisses = mSubmitMisses.load(std::memory_order_relaxed); - return metrics; -} - -void SyntheticInputProducer::ThreadMain() -{ - const unsigned rowBytes = VideoIORowBytes(VideoIOPixelFormat::Bgra8, mConfig.width); - std::vector buffer(static_cast(rowBytes) * static_cast(mConfig.height)); - const auto frameDuration = std::chrono::duration_cast( - std::chrono::duration(mConfig.frameDurationMilliseconds)); - - uint64_t frameIndex = 0; - auto nextFrameTime = std::chrono::steady_clock::now(); - while (!mStopping.load(std::memory_order_acquire)) - { - const auto now = std::chrono::steady_clock::now(); - if (now < nextFrameTime) - { - std::this_thread::sleep_for((std::min)(std::chrono::milliseconds(1), std::chrono::duration_cast(nextFrameTime - now))); - continue; - } - - GenerateFrame(frameIndex, buffer); - if (!mMailbox.SubmitFrame(buffer.data(), rowBytes, frameIndex)) - mSubmitMisses.fetch_add(1, std::memory_order_relaxed); - mGeneratedFrames.fetch_add(1, std::memory_order_relaxed); - ++frameIndex; - nextFrameTime += frameDuration; - if (std::chrono::steady_clock::now() - nextFrameTime > frameDuration * 4) - nextFrameTime = std::chrono::steady_clock::now() + frameDuration; - } -} - -void SyntheticInputProducer::GenerateFrame(uint64_t frameIndex, std::vector& buffer) const -{ - const float t = static_cast(frameIndex) / 60.0f; - const unsigned boxWidth = (std::max)(1u, mConfig.width / 5u); - const unsigned boxHeight = (std::max)(1u, mConfig.height / 6u); - const unsigned maxX = mConfig.width > boxWidth ? mConfig.width - boxWidth : 0u; - const unsigned maxY = mConfig.height > boxHeight ? mConfig.height - boxHeight : 0u; - const unsigned boxX = static_cast((0.5f + 0.5f * std::sin(t * 1.4f)) * static_cast(maxX)); - const unsigned boxY = static_cast((0.5f + 0.5f * std::sin(t * 0.9f + 1.2f)) * static_cast(maxY)); - const unsigned rowBytes = VideoIORowBytes(VideoIOPixelFormat::Bgra8, mConfig.width); - - for (unsigned y = 0; y < mConfig.height; ++y) - { - for (unsigned x = 0; x < mConfig.width; ++x) - { - const std::size_t offset = static_cast(y) * rowBytes + static_cast(x) * 4; - const unsigned checker = ((x / 80u) + (y / 80u) + static_cast(frameIndex / 15u)) & 1u; - unsigned char red = checker ? 42 : 16; - unsigned char green = checker ? 70 : 28; - unsigned char blue = checker ? 110 : 55; - if (x >= boxX && x < boxX + boxWidth && y >= boxY && y < boxY + boxHeight) - { - red = 245; - green = static_cast(160 + 60 * (0.5f + 0.5f * std::sin(t * 2.1f))); - blue = 35; - } - - buffer[offset + 0] = blue; - buffer[offset + 1] = green; - buffer[offset + 2] = red; - buffer[offset + 3] = 255; - } - } -} -} diff --git a/apps/RenderCadenceCompositor/video/SyntheticInputProducer.h b/apps/RenderCadenceCompositor/video/SyntheticInputProducer.h deleted file mode 100644 index 94cf0a5..0000000 --- a/apps/RenderCadenceCompositor/video/SyntheticInputProducer.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include "../frames/InputFrameMailbox.h" - -#include -#include -#include -#include -#include - -namespace RenderCadenceCompositor -{ -struct SyntheticInputProducerConfig -{ - unsigned width = 1920; - unsigned height = 1080; - double frameDurationMilliseconds = 1000.0 / 59.94; -}; - -struct SyntheticInputProducerMetrics -{ - uint64_t generatedFrames = 0; - uint64_t submitMisses = 0; -}; - -class SyntheticInputProducer -{ -public: - SyntheticInputProducer(InputFrameMailbox& mailbox, SyntheticInputProducerConfig config); - SyntheticInputProducer(const SyntheticInputProducer&) = delete; - SyntheticInputProducer& operator=(const SyntheticInputProducer&) = delete; - ~SyntheticInputProducer(); - - bool Start(); - void Stop(); - SyntheticInputProducerMetrics Metrics() const; - -private: - void ThreadMain(); - void GenerateFrame(uint64_t frameIndex, std::vector& buffer) const; - - InputFrameMailbox& mMailbox; - SyntheticInputProducerConfig mConfig; - std::thread mThread; - std::atomic mStopping{ false }; - std::atomic mGeneratedFrames{ 0 }; - std::atomic mSubmitMisses{ 0 }; -}; -} diff --git a/tests/RenderCadenceCompositorTelemetryTests.cpp b/tests/RenderCadenceCompositorTelemetryTests.cpp index e92354e..fdb71ec 100644 --- a/tests/RenderCadenceCompositorTelemetryTests.cpp +++ b/tests/RenderCadenceCompositorTelemetryTests.cpp @@ -67,6 +67,12 @@ struct FakeRenderThreadMetrics { uint64_t shaderBuildsCommitted = 0; uint64_t shaderBuildFailures = 0; + uint64_t inputFramesReceived = 0; + uint64_t inputFramesDropped = 0; + double inputLatestAgeMilliseconds = 0.0; + double inputUploadMilliseconds = 0.0; + bool inputFormatSupported = true; + bool inputSignalPresent = false; }; struct FakeRenderThread @@ -96,6 +102,12 @@ void TestTelemetrySamplesCompletedPollMissesAndShaderCounts() FakeRenderThread renderThread; renderThread.metrics.shaderBuildsCommitted = 1; renderThread.metrics.shaderBuildFailures = 0; + renderThread.metrics.inputFramesReceived = 9; + renderThread.metrics.inputFramesDropped = 2; + renderThread.metrics.inputLatestAgeMilliseconds = 4.5; + renderThread.metrics.inputUploadMilliseconds = 0.25; + renderThread.metrics.inputFormatSupported = true; + renderThread.metrics.inputSignalPresent = true; const auto snapshot = telemetry.Sample(exchange, output, outputThread, renderThread); Expect(snapshot.freeFrames == 7, "free frame count is sampled"); @@ -104,6 +116,12 @@ void TestTelemetrySamplesCompletedPollMissesAndShaderCounts() Expect(snapshot.completedPollMisses == 12, "completed poll misses are sampled"); Expect(snapshot.shaderBuildsCommitted == 1, "shader committed count is sampled"); Expect(snapshot.shaderBuildFailures == 0, "shader failure count is sampled"); + Expect(snapshot.inputFramesReceived == 9, "input received count is sampled"); + Expect(snapshot.inputFramesDropped == 2, "input dropped count is sampled"); + Expect(snapshot.inputLatestAgeMilliseconds == 4.5, "input latest age is sampled"); + Expect(snapshot.inputUploadMilliseconds == 0.25, "input upload timing is sampled"); + Expect(snapshot.inputFormatSupported, "input format support is sampled"); + Expect(snapshot.inputSignalPresent, "input signal present is sampled"); Expect(snapshot.deckLinkBufferedAvailable, "buffer telemetry availability is sampled"); Expect(snapshot.deckLinkBuffered == 4, "buffer depth is sampled"); } @@ -148,6 +166,19 @@ void TestTelemetrySerializesToJson() snapshot.dropped = 2; snapshot.shaderBuildsCommitted = 1; snapshot.shaderBuildFailures = 0; + snapshot.inputFramesReceived = 10; + snapshot.inputFramesDropped = 1; + snapshot.inputLatestAgeMilliseconds = 3.5; + snapshot.inputUploadMilliseconds = 0.75; + snapshot.inputFormatSupported = true; + snapshot.inputSignalPresent = true; + snapshot.inputCaptureFps = 59.94; + snapshot.inputConvertMilliseconds = 4.25; + snapshot.inputSubmitMilliseconds = 0.35; + snapshot.inputNoSignalFrames = 2; + snapshot.inputUnsupportedFrames = 3; + snapshot.inputSubmitMisses = 4; + snapshot.inputCaptureFormat = "UYVY8"; snapshot.deckLinkBufferedAvailable = true; snapshot.deckLinkBuffered = 4; snapshot.deckLinkScheduleCallMilliseconds = 1.25; @@ -160,6 +191,13 @@ void TestTelemetrySerializesToJson() "\"completedPollMisses\":3,\"scheduleFailures\":0," "\"completions\":117,\"late\":1,\"dropped\":2," "\"shaderCommitted\":1,\"shaderFailures\":0," + "\"inputFramesReceived\":10,\"inputFramesDropped\":1," + "\"inputLatestAgeMs\":3.5,\"inputUploadMs\":0.75," + "\"inputFormatSupported\":true,\"inputSignalPresent\":true," + "\"inputCaptureFps\":59.94,\"inputConvertMs\":4.25," + "\"inputSubmitMs\":0.35,\"inputNoSignalFrames\":2," + "\"inputUnsupportedFrames\":3,\"inputSubmitMisses\":4," + "\"inputCaptureFormat\":\"UYVY8\"," "\"deckLinkBufferedAvailable\":true,\"deckLinkBuffered\":4," "\"scheduleCallMs\":1.25}"; Expect(json == expected, "telemetry snapshot serializes to stable JSON");