diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ffd8b0..e780b24 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,6 +318,8 @@ set(RENDER_CADENCE_APP_SOURCES "${RENDER_CADENCE_APP_DIR}/control/http/HttpControlServer.h" "${RENDER_CADENCE_APP_DIR}/control/http/HttpControlServerWebSocket.cpp" "${RENDER_CADENCE_APP_DIR}/control/RuntimeStateJson.h" + "${RENDER_CADENCE_APP_DIR}/frames/InputFrameMailbox.cpp" + "${RENDER_CADENCE_APP_DIR}/frames/InputFrameMailbox.h" "${RENDER_CADENCE_APP_DIR}/frames/SystemFrameExchange.cpp" "${RENDER_CADENCE_APP_DIR}/frames/SystemFrameExchange.h" "${RENDER_CADENCE_APP_DIR}/frames/SystemFrameTypes.h" @@ -327,6 +329,8 @@ set(RENDER_CADENCE_APP_SOURCES "${RENDER_CADENCE_APP_DIR}/logging/Logger.h" "${RENDER_CADENCE_APP_DIR}/platform/HiddenGlWindow.cpp" "${RENDER_CADENCE_APP_DIR}/platform/HiddenGlWindow.h" + "${RENDER_CADENCE_APP_DIR}/render/InputFrameTexture.cpp" + "${RENDER_CADENCE_APP_DIR}/render/InputFrameTexture.h" "${RENDER_CADENCE_APP_DIR}/render/readback/Bgra8ReadbackPipeline.cpp" "${RENDER_CADENCE_APP_DIR}/render/readback/Bgra8ReadbackPipeline.h" "${RENDER_CADENCE_APP_DIR}/render/readback/PboReadbackRing.cpp" @@ -360,6 +364,8 @@ set(RENDER_CADENCE_APP_SOURCES "${RENDER_CADENCE_APP_DIR}/video/DeckLinkOutput.cpp" "${RENDER_CADENCE_APP_DIR}/video/DeckLinkOutput.h" "${RENDER_CADENCE_APP_DIR}/video/DeckLinkOutputThread.h" + "${RENDER_CADENCE_APP_DIR}/video/SyntheticInputProducer.cpp" + "${RENDER_CADENCE_APP_DIR}/video/SyntheticInputProducer.h" ) add_executable(RenderCadenceCompositor ${RENDER_CADENCE_APP_SOURCES}) @@ -790,6 +796,23 @@ endif() add_test(NAME RenderCadenceCompositorFrameExchangeTests COMMAND RenderCadenceCompositorFrameExchangeTests) +add_executable(RenderCadenceCompositorInputFrameMailboxTests + "${APP_DIR}/videoio/VideoIOFormat.cpp" + "${RENDER_CADENCE_APP_DIR}/frames/InputFrameMailbox.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/RenderCadenceCompositorInputFrameMailboxTests.cpp" +) + +target_include_directories(RenderCadenceCompositorInputFrameMailboxTests PRIVATE + "${APP_DIR}/videoio" + "${RENDER_CADENCE_APP_DIR}/frames" +) + +if(MSVC) + target_compile_options(RenderCadenceCompositorInputFrameMailboxTests PRIVATE /W3) +endif() + +add_test(NAME RenderCadenceCompositorInputFrameMailboxTests COMMAND RenderCadenceCompositorInputFrameMailboxTests) + add_executable(RenderCadenceCompositorClockTests "${RENDER_CADENCE_APP_DIR}/render/RenderCadenceClock.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/tests/RenderCadenceCompositorClockTests.cpp" diff --git a/apps/RenderCadenceCompositor/README.md b/apps/RenderCadenceCompositor/README.md index fd0ba34..f56e710 100644 --- a/apps/RenderCadenceCompositor/README.md +++ b/apps/RenderCadenceCompositor/README.md @@ -11,10 +11,17 @@ Before adding features here, read the guardrails in [Render Cadence Golden Rules ```text RenderThread owns a hidden OpenGL context + polls latest input frames without waiting + uploads input frames into a render-owned GL texture renders simple BGRA8 motion at selected cadence queues async PBO readback publishes completed frames into SystemFrameExchange +InputFrameMailbox + owns latest disposable CPU input slots + drops older unsampled input frames when newer frames arrive + protects the one frame currently being uploaded by render + SystemFrameExchange owns Free / Rendering / Completed / Scheduled slots drops old completed unscheduled frames when render needs space @@ -37,6 +44,9 @@ Included now: - hidden render-thread-owned OpenGL context - simple smooth-motion renderer - BGRA8-only output +- synthetic BGRA8 input producer +- non-blocking latest-frame input mailbox +- render-thread-owned input texture upload - async PBO readback - latest-N system-memory frame exchange - rendered-frame warmup @@ -55,18 +65,24 @@ Included now: - JSON serialization for cadence telemetry snapshots - background logging with `log`, `warning`, and `error` levels - local HTTP control server matching the OpenAPI route surface +- HTTP layer controls for add, remove, reorder, bypass, shader change, parameter update, and parameter reset +- trigger parameters as latest-pulse controls with shader-visible count/time - startup config provider for `config/runtime-host.json` - quiet telemetry health monitor - non-GL frame-exchange tests +- non-GL input-mailbox tests Intentionally not included yet: -- DeckLink input +- real DeckLink input capture +- input format conversion - temporal/history/feedback shader storage - texture/LUT asset upload - text-parameter rasterization - runtime state -- OSC/API control +- OSC control +- persistent control/state writes +- trigger event history for stacked repeated pulses - preview - screenshots - persistence @@ -99,16 +115,22 @@ This tracks parity with `apps/LoopThroughWithOpenGLCompositing`. - [x] Startup config loading from `config/runtime-host.json` - [x] Cadence telemetry JSON - [x] Health logging for schedule/drop/starvation events +- [x] Runtime parameter updates from HTTP controls +- [x] Layer reorder/bypass/set-shader/update-parameter/reset-parameter HTTP controls +- [x] Trigger parameter pulse count/time for latest trigger events +- [x] Synthetic BGRA8 frame input producer +- [x] Latest-frame CPU input mailbox +- [x] Render-owned input texture upload +- [x] Runtime shaders receive input through `gVideoInput` - [ ] DeckLink input capture -- [ ] Input frame upload into the render scene -- [ ] Live video input bound to `gVideoInput` +- [ ] Live DeckLink input bound to `gVideoInput` +- [ ] Input format conversion/scaling - [ ] Temporal history buffers - [ ] Feedback buffers - [ ] Texture asset loading and upload - [ ] LUT asset loading and upload - [ ] Text parameter rasterization -- [ ] Runtime parameter updates from controls -- [ ] Layer reorder/bypass/set-shader/update-parameter/reset-parameter controls +- [ ] Trigger history/event buffers for overlapping repeated trigger effects - [ ] Full runtime state store/read model - [ ] Persistent layer stack/config writes - [ ] OSC ingress @@ -256,13 +278,17 @@ Current runtime shader support is deliberately limited to stateless full-frame p - no feedback storage - no texture/LUT assets yet - no text parameters yet -- manifest defaults are used for parameters +- manifest defaults initialize parameters +- HTTP controls can update runtime parameter values without rebuilding GL programs when the shader program is unchanged +- trigger parameters are treated as latest-pulse controls: each press increments the trigger count and records the current runtime time +- repeated trigger history is not stored yet, so effects such as `trigger-ripple` restart from the latest trigger rather than accumulating overlapping ripples - the first layer receives a small fallback source texture until DeckLink input is added -- stacked layers receive the previous ready layer output through both `gVideoInput` and `gLayerInput` +- the first layer receives the latest synthetic input texture through both `gVideoInput` and `gLayerInput` when input frames are available +- stacked layers receive the original input through `gVideoInput` and the previous ready layer output through `gLayerInput` The `/api/state` shader list uses the same support rules as runtime shader compilation and reports only packages this app can run today. Unsupported manifest feature sets such as temporal, feedback, texture-backed, font-backed, or text-parameter shaders are hidden from the control UI for now. -Runtime shaders are exposed through `RuntimeLayerModel` as display layers with manifest parameter defaults. The model also records whether each layer has a render-ready artifact. Add/remove POST controls mutate this app-owned model and may start background shader builds. +Runtime shaders are exposed through `RuntimeLayerModel` as display layers with manifest parameter definitions, current parameter values, build status, and render-ready artifacts. POST controls mutate this app-owned model and may start background shader builds when the selected shader changes. When a layer becomes render-ready, the app publishes the ready render-layer snapshot to the render thread. The render thread owns the GL-side `RuntimeRenderScene`, diffs that snapshot at a frame boundary, queues new or changed pass programs to the shared-context prepare worker, swaps in a full prepared render plan only after every pass is ready, removes obsolete GL programs, and renders ready layers in order. Stacked stateless full-frame shaders render through internal ping-pong targets so each layer can sample the previous layer through `gLayerInput`; multipass shaders route named intermediate outputs through their manifest-declared pass inputs, and the final ready layer renders to the output target. @@ -314,6 +340,9 @@ This app keeps the same core behavior but splits it into modules that can grow: - `frames/`: system-memory handoff - `platform/`: COM/Win32/hidden GL context support - `render/`: cadence thread, clock, and simple renderer +- `frames/InputFrameMailbox`: non-blocking latest-frame CPU input handoff +- `video/SyntheticInputProducer`: temporary BGRA8 test-pattern producer for proving the frame-input path +- `render/InputFrameTexture`: render-thread-owned upload of the latest CPU input frame into GL - `render/readback/`: PBO-backed BGRA8 readback and completed-frame publication - `render/runtime/RuntimeRenderScene`: render-thread-owned GL scene for ready runtime shader layers - `render/runtime/RuntimeShaderPrepareWorker`: shared-context runtime shader program compile/link worker @@ -336,4 +365,4 @@ Only after this app matches the probe's smooth output: 3. port runtime snapshots/live state 4. add control services 5. add preview/screenshot from system-memory frames -6. add DeckLink input as a CPU latest-frame mailbox +6. replace synthetic input with DeckLink input capture into the existing CPU latest-frame mailbox diff --git a/apps/RenderCadenceCompositor/RenderCadenceCompositor.cpp b/apps/RenderCadenceCompositor/RenderCadenceCompositor.cpp index 09cbe7b..cb4515b 100644 --- a/apps/RenderCadenceCompositor/RenderCadenceCompositor.cpp +++ b/apps/RenderCadenceCompositor/RenderCadenceCompositor.cpp @@ -1,9 +1,11 @@ #include "app/AppConfig.h" #include "app/AppConfigProvider.h" #include "app/RenderCadenceApp.h" +#include "frames/InputFrameMailbox.h" #include "frames/SystemFrameExchange.h" #include "logging/Logger.h" #include "render/RenderThread.h" +#include "video/SyntheticInputProducer.h" #include "VideoIOFormat.h" #include @@ -80,13 +82,33 @@ int main(int argc, char** argv) SystemFrameExchange frameExchange(frameExchangeConfig); + InputFrameMailboxConfig inputMailboxConfig; + RenderCadenceCompositor::VideoFormatDimensions( + appConfig.inputVideoFormat, + inputMailboxConfig.width, + inputMailboxConfig.height); + inputMailboxConfig.pixelFormat = VideoIOPixelFormat::Bgra8; + inputMailboxConfig.rowBytes = VideoIORowBytes(inputMailboxConfig.pixelFormat, inputMailboxConfig.width); + inputMailboxConfig.capacity = 4; + InputFrameMailbox inputMailbox(inputMailboxConfig); + + RenderCadenceCompositor::SyntheticInputProducerConfig inputProducerConfig; + inputProducerConfig.width = inputMailboxConfig.width; + inputProducerConfig.height = inputMailboxConfig.height; + inputProducerConfig.frameDurationMilliseconds = RenderCadenceCompositor::FrameDurationMillisecondsFromRateString(appConfig.inputFrameRate); + RenderCadenceCompositor::SyntheticInputProducer syntheticInput(inputMailbox, inputProducerConfig); + if (syntheticInput.Start()) + RenderCadenceCompositor::Log("app", "Synthetic BGRA8 input producer started."); + else + RenderCadenceCompositor::LogWarning("app", "Synthetic input producer did not start; shaders will use fallback input."); + RenderThread::Config renderConfig; renderConfig.width = frameExchangeConfig.width; renderConfig.height = frameExchangeConfig.height; renderConfig.frameDurationMilliseconds = RenderCadenceCompositor::FrameDurationMillisecondsFromRateString(appConfig.outputFrameRate); renderConfig.pboDepth = 6; - RenderThread renderThread(frameExchange, renderConfig); + RenderThread renderThread(frameExchange, &inputMailbox, renderConfig); RenderCadenceCompositor::RenderCadenceApp app(renderThread, frameExchange, appConfig); @@ -94,6 +116,7 @@ int main(int argc, char** argv) if (!app.Start(error)) { RenderCadenceCompositor::LogError("app", "RenderCadenceCompositor start failed: " + error); + syntheticInput.Stop(); RenderCadenceCompositor::Logger::Instance().Stop(); return 1; } @@ -101,6 +124,7 @@ int main(int argc, char** argv) std::string line; std::getline(std::cin, line); app.Stop(); + syntheticInput.Stop(); RenderCadenceCompositor::Log("app", "RenderCadenceCompositor stopped."); RenderCadenceCompositor::Logger::Instance().Stop(); return 0; diff --git a/apps/RenderCadenceCompositor/frames/InputFrameMailbox.cpp b/apps/RenderCadenceCompositor/frames/InputFrameMailbox.cpp new file mode 100644 index 0000000..e1482e5 --- /dev/null +++ b/apps/RenderCadenceCompositor/frames/InputFrameMailbox.cpp @@ -0,0 +1,233 @@ +#include "InputFrameMailbox.h" + +#include +#include + +namespace +{ +InputFrameMailboxConfig NormalizeConfig(InputFrameMailboxConfig config) +{ + if (config.rowBytes == 0) + config.rowBytes = VideoIORowBytes(config.pixelFormat, config.width); + return config; +} +} + +InputFrameMailbox::InputFrameMailbox(const InputFrameMailboxConfig& config) +{ + Configure(config); +} + +void InputFrameMailbox::Configure(const InputFrameMailboxConfig& config) +{ + std::lock_guard lock(mMutex); + mConfig = NormalizeConfig(config); + mReadyIndices.clear(); + mSlots.clear(); + mSlots.resize(mConfig.capacity); + + const std::size_t byteCount = FrameByteCount(); + for (Slot& slot : mSlots) + { + slot.bytes.resize(byteCount); + slot.state = InputFrameSlotState::Free; + slot.frameIndex = 0; + ++slot.generation; + } + + mCounters = InputFrameMailboxMetrics(); +} + +InputFrameMailboxConfig InputFrameMailbox::Config() const +{ + std::lock_guard lock(mMutex); + return mConfig; +} + +bool InputFrameMailbox::SubmitFrame(const void* bytes, unsigned rowBytes, uint64_t frameIndex) +{ + if (bytes == nullptr || rowBytes == 0) + return false; + + std::lock_guard lock(mMutex); + if (mSlots.empty() || mConfig.width == 0 || mConfig.height == 0) + return false; + + std::size_t slotIndex = mSlots.size(); + for (std::size_t index = 0; index < mSlots.size(); ++index) + { + if (mSlots[index].state == InputFrameSlotState::Free) + { + slotIndex = index; + break; + } + } + + if (slotIndex == mSlots.size()) + { + if (!DropOldestReadyLocked()) + { + ++mCounters.submitMisses; + return false; + } + + for (std::size_t index = 0; index < mSlots.size(); ++index) + { + if (mSlots[index].state == InputFrameSlotState::Free) + { + slotIndex = index; + break; + } + } + } + + if (slotIndex == mSlots.size()) + { + ++mCounters.submitMisses; + return false; + } + + Slot& slot = mSlots[slotIndex]; + const std::size_t destinationRowBytes = mConfig.rowBytes; + const std::size_t copyRowBytes = (std::min)(static_cast(rowBytes), destinationRowBytes); + const unsigned char* source = static_cast(bytes); + for (unsigned y = 0; y < mConfig.height; ++y) + { + std::memcpy(slot.bytes.data() + static_cast(y) * destinationRowBytes, source + static_cast(y) * rowBytes, copyRowBytes); + } + + slot.state = InputFrameSlotState::Ready; + slot.frameIndex = frameIndex; + ++slot.generation; + mReadyIndices.push_back(slotIndex); + ++mCounters.submittedFrames; + mCounters.latestFrameIndex = frameIndex; + return true; +} + +bool InputFrameMailbox::TryAcquireLatest(InputFrame& frame) +{ + std::lock_guard lock(mMutex); + while (!mReadyIndices.empty()) + { + const std::size_t index = mReadyIndices.back(); + mReadyIndices.pop_back(); + if (index >= mSlots.size() || mSlots[index].state != InputFrameSlotState::Ready) + continue; + + while (!mReadyIndices.empty()) + { + const std::size_t olderIndex = mReadyIndices.front(); + mReadyIndices.pop_front(); + if (olderIndex >= mSlots.size() || mSlots[olderIndex].state != InputFrameSlotState::Ready) + continue; + mSlots[olderIndex].state = InputFrameSlotState::Free; + ++mSlots[olderIndex].generation; + ++mCounters.droppedReadyFrames; + } + + mSlots[index].state = InputFrameSlotState::Reading; + FillFrameLocked(index, frame); + ++mCounters.consumedFrames; + return true; + } + + frame = InputFrame(); + ++mCounters.consumeMisses; + return false; +} + +bool InputFrameMailbox::Release(const InputFrame& frame) +{ + std::lock_guard lock(mMutex); + if (!IsValidLocked(frame)) + return false; + + Slot& slot = mSlots[frame.index]; + if (slot.state != InputFrameSlotState::Reading) + return false; + + slot.state = InputFrameSlotState::Free; + slot.frameIndex = 0; + ++slot.generation; + return true; +} + +void InputFrameMailbox::Clear() +{ + std::lock_guard lock(mMutex); + mReadyIndices.clear(); + for (Slot& slot : mSlots) + { + slot.state = InputFrameSlotState::Free; + slot.frameIndex = 0; + ++slot.generation; + } +} + +InputFrameMailboxMetrics InputFrameMailbox::Metrics() const +{ + std::lock_guard lock(mMutex); + InputFrameMailboxMetrics metrics = mCounters; + metrics.capacity = mSlots.size(); + + for (const Slot& slot : mSlots) + { + switch (slot.state) + { + case InputFrameSlotState::Free: + ++metrics.freeCount; + break; + case InputFrameSlotState::Ready: + ++metrics.readyCount; + break; + case InputFrameSlotState::Reading: + ++metrics.readingCount; + break; + } + } + + return metrics; +} + +bool InputFrameMailbox::IsValidLocked(const InputFrame& frame) const +{ + return frame.index < mSlots.size() && mSlots[frame.index].generation == frame.generation; +} + +void InputFrameMailbox::FillFrameLocked(std::size_t index, InputFrame& frame) const +{ + const Slot& slot = mSlots[index]; + frame.bytes = slot.bytes.empty() ? nullptr : slot.bytes.data(); + frame.rowBytes = static_cast(mConfig.rowBytes); + frame.width = mConfig.width; + frame.height = mConfig.height; + frame.pixelFormat = mConfig.pixelFormat; + frame.index = index; + frame.generation = slot.generation; + frame.frameIndex = slot.frameIndex; +} + +bool InputFrameMailbox::DropOldestReadyLocked() +{ + while (!mReadyIndices.empty()) + { + const std::size_t index = mReadyIndices.front(); + mReadyIndices.pop_front(); + if (index >= mSlots.size() || mSlots[index].state != InputFrameSlotState::Ready) + continue; + + mSlots[index].state = InputFrameSlotState::Free; + mSlots[index].frameIndex = 0; + ++mSlots[index].generation; + ++mCounters.droppedReadyFrames; + return true; + } + + return false; +} + +std::size_t InputFrameMailbox::FrameByteCount() const +{ + return static_cast(mConfig.rowBytes) * static_cast(mConfig.height); +} diff --git a/apps/RenderCadenceCompositor/frames/InputFrameMailbox.h b/apps/RenderCadenceCompositor/frames/InputFrameMailbox.h new file mode 100644 index 0000000..48d90cb --- /dev/null +++ b/apps/RenderCadenceCompositor/frames/InputFrameMailbox.h @@ -0,0 +1,87 @@ +#pragma once + +#include "VideoIOFormat.h" + +#include +#include +#include +#include +#include + +enum class InputFrameSlotState +{ + Free, + Ready, + Reading +}; + +struct InputFrameMailboxConfig +{ + unsigned width = 0; + unsigned height = 0; + VideoIOPixelFormat pixelFormat = VideoIOPixelFormat::Bgra8; + unsigned rowBytes = 0; + std::size_t capacity = 0; +}; + +struct InputFrame +{ + const void* bytes = nullptr; + long rowBytes = 0; + unsigned width = 0; + unsigned height = 0; + VideoIOPixelFormat pixelFormat = VideoIOPixelFormat::Bgra8; + std::size_t index = 0; + uint64_t generation = 0; + uint64_t frameIndex = 0; +}; + +struct InputFrameMailboxMetrics +{ + std::size_t capacity = 0; + std::size_t freeCount = 0; + std::size_t readyCount = 0; + std::size_t readingCount = 0; + uint64_t submittedFrames = 0; + uint64_t consumedFrames = 0; + uint64_t droppedReadyFrames = 0; + uint64_t submitMisses = 0; + uint64_t consumeMisses = 0; + uint64_t latestFrameIndex = 0; +}; + +class InputFrameMailbox +{ +public: + InputFrameMailbox() = default; + explicit InputFrameMailbox(const InputFrameMailboxConfig& config); + + void Configure(const InputFrameMailboxConfig& config); + InputFrameMailboxConfig Config() const; + + bool SubmitFrame(const void* bytes, unsigned rowBytes, uint64_t frameIndex); + bool TryAcquireLatest(InputFrame& frame); + bool Release(const InputFrame& frame); + void Clear(); + InputFrameMailboxMetrics Metrics() const; + +private: + struct Slot + { + std::vector bytes; + InputFrameSlotState state = InputFrameSlotState::Free; + uint64_t frameIndex = 0; + uint64_t generation = 0; + }; + + bool IsValidLocked(const InputFrame& frame) const; + void FillFrameLocked(std::size_t index, InputFrame& frame) const; + bool DropOldestReadyLocked(); + std::size_t FrameByteCount() const; + + mutable std::mutex mMutex; + InputFrameMailboxConfig mConfig; + std::vector mSlots; + std::deque mReadyIndices; + InputFrameMailboxMetrics mCounters; +}; diff --git a/apps/RenderCadenceCompositor/render/InputFrameTexture.cpp b/apps/RenderCadenceCompositor/render/InputFrameTexture.cpp new file mode 100644 index 0000000..e99d183 --- /dev/null +++ b/apps/RenderCadenceCompositor/render/InputFrameTexture.cpp @@ -0,0 +1,83 @@ +#include "InputFrameTexture.h" + +InputFrameTexture::~InputFrameTexture() +{ + ShutdownGl(); +} + +GLuint InputFrameTexture::PollAndUpload(InputFrameMailbox* mailbox) +{ + if (mailbox == nullptr) + return mTexture; + + InputFrame frame; + if (!mailbox->TryAcquireLatest(frame)) + { + ++mUploadMisses; + return mTexture; + } + + if (frame.bytes != nullptr && frame.pixelFormat == VideoIOPixelFormat::Bgra8 && EnsureTexture(frame)) + { + glBindTexture(GL_TEXTURE_2D, mTexture); + glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.rowBytes > 0 ? static_cast(frame.rowBytes / 4) : 0); + glTexSubImage2D( + GL_TEXTURE_2D, + 0, + 0, + 0, + static_cast(frame.width), + static_cast(frame.height), + GL_BGRA, + GL_UNSIGNED_INT_8_8_8_8_REV, + frame.bytes); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + glBindTexture(GL_TEXTURE_2D, 0); + ++mUploadedFrames; + } + + mailbox->Release(frame); + return mTexture; +} + +void InputFrameTexture::ShutdownGl() +{ + if (mTexture != 0) + glDeleteTextures(1, &mTexture); + mTexture = 0; + mWidth = 0; + mHeight = 0; +} + +bool InputFrameTexture::EnsureTexture(const InputFrame& frame) +{ + if (frame.width == 0 || frame.height == 0) + return false; + + if (mTexture != 0 && mWidth == frame.width && mHeight == frame.height) + return true; + + ShutdownGl(); + glGenTextures(1, &mTexture); + glBindTexture(GL_TEXTURE_2D, mTexture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexImage2D( + GL_TEXTURE_2D, + 0, + GL_RGBA8, + static_cast(frame.width), + static_cast(frame.height), + 0, + GL_BGRA, + GL_UNSIGNED_INT_8_8_8_8_REV, + nullptr); + glBindTexture(GL_TEXTURE_2D, 0); + + mWidth = frame.width; + mHeight = frame.height; + return mTexture != 0; +} diff --git a/apps/RenderCadenceCompositor/render/InputFrameTexture.h b/apps/RenderCadenceCompositor/render/InputFrameTexture.h new file mode 100644 index 0000000..bda5497 --- /dev/null +++ b/apps/RenderCadenceCompositor/render/InputFrameTexture.h @@ -0,0 +1,30 @@ +#pragma once + +#include "../frames/InputFrameMailbox.h" +#include "GLExtensions.h" + +#include + +class InputFrameTexture +{ +public: + InputFrameTexture() = default; + InputFrameTexture(const InputFrameTexture&) = delete; + InputFrameTexture& operator=(const InputFrameTexture&) = delete; + ~InputFrameTexture(); + + GLuint PollAndUpload(InputFrameMailbox* mailbox); + GLuint Texture() const { return mTexture; } + uint64_t UploadedFrames() const { return mUploadedFrames; } + uint64_t UploadMisses() const { return mUploadMisses; } + void ShutdownGl(); + +private: + bool EnsureTexture(const InputFrame& frame); + + GLuint mTexture = 0; + unsigned mWidth = 0; + unsigned mHeight = 0; + uint64_t mUploadedFrames = 0; + uint64_t mUploadMisses = 0; +}; diff --git a/apps/RenderCadenceCompositor/render/RenderThread.cpp b/apps/RenderCadenceCompositor/render/RenderThread.cpp index 3579bde..91fb3b3 100644 --- a/apps/RenderCadenceCompositor/render/RenderThread.cpp +++ b/apps/RenderCadenceCompositor/render/RenderThread.cpp @@ -1,9 +1,11 @@ #include "RenderThread.h" +#include "../frames/InputFrameMailbox.h" #include "../frames/SystemFrameExchange.h" #include "../frames/SystemFrameTypes.h" #include "../logging/Logger.h" #include "../platform/HiddenGlWindow.h" +#include "InputFrameTexture.h" #include "readback/Bgra8ReadbackPipeline.h" #include "GLExtensions.h" #include "runtime/RuntimeRenderScene.h" @@ -20,6 +22,13 @@ RenderThread::RenderThread(SystemFrameExchange& frameExchange, Config config) : { } +RenderThread::RenderThread(SystemFrameExchange& frameExchange, InputFrameMailbox* inputMailbox, Config config) : + mFrameExchange(frameExchange), + mInputMailbox(inputMailbox), + mConfig(config) +{ +} + RenderThread::~RenderThread() { Stop(); @@ -109,6 +118,7 @@ void RenderThread::ThreadMain() SimpleMotionRenderer renderer; RuntimeRenderScene runtimeRenderScene; Bgra8ReadbackPipeline readback; + InputFrameTexture inputTexture; if (!runtimeRenderScene.StartPrepareWorker(std::move(prepareWindow), error)) { SignalStartupFailure(error.empty() ? "Runtime shader prepare worker initialization failed." : error); @@ -145,9 +155,10 @@ void RenderThread::ThreadMain() } TryCommitReadyRuntimeShader(runtimeRenderScene); - if (!readback.RenderAndQueue(frameIndex, [this, &renderer, &runtimeRenderScene](uint64_t index) { + const GLuint videoInputTexture = inputTexture.PollAndUpload(mInputMailbox); + if (!readback.RenderAndQueue(frameIndex, [this, &renderer, &runtimeRenderScene, videoInputTexture](uint64_t index) { if (runtimeRenderScene.HasLayers()) - runtimeRenderScene.RenderFrame(index, mConfig.width, mConfig.height); + runtimeRenderScene.RenderFrame(index, mConfig.width, mConfig.height, videoInputTexture); else renderer.RenderFrame(index); })) @@ -175,6 +186,7 @@ void RenderThread::ThreadMain() } readback.Shutdown(); + inputTexture.ShutdownGl(); runtimeRenderScene.ShutdownGl(); renderer.ShutdownGl(); window.ClearCurrent(); diff --git a/apps/RenderCadenceCompositor/render/RenderThread.h b/apps/RenderCadenceCompositor/render/RenderThread.h index c280906..fefc0b7 100644 --- a/apps/RenderCadenceCompositor/render/RenderThread.h +++ b/apps/RenderCadenceCompositor/render/RenderThread.h @@ -14,6 +14,7 @@ #include class SystemFrameExchange; +class InputFrameMailbox; class RenderThread { @@ -39,6 +40,7 @@ public: }; RenderThread(SystemFrameExchange& frameExchange, Config config); + RenderThread(SystemFrameExchange& frameExchange, InputFrameMailbox* inputMailbox, Config config); RenderThread(const RenderThread&) = delete; RenderThread& operator=(const RenderThread&) = delete; ~RenderThread(); @@ -63,6 +65,7 @@ private: bool TryTakePendingRenderLayers(std::vector& layers); SystemFrameExchange& mFrameExchange; + InputFrameMailbox* mInputMailbox = nullptr; Config mConfig; std::thread mThread; std::atomic mStopping{ false }; diff --git a/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.cpp b/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.cpp index 663848f..530bd4f 100644 --- a/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.cpp +++ b/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.cpp @@ -120,7 +120,7 @@ bool RuntimeRenderScene::HasLayers() return false; } -void RuntimeRenderScene::RenderFrame(uint64_t frameIndex, unsigned width, unsigned height) +void RuntimeRenderScene::RenderFrame(uint64_t frameIndex, unsigned width, unsigned height, GLuint videoInputTexture) { ConsumePreparedPrograms(); @@ -148,18 +148,18 @@ void RuntimeRenderScene::RenderFrame(uint64_t frameIndex, unsigned width, unsign if (readyLayers.size() == 1) { - RenderLayer(*readyLayers.front(), frameIndex, width, height, 0, static_cast(outputFramebuffer), true); + RenderLayer(*readyLayers.front(), frameIndex, width, height, videoInputTexture, videoInputTexture, static_cast(outputFramebuffer), true); return; } if (!EnsureLayerTargets(width, height)) { glBindFramebuffer(GL_FRAMEBUFFER, static_cast(outputFramebuffer)); - RenderLayer(*readyLayers.back(), frameIndex, width, height, 0, static_cast(outputFramebuffer), true); + RenderLayer(*readyLayers.back(), frameIndex, width, height, videoInputTexture, videoInputTexture, static_cast(outputFramebuffer), true); return; } - GLuint layerInputTexture = 0; + GLuint layerInputTexture = videoInputTexture; std::size_t nextTargetIndex = 0; for (std::size_t layerIndex = 0; layerIndex < readyLayers.size(); ++layerIndex) { @@ -167,11 +167,11 @@ void RuntimeRenderScene::RenderFrame(uint64_t frameIndex, unsigned width, unsign if (isFinalLayer) { glBindFramebuffer(GL_FRAMEBUFFER, static_cast(outputFramebuffer)); - RenderLayer(*readyLayers[layerIndex], frameIndex, width, height, layerInputTexture, static_cast(outputFramebuffer), true); + RenderLayer(*readyLayers[layerIndex], frameIndex, width, height, videoInputTexture, layerInputTexture, static_cast(outputFramebuffer), true); continue; } - RenderLayer(*readyLayers[layerIndex], frameIndex, width, height, layerInputTexture, mLayerFramebuffers[nextTargetIndex], true); + RenderLayer(*readyLayers[layerIndex], frameIndex, width, height, videoInputTexture, layerInputTexture, mLayerFramebuffers[nextTargetIndex], true); layerInputTexture = mLayerTextures[nextTargetIndex]; nextTargetIndex = 1 - nextTargetIndex; } @@ -309,6 +309,7 @@ GLuint RuntimeRenderScene::RenderLayer( uint64_t frameIndex, unsigned width, unsigned height, + GLuint videoInputTexture, GLuint layerInputTexture, GLuint outputFramebuffer, bool renderToOutput) @@ -327,7 +328,11 @@ GLuint RuntimeRenderScene::RenderLayer( if (!pass.inputNames.empty()) { const std::string& inputName = pass.inputNames.front(); - if (inputName != "layerInput" && inputName != "videoInput") + if (inputName == "videoInput") + { + sourceTexture = videoInputTexture; + } + else if (inputName != "layerInput") { for (std::size_t index = 0; index < 2; ++index) { @@ -344,7 +349,7 @@ GLuint RuntimeRenderScene::RenderLayer( if (writesLayerOutput && renderToOutput) { glBindFramebuffer(GL_FRAMEBUFFER, outputFramebuffer); - pass.renderer->RenderFrame(frameIndex, width, height, sourceTexture, sourceTexture); + pass.renderer->RenderFrame(frameIndex, width, height, sourceTexture, layerInputTexture); lastOutputTexture = 0; continue; } @@ -355,7 +360,7 @@ GLuint RuntimeRenderScene::RenderLayer( const std::size_t targetIndex = nextTargetIndex; nextTargetIndex = nextTargetIndex == 2 ? 3 : 2; glBindFramebuffer(GL_FRAMEBUFFER, mLayerFramebuffers[targetIndex]); - pass.renderer->RenderFrame(frameIndex, width, height, sourceTexture, sourceTexture); + pass.renderer->RenderFrame(frameIndex, width, height, sourceTexture, layerInputTexture); const std::size_t namedIndex = targetIndex - 2; namedOutputs[namedIndex] = mLayerTextures[targetIndex]; namedOutputNames[namedIndex] = pass.outputName; diff --git a/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.h b/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.h index 25824bb..65bb5da 100644 --- a/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.h +++ b/apps/RenderCadenceCompositor/render/runtime/RuntimeRenderScene.h @@ -22,7 +22,7 @@ public: bool StartPrepareWorker(std::unique_ptr sharedWindow, std::string& error); bool CommitRenderLayers(const std::vector& layers, std::string& error); bool HasLayers(); - void RenderFrame(uint64_t frameIndex, unsigned width, unsigned height); + void RenderFrame(uint64_t frameIndex, unsigned width, unsigned height, GLuint videoInputTexture = 0); void ShutdownGl(); private: @@ -48,7 +48,7 @@ private: void TryCommitPendingPrograms(LayerProgram& layer); bool EnsureLayerTargets(unsigned width, unsigned height); void DestroyLayerTargets(); - GLuint RenderLayer(LayerProgram& layer, uint64_t frameIndex, unsigned width, unsigned height, GLuint layerInputTexture, GLuint outputFramebuffer, bool renderToOutput); + GLuint RenderLayer(LayerProgram& layer, uint64_t frameIndex, unsigned width, unsigned height, GLuint videoInputTexture, GLuint layerInputTexture, GLuint outputFramebuffer, bool renderToOutput); LayerProgram* FindLayer(const std::string& layerId); const LayerProgram* FindLayer(const std::string& layerId) const; LayerProgram::PassProgram* FindPass(LayerProgram& layer, const std::string& passId); diff --git a/apps/RenderCadenceCompositor/video/SyntheticInputProducer.cpp b/apps/RenderCadenceCompositor/video/SyntheticInputProducer.cpp new file mode 100644 index 0000000..c5f4e02 --- /dev/null +++ b/apps/RenderCadenceCompositor/video/SyntheticInputProducer.cpp @@ -0,0 +1,111 @@ +#include "SyntheticInputProducer.h" + +#include "VideoIOFormat.h" + +#include +#include + +namespace RenderCadenceCompositor +{ +SyntheticInputProducer::SyntheticInputProducer(InputFrameMailbox& mailbox, SyntheticInputProducerConfig config) : + mMailbox(mailbox), + mConfig(config) +{ +} + +SyntheticInputProducer::~SyntheticInputProducer() +{ + Stop(); +} + +bool SyntheticInputProducer::Start() +{ + if (mThread.joinable()) + return true; + if (mConfig.width == 0 || mConfig.height == 0) + return false; + + mStopping.store(false, std::memory_order_release); + mThread = std::thread([this]() { ThreadMain(); }); + return true; +} + +void SyntheticInputProducer::Stop() +{ + mStopping.store(true, std::memory_order_release); + if (mThread.joinable()) + mThread.join(); +} + +SyntheticInputProducerMetrics SyntheticInputProducer::Metrics() const +{ + SyntheticInputProducerMetrics metrics; + metrics.generatedFrames = mGeneratedFrames.load(std::memory_order_relaxed); + metrics.submitMisses = mSubmitMisses.load(std::memory_order_relaxed); + return metrics; +} + +void SyntheticInputProducer::ThreadMain() +{ + const unsigned rowBytes = VideoIORowBytes(VideoIOPixelFormat::Bgra8, mConfig.width); + std::vector buffer(static_cast(rowBytes) * static_cast(mConfig.height)); + const auto frameDuration = std::chrono::duration_cast( + std::chrono::duration(mConfig.frameDurationMilliseconds)); + + uint64_t frameIndex = 0; + auto nextFrameTime = std::chrono::steady_clock::now(); + while (!mStopping.load(std::memory_order_acquire)) + { + const auto now = std::chrono::steady_clock::now(); + if (now < nextFrameTime) + { + std::this_thread::sleep_for((std::min)(std::chrono::milliseconds(1), std::chrono::duration_cast(nextFrameTime - now))); + continue; + } + + GenerateFrame(frameIndex, buffer); + if (!mMailbox.SubmitFrame(buffer.data(), rowBytes, frameIndex)) + mSubmitMisses.fetch_add(1, std::memory_order_relaxed); + mGeneratedFrames.fetch_add(1, std::memory_order_relaxed); + ++frameIndex; + nextFrameTime += frameDuration; + if (std::chrono::steady_clock::now() - nextFrameTime > frameDuration * 4) + nextFrameTime = std::chrono::steady_clock::now() + frameDuration; + } +} + +void SyntheticInputProducer::GenerateFrame(uint64_t frameIndex, std::vector& buffer) const +{ + const float t = static_cast(frameIndex) / 60.0f; + const unsigned boxWidth = (std::max)(1u, mConfig.width / 5u); + const unsigned boxHeight = (std::max)(1u, mConfig.height / 6u); + const unsigned maxX = mConfig.width > boxWidth ? mConfig.width - boxWidth : 0u; + const unsigned maxY = mConfig.height > boxHeight ? mConfig.height - boxHeight : 0u; + const unsigned boxX = static_cast((0.5f + 0.5f * std::sin(t * 1.4f)) * static_cast(maxX)); + const unsigned boxY = static_cast((0.5f + 0.5f * std::sin(t * 0.9f + 1.2f)) * static_cast(maxY)); + const unsigned rowBytes = VideoIORowBytes(VideoIOPixelFormat::Bgra8, mConfig.width); + + for (unsigned y = 0; y < mConfig.height; ++y) + { + for (unsigned x = 0; x < mConfig.width; ++x) + { + const std::size_t offset = static_cast(y) * rowBytes + static_cast(x) * 4; + const unsigned checker = ((x / 80u) + (y / 80u) + static_cast(frameIndex / 15u)) & 1u; + unsigned char red = checker ? 42 : 16; + unsigned char green = checker ? 70 : 28; + unsigned char blue = checker ? 110 : 55; + if (x >= boxX && x < boxX + boxWidth && y >= boxY && y < boxY + boxHeight) + { + red = 245; + green = static_cast(160 + 60 * (0.5f + 0.5f * std::sin(t * 2.1f))); + blue = 35; + } + + buffer[offset + 0] = blue; + buffer[offset + 1] = green; + buffer[offset + 2] = red; + buffer[offset + 3] = 255; + } + } +} +} diff --git a/apps/RenderCadenceCompositor/video/SyntheticInputProducer.h b/apps/RenderCadenceCompositor/video/SyntheticInputProducer.h new file mode 100644 index 0000000..94cf0a5 --- /dev/null +++ b/apps/RenderCadenceCompositor/video/SyntheticInputProducer.h @@ -0,0 +1,49 @@ +#pragma once + +#include "../frames/InputFrameMailbox.h" + +#include +#include +#include +#include +#include + +namespace RenderCadenceCompositor +{ +struct SyntheticInputProducerConfig +{ + unsigned width = 1920; + unsigned height = 1080; + double frameDurationMilliseconds = 1000.0 / 59.94; +}; + +struct SyntheticInputProducerMetrics +{ + uint64_t generatedFrames = 0; + uint64_t submitMisses = 0; +}; + +class SyntheticInputProducer +{ +public: + SyntheticInputProducer(InputFrameMailbox& mailbox, SyntheticInputProducerConfig config); + SyntheticInputProducer(const SyntheticInputProducer&) = delete; + SyntheticInputProducer& operator=(const SyntheticInputProducer&) = delete; + ~SyntheticInputProducer(); + + bool Start(); + void Stop(); + SyntheticInputProducerMetrics Metrics() const; + +private: + void ThreadMain(); + void GenerateFrame(uint64_t frameIndex, std::vector& buffer) const; + + InputFrameMailbox& mMailbox; + SyntheticInputProducerConfig mConfig; + std::thread mThread; + std::atomic mStopping{ false }; + std::atomic mGeneratedFrames{ 0 }; + std::atomic mSubmitMisses{ 0 }; +}; +} diff --git a/tests/RenderCadenceCompositorInputFrameMailboxTests.cpp b/tests/RenderCadenceCompositorInputFrameMailboxTests.cpp new file mode 100644 index 0000000..bc6a28b --- /dev/null +++ b/tests/RenderCadenceCompositorInputFrameMailboxTests.cpp @@ -0,0 +1,109 @@ +#include "InputFrameMailbox.h" + +#include +#include +#include +#include + +namespace +{ +int gFailures = 0; + +void Expect(bool condition, const char* message) +{ + if (condition) + return; + + ++gFailures; + std::cerr << "FAIL: " << message << "\n"; +} + +InputFrameMailboxConfig MakeConfig(std::size_t capacity = 2) +{ + InputFrameMailboxConfig config; + config.width = 2; + config.height = 2; + config.pixelFormat = VideoIOPixelFormat::Bgra8; + config.rowBytes = VideoIORowBytes(config.pixelFormat, config.width); + config.capacity = capacity; + return config; +} + +std::vector MakeFrame(unsigned char value) +{ + return std::vector(16, value); +} + +void TestAcquireLatestDropsOlderReadyFrames() +{ + InputFrameMailbox mailbox(MakeConfig(3)); + const std::vector frame1 = MakeFrame(1); + const std::vector frame2 = MakeFrame(2); + const std::vector frame3 = MakeFrame(3); + + Expect(mailbox.SubmitFrame(frame1.data(), 8, 1), "first input frame submits"); + Expect(mailbox.SubmitFrame(frame2.data(), 8, 2), "second input frame submits"); + Expect(mailbox.SubmitFrame(frame3.data(), 8, 3), "third input frame submits"); + + InputFrame latest; + Expect(mailbox.TryAcquireLatest(latest), "latest input frame can be acquired"); + Expect(latest.frameIndex == 3, "mailbox returns newest frame"); + Expect(latest.bytes != nullptr && static_cast(latest.bytes)[0] == 3, "latest frame bytes match newest frame"); + Expect(mailbox.Release(latest), "latest input frame can be released"); + + const InputFrameMailboxMetrics metrics = mailbox.Metrics(); + Expect(metrics.droppedReadyFrames == 2, "older ready input frames are dropped after latest acquire"); + Expect(metrics.freeCount == 3, "all slots are free after release"); +} + +void TestSubmitDropsOldestWhenFull() +{ + InputFrameMailbox mailbox(MakeConfig(2)); + const std::vector frame1 = MakeFrame(1); + const std::vector frame2 = MakeFrame(2); + const std::vector frame3 = MakeFrame(3); + + Expect(mailbox.SubmitFrame(frame1.data(), 8, 1), "first frame submits into full test"); + Expect(mailbox.SubmitFrame(frame2.data(), 8, 2), "second frame submits into full test"); + Expect(mailbox.SubmitFrame(frame3.data(), 8, 3), "third frame submits by dropping oldest ready frame"); + + InputFrame latest; + Expect(mailbox.TryAcquireLatest(latest), "latest frame available after drop"); + Expect(latest.frameIndex == 3, "newest frame survived full mailbox"); + Expect(mailbox.Release(latest), "newest frame releases"); + + const InputFrameMailboxMetrics metrics = mailbox.Metrics(); + Expect(metrics.droppedReadyFrames >= 1, "full mailbox records ready drop"); + Expect(metrics.submitMisses == 0, "full mailbox did not block producer when ready slots were disposable"); +} + +void TestReadingFrameIsProtected() +{ + InputFrameMailbox mailbox(MakeConfig(1)); + const std::vector frame1 = MakeFrame(1); + const std::vector frame2 = MakeFrame(2); + + Expect(mailbox.SubmitFrame(frame1.data(), 8, 1), "protected frame submits"); + InputFrame acquired; + Expect(mailbox.TryAcquireLatest(acquired), "protected frame acquired"); + Expect(!mailbox.SubmitFrame(frame2.data(), 8, 2), "producer cannot overwrite frame currently being read"); + Expect(mailbox.Release(acquired), "protected frame releases"); + Expect(mailbox.SubmitFrame(frame2.data(), 8, 2), "producer can submit after release"); +} +} + +int main() +{ + TestAcquireLatestDropsOlderReadyFrames(); + TestSubmitDropsOldestWhenFull(); + TestReadingFrameIsProtected(); + + if (gFailures != 0) + { + std::cerr << gFailures << " RenderCadenceCompositorInputFrameMailbox test failure(s).\n"; + return 1; + } + + std::cout << "RenderCadenceCompositorInputFrameMailbox tests passed.\n"; + return 0; +}