diff --git a/SHADER_CONTRACT.md b/SHADER_CONTRACT.md index bf24fa6..c69594f 100644 --- a/SHADER_CONTRACT.md +++ b/SHADER_CONTRACT.md @@ -15,6 +15,7 @@ Each shader package lives under `shaders//` and includes: - `category` - `entryPoint` - `parameters` +- optional `temporal` Supported parameter types: @@ -24,6 +25,34 @@ Supported parameter types: - `bool` - `enum` +## Temporal manifests + +Shaders can optionally declare temporal history needs: + +```json +{ + "temporal": { + "enabled": true, + "historySource": "source", + "historyLength": 4 + } +} +``` + +Supported temporal history sources: + +- `source` - decoded source-video history from previous frames +- `preLayerInput` - history of the input arriving at that layer before the shader runs + +`historyLength` is requested by the shader and clamped by `config/runtime-host.json` via `maxTemporalHistoryFrames`. + +Temporal history resets automatically when: + +- layers are added, removed, or reordered +- a layer bypass state changes +- a layer changes to a different shader +- a shader is reloaded or recompiled + ## Slang contract The runtime owns the fragment entry point, the UYVY-to-RGBA decode pass, and final mix/bypass behavior. @@ -47,9 +76,13 @@ Available built-ins through `ShaderContext`: - `frameCount` - `mixAmount` - `bypass` +- `sourceHistoryLength` +- `temporalHistoryLength` Manifest parameters are exposed to the shader as globals named by their `id`. Helper function: - `sampleVideo(float2 uv)` returns decoded full-resolution RGBA video from the live DeckLink input. +- `sampleSourceHistory(int framesAgo, float2 uv)` samples the most recent available source history frame, clamping to the oldest available frame if needed. +- `sampleTemporalHistory(int framesAgo, float2 uv)` samples the most recent available pre-layer history frame for temporal shaders, clamping to the oldest available frame if needed. diff --git a/apps/LoopThroughWithOpenGLCompositing/GLExtensions.cpp b/apps/LoopThroughWithOpenGLCompositing/GLExtensions.cpp index 12e0c8d..f591c04 100644 --- a/apps/LoopThroughWithOpenGLCompositing/GLExtensions.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/GLExtensions.cpp @@ -82,6 +82,8 @@ PFNGLGETPROGRAMIVPROC glGetProgramiv; PFNGLGETPROGRAMINFOLOGPROC glGetProgramInfoLog; PFNGLUSEPROGRAMPROC glUseProgram; PFNGLGETUNIFORMLOCATIONPROC glGetUniformLocation; +PFNGLGETUNIFORMBLOCKINDEXPROC glGetUniformBlockIndex; +PFNGLUNIFORMBLOCKBINDINGPROC glUniformBlockBinding; PFNGLUNIFORM1IPROC glUniform1i; PFNGLUNIFORM1FPROC glUniform1f; PFNGLUNIFORM2FPROC glUniform2f; @@ -149,6 +151,8 @@ bool ResolveGLExtensions() glGetProgramInfoLog = (PFNGLGETPROGRAMINFOLOGPROC) wglGetProcAddress("glGetProgramInfoLog"); glUseProgram = (PFNGLUSEPROGRAMPROC) wglGetProcAddress("glUseProgram"); glGetUniformLocation = (PFNGLGETUNIFORMLOCATIONPROC) wglGetProcAddress("glGetUniformLocation"); + glGetUniformBlockIndex = (PFNGLGETUNIFORMBLOCKINDEXPROC) wglGetProcAddress("glGetUniformBlockIndex"); + glUniformBlockBinding = (PFNGLUNIFORMBLOCKBINDINGPROC) wglGetProcAddress("glUniformBlockBinding"); glUniform1i = (PFNGLUNIFORM1IPROC) wglGetProcAddress("glUniform1i"); glUniform1f = (PFNGLUNIFORM1FPROC) wglGetProcAddress("glUniform1f"); glUniform2f = (PFNGLUNIFORM2FPROC) wglGetProcAddress("glUniform2f"); @@ -192,6 +196,8 @@ bool ResolveGLExtensions() && glGetProgramInfoLog && glUseProgram && glGetUniformLocation + && glGetUniformBlockIndex + && glUniformBlockBinding && glUniform1i && glUniform1f && glUniform2f diff --git a/apps/LoopThroughWithOpenGLCompositing/GLExtensions.h b/apps/LoopThroughWithOpenGLCompositing/GLExtensions.h index 48d3d0d..88c0f03 100644 --- a/apps/LoopThroughWithOpenGLCompositing/GLExtensions.h +++ b/apps/LoopThroughWithOpenGLCompositing/GLExtensions.h @@ -112,6 +112,8 @@ typedef void (APIENTRYP PFNGLUNIFORM1FPROC) (GLint location, GLfloat v0); typedef void (APIENTRYP PFNGLUNIFORM1IPROC) (GLint location, GLint v0); typedef void (APIENTRYP PFNGLUNIFORM2FPROC) (GLint location, GLfloat v0, GLfloat v1); typedef void (APIENTRYP PFNGLUNIFORM4FPROC) (GLint location, GLfloat v0, GLfloat v1, GLfloat v2, GLfloat v3); +typedef GLuint (APIENTRYP PFNGLGETUNIFORMBLOCKINDEXPROC) (GLuint program, const GLchar* uniformBlockName); +typedef void (APIENTRYP PFNGLUNIFORMBLOCKBINDINGPROC) (GLuint program, GLuint uniformBlockIndex, GLuint uniformBlockBinding); typedef void (APIENTRYP PFNGLACTIVETEXTUREPROC) (GLenum texture); typedef void (APIENTRYP PFNGLBINDBUFFERBASEPROC) (GLenum target, GLuint index, GLuint buffer); typedef void (APIENTRYP PFNGLBUFFERSUBDATAPROC) (GLenum target, GLintptr offset, GLsizeiptr size, const GLvoid* data); @@ -171,6 +173,8 @@ extern PFNGLGETPROGRAMIVPROC glGetProgramiv; extern PFNGLGETPROGRAMINFOLOGPROC glGetProgramInfoLog; extern PFNGLUSEPROGRAMPROC glUseProgram; extern PFNGLGETUNIFORMLOCATIONPROC glGetUniformLocation; +extern PFNGLGETUNIFORMBLOCKINDEXPROC glGetUniformBlockIndex; +extern PFNGLUNIFORMBLOCKBINDINGPROC glUniformBlockBinding; extern PFNGLUNIFORM1IPROC glUniform1i; extern PFNGLUNIFORM1FPROC glUniform1f; extern PFNGLUNIFORM2FPROC glUniform2f; diff --git a/apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.cpp b/apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.cpp index 8c3db7e..fa271fb 100644 --- a/apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.cpp @@ -45,6 +45,8 @@ #include #include #include +#include +#include #include #include @@ -55,6 +57,7 @@ DEFINE_GUID(IID_PinnedMemoryAllocator, namespace { constexpr GLuint kDecodedVideoTextureUnit = 1; +constexpr GLuint kSourceHistoryTextureUnitBase = 2; constexpr GLuint kPackedVideoTextureUnit = 2; constexpr GLuint kGlobalParamsBindingPoint = 0; const char* kDisplayModeName = "1080p59.94"; @@ -149,23 +152,34 @@ void AppendStd140Vec4(std::vector& buffer, float x, float y, floa OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) : hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC), mCaptureDelegate(NULL), mPlayoutDelegate(NULL), - mDLInput(NULL), mDLOutput(NULL), + mDLInput(NULL), mDLOutput(NULL), mDLKeyer(NULL), mPlayoutAllocator(NULL), mFrameWidth(0), mFrameHeight(0), mHasNoInputSource(true), + mDeckLinkSupportsInternalKeying(false), + mDeckLinkSupportsExternalKeying(false), + mDeckLinkKeyerInterfaceAvailable(false), + mDeckLinkExternalKeyingActive(false), mFastTransferExtensionAvailable(false), mCaptureTexture(0), mDecodedTexture(0), mLayerTempTexture(0), mFBOTexture(0), + mUnpinnedTextureBuffer(0), mDecodeFrameBuf(0), mLayerTempFrameBuf(0), + mIdFrameBuf(0), + mIdColorBuf(0), + mIdDepthBuf(0), mFullscreenVAO(0), mGlobalParamsUBO(0), mDecodeProgram(0), mDecodeVertexShader(0), mDecodeFragmentShader(0), - mGlobalParamsUBOSize(0) + mGlobalParamsUBOSize(0), + mViewWidth(0), + mViewHeight(0), + mTemporalHistoryNeedsReset(true) { InitializeCriticalSection(&pMutex); mRuntimeHost = std::make_unique(); @@ -203,6 +217,13 @@ OpenGLComposite::~OpenGLComposite() if (mDLOutput != NULL) { + if (mDLKeyer != NULL) + { + mDLKeyer->Disable(); + mDLKeyer->Release(); + mDLKeyer = NULL; + } + mDLOutput->SetScheduledFrameCompletionCallback(NULL); mDLOutput->Release(); @@ -246,6 +267,7 @@ OpenGLComposite::~OpenGLComposite() if (mUnpinnedTextureBuffer != 0) glDeleteBuffers(1, &mUnpinnedTextureBuffer); + destroyTemporalHistoryResources(); destroyLayerPrograms(); destroyDecodeShaderProgram(); if (mControlServer) @@ -276,6 +298,9 @@ bool OpenGLComposite::InitDeckLink() while (pDLIterator->Next(&pDL) == S_OK) { int64_t duplexMode; + bool supportsInternalKeying = false; + bool supportsExternalKeying = false; + std::string modelName; if (result = pDL->QueryInterface(IID_IDeckLinkProfileAttributes, (void**)&deckLinkAttributes) != S_OK) { @@ -286,6 +311,20 @@ bool OpenGLComposite::InitDeckLink() } result = deckLinkAttributes->GetInt(BMDDeckLinkDuplex, &duplexMode); + BOOL attributeFlag = FALSE; + if (deckLinkAttributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &attributeFlag) == S_OK) + supportsInternalKeying = (attributeFlag != FALSE); + attributeFlag = FALSE; + if (deckLinkAttributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &attributeFlag) == S_OK) + supportsExternalKeying = (attributeFlag != FALSE); + BSTR modelNameBstr = NULL; + if (deckLinkAttributes->GetString(BMDDeckLinkModelName, &modelNameBstr) == S_OK && modelNameBstr != NULL) + { + _bstr_t modelNameWrapper(modelNameBstr, false); + const char* modelNameChars = modelNameWrapper; + if (modelNameChars != NULL) + modelName = modelNameChars; + } deckLinkAttributes->Release(); deckLinkAttributes = NULL; @@ -306,6 +345,12 @@ bool OpenGLComposite::InitDeckLink() { if (pDL->QueryInterface(IID_IDeckLinkOutput, (void**)&mDLOutput) != S_OK) mDLOutput = NULL; + else + { + mDeckLinkOutputModelName = modelName; + mDeckLinkSupportsInternalKeying = supportsInternalKeying; + mDeckLinkSupportsExternalKeying = supportsExternalKeying; + } } pDL->Release(); @@ -353,6 +398,21 @@ bool OpenGLComposite::InitDeckLink() if (! InitOpenGLState()) goto error; + if (mRuntimeHost) + { + mDeckLinkStatusMessage = mDeckLinkOutputModelName.empty() + ? "DeckLink output device selected." + : ("Selected output device: " + mDeckLinkOutputModelName); + mRuntimeHost->SetDeckLinkOutputStatus( + mDeckLinkOutputModelName, + mDeckLinkSupportsInternalKeying, + mDeckLinkSupportsExternalKeying, + mDeckLinkKeyerInterfaceAvailable, + mRuntimeHost->ExternalKeyingEnabled(), + mDeckLinkExternalKeyingActive, + mDeckLinkStatusMessage); + } + pDLDisplayMode->GetFrameRate(&mFrameDuration, &mFrameTimescale); // Resize window to match video frame, but scale large formats down by half for viewing @@ -392,6 +452,46 @@ bool OpenGLComposite::InitDeckLink() if (mDLOutput->EnableVideoOutput(displayMode, bmdVideoOutputFlagDefault) != S_OK) goto error; + if (mDLOutput->QueryInterface(IID_IDeckLinkKeyer, (void**)&mDLKeyer) == S_OK && mDLKeyer != NULL) + mDeckLinkKeyerInterfaceAvailable = true; + + if (mRuntimeHost && mRuntimeHost->ExternalKeyingEnabled()) + { + if (!mDeckLinkSupportsExternalKeying) + { + mDeckLinkStatusMessage = "External keying was requested, but the selected DeckLink output does not report external keying support."; + } + else if (!mDeckLinkKeyerInterfaceAvailable) + { + mDeckLinkStatusMessage = "External keying was requested, but the selected DeckLink output does not expose the IDeckLinkKeyer interface."; + } + else if (mDLKeyer->Enable(TRUE) != S_OK || mDLKeyer->SetLevel(255) != S_OK) + { + mDeckLinkStatusMessage = "External keying was requested, but enabling the DeckLink keyer failed."; + } + else + { + mDeckLinkExternalKeyingActive = true; + mDeckLinkStatusMessage = "External keying is active on the selected DeckLink output."; + } + } + else if (mDeckLinkSupportsExternalKeying) + { + mDeckLinkStatusMessage = "Selected DeckLink output supports external keying. Set enableExternalKeying to true in runtime-host.json to request it."; + } + + if (mRuntimeHost) + { + mRuntimeHost->SetDeckLinkOutputStatus( + mDeckLinkOutputModelName, + mDeckLinkSupportsInternalKeying, + mDeckLinkSupportsExternalKeying, + mDeckLinkKeyerInterfaceAvailable, + mRuntimeHost->ExternalKeyingEnabled(), + mDeckLinkExternalKeyingActive, + mDeckLinkStatusMessage); + } + // Create a queue of 10 IDeckLinkMutableVideoFrame objects to use for scheduling output video frames. // The ScheduledFrameCompleted() callback will immediately schedule a new frame using the next video frame from this queue. for (int i = 0; i < 10; i++) @@ -580,6 +680,7 @@ bool OpenGLComposite::InitOpenGLState() MessageBoxA(NULL, compilerErrorMessage, "OpenGL shader failed to load or compile", MB_OK); return false; } + resetTemporalHistoryState(); glClearColor( 0.0f, 0.0f, 0.0f, 0.5f ); // Black background glDisable(GL_DEPTH_TEST); @@ -769,8 +870,13 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, // Draw the effect output to the off-screen framebuffer. const auto renderStartTime = std::chrono::steady_clock::now(); + if (mFastTransferExtensionAvailable) + VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::GPUtoCPU); glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf); renderEffect(); + glFlush(); + if (mFastTransferExtensionAvailable) + VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::GPUtoCPU); const auto renderEndTime = std::chrono::steady_clock::now(); if (mRuntimeHost) { @@ -815,6 +921,7 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, } else { + glBindFramebuffer(GL_READ_FRAMEBUFFER, mIdFrameBuf); glReadPixels(0, 0, mFrameWidth, mFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame); paintGL(); } @@ -976,6 +1083,29 @@ bool OpenGLComposite::compileSingleLayerProgram(const RuntimeRenderState& state, return false; } + const GLuint globalParamsIndex = glGetUniformBlockIndex(newProgram, "GlobalParams"); + if (globalParamsIndex != GL_INVALID_INDEX) + glUniformBlockBinding(newProgram, globalParamsIndex, kGlobalParamsBindingPoint); + + const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; + glUseProgram(newProgram); + const GLint videoInputLocation = glGetUniformLocation(newProgram, "gVideoInput"); + if (videoInputLocation >= 0) + glUniform1i(videoInputLocation, static_cast(kDecodedVideoTextureUnit)); + for (unsigned index = 0; index < historyCap; ++index) + { + const std::string sourceSamplerName = "gSourceHistory" + std::to_string(index); + const GLint sourceSamplerLocation = glGetUniformLocation(newProgram, sourceSamplerName.c_str()); + if (sourceSamplerLocation >= 0) + glUniform1i(sourceSamplerLocation, static_cast(kSourceHistoryTextureUnitBase + index)); + + const std::string temporalSamplerName = "gTemporalHistory" + std::to_string(index); + const GLint temporalSamplerLocation = glGetUniformLocation(newProgram, temporalSamplerName.c_str()); + if (temporalSamplerLocation >= 0) + glUniform1i(temporalSamplerLocation, static_cast(kSourceHistoryTextureUnitBase + historyCap + index)); + } + glUseProgram(0); + layerProgram.layerId = state.layerId; layerProgram.shaderId = state.shaderId; layerProgram.program = newProgram; @@ -987,6 +1117,17 @@ bool OpenGLComposite::compileSingleLayerProgram(const RuntimeRenderState& state, bool OpenGLComposite::compileLayerPrograms(int errorMessageSize, char* errorMessage) { const std::vector layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mFrameWidth, mFrameHeight) : std::vector(); + std::string temporalError; + if (!validateTemporalTextureUnitBudget(temporalError)) + { + CopyErrorMessage(temporalError, errorMessageSize, errorMessage); + return false; + } + if (!ensureTemporalHistoryResources(layerStates, temporalError)) + { + CopyErrorMessage(temporalError, errorMessageSize, errorMessage); + return false; + } std::vector newPrograms; newPrograms.reserve(layerStates.size()); @@ -1115,6 +1256,211 @@ void OpenGLComposite::destroyDecodeShaderProgram() } } +bool OpenGLComposite::validateTemporalTextureUnitBudget(std::string& error) const +{ + const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; + GLint maxTextureUnits = 0; + glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureUnits); + const unsigned requiredUnits = kSourceHistoryTextureUnitBase + historyCap + historyCap; + const unsigned availableUnits = maxTextureUnits > 0 ? static_cast(maxTextureUnits) : 0u; + if (requiredUnits > availableUnits) + { + std::ostringstream message; + message << "Temporal history cap requires " << requiredUnits + << " fragment texture units, but only " << maxTextureUnits << " are available."; + error = message.str(); + return false; + } + return true; +} + +bool OpenGLComposite::createHistoryRing(HistoryRing& ring, unsigned effectiveLength, TemporalHistorySource historySource, std::string& error) +{ + destroyHistoryRing(ring); + ring.effectiveLength = effectiveLength; + ring.historySource = historySource; + if (effectiveLength == 0) + return true; + + ring.slots.resize(effectiveLength); + for (HistorySlot& slot : ring.slots) + { + glGenTextures(1, &slot.texture); + glBindTexture(GL_TEXTURE_2D, slot.texture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mFrameWidth, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); + + glGenFramebuffers(1, &slot.framebuffer); + glBindFramebuffer(GL_FRAMEBUFFER, slot.framebuffer); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, slot.texture, 0); + const GLenum framebufferStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER); + if (framebufferStatus != GL_FRAMEBUFFER_COMPLETE) + { + error = "Failed to initialize a temporal history framebuffer."; + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glBindTexture(GL_TEXTURE_2D, 0); + destroyHistoryRing(ring); + return false; + } + } + + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glBindTexture(GL_TEXTURE_2D, 0); + return true; +} + +void OpenGLComposite::destroyHistoryRing(HistoryRing& ring) +{ + for (HistorySlot& slot : ring.slots) + { + if (slot.framebuffer != 0) + glDeleteFramebuffers(1, &slot.framebuffer); + if (slot.texture != 0) + glDeleteTextures(1, &slot.texture); + slot.framebuffer = 0; + slot.texture = 0; + } + ring.slots.clear(); + ring.nextWriteIndex = 0; + ring.filledCount = 0; + ring.effectiveLength = 0; + ring.historySource = TemporalHistorySource::None; +} + +void OpenGLComposite::destroyTemporalHistoryResources() +{ + destroyHistoryRing(mSourceHistoryRing); + for (auto& historyEntry : mPreLayerHistoryByLayerId) + destroyHistoryRing(historyEntry.second); + mPreLayerHistoryByLayerId.clear(); +} + +void OpenGLComposite::resetTemporalHistoryState() +{ + mSourceHistoryRing.nextWriteIndex = 0; + mSourceHistoryRing.filledCount = 0; + for (auto& historyEntry : mPreLayerHistoryByLayerId) + { + historyEntry.second.nextWriteIndex = 0; + historyEntry.second.filledCount = 0; + } + mTemporalHistoryNeedsReset = false; +} + +bool OpenGLComposite::ensureTemporalHistoryResources(const std::vector& layerStates, std::string& error) +{ + const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; + const bool sourceHistoryNeeded = std::any_of(layerStates.begin(), layerStates.end(), + [](const RuntimeRenderState& state) { return state.isTemporal && state.effectiveTemporalHistoryLength > 0; }); + const unsigned sourceHistoryLength = sourceHistoryNeeded ? historyCap : 0; + + if (mSourceHistoryRing.effectiveLength != sourceHistoryLength) + { + if (!createHistoryRing(mSourceHistoryRing, sourceHistoryLength, TemporalHistorySource::Source, error)) + return false; + mTemporalHistoryNeedsReset = true; + } + + std::set requiredPreLayerIds; + for (const RuntimeRenderState& state : layerStates) + { + if (!state.isTemporal || state.temporalHistorySource != TemporalHistorySource::PreLayerInput) + continue; + requiredPreLayerIds.insert(state.layerId); + auto historyIt = mPreLayerHistoryByLayerId.find(state.layerId); + if (historyIt == mPreLayerHistoryByLayerId.end() || historyIt->second.effectiveLength != state.effectiveTemporalHistoryLength) + { + HistoryRing replacement; + if (!createHistoryRing(replacement, state.effectiveTemporalHistoryLength, TemporalHistorySource::PreLayerInput, error)) + return false; + mPreLayerHistoryByLayerId[state.layerId] = std::move(replacement); + mTemporalHistoryNeedsReset = true; + } + } + + for (auto it = mPreLayerHistoryByLayerId.begin(); it != mPreLayerHistoryByLayerId.end();) + { + if (requiredPreLayerIds.find(it->first) == requiredPreLayerIds.end()) + { + destroyHistoryRing(it->second); + it = mPreLayerHistoryByLayerId.erase(it); + mTemporalHistoryNeedsReset = true; + } + else + { + ++it; + } + } + + if (mTemporalHistoryNeedsReset) + resetTemporalHistoryState(); + + return true; +} + +void OpenGLComposite::pushFramebufferToHistoryRing(GLuint sourceFramebuffer, HistoryRing& ring) +{ + if (ring.effectiveLength == 0 || ring.slots.empty()) + return; + + HistorySlot& targetSlot = ring.slots[ring.nextWriteIndex]; + glBindFramebuffer(GL_READ_FRAMEBUFFER, sourceFramebuffer); + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, targetSlot.framebuffer); + glBlitFramebuffer(0, 0, mFrameWidth, mFrameHeight, 0, 0, mFrameWidth, mFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); + ring.nextWriteIndex = (ring.nextWriteIndex + 1) % ring.slots.size(); + ring.filledCount = std::min(ring.filledCount + 1, ring.slots.size()); +} + +GLuint OpenGLComposite::resolveHistoryTexture(const HistoryRing& ring, GLuint fallbackTexture, std::size_t framesAgo) const +{ + if (ring.filledCount == 0 || ring.slots.empty()) + return fallbackTexture; + + const std::size_t clampedOffset = std::min(framesAgo, ring.filledCount - 1); + const std::size_t newestIndex = (ring.nextWriteIndex + ring.slots.size() - 1) % ring.slots.size(); + const std::size_t slotIndex = (newestIndex + ring.slots.size() - clampedOffset) % ring.slots.size(); + return ring.slots[slotIndex].texture != 0 ? ring.slots[slotIndex].texture : fallbackTexture; +} + +unsigned OpenGLComposite::sourceHistoryAvailableCount() const +{ + return static_cast(mSourceHistoryRing.filledCount); +} + +unsigned OpenGLComposite::temporalHistoryAvailableCountForLayer(const std::string& layerId) const +{ + auto it = mPreLayerHistoryByLayerId.find(layerId); + if (it == mPreLayerHistoryByLayerId.end()) + return 0; + return static_cast(it->second.filledCount); +} + +void OpenGLComposite::bindHistorySamplers(const RuntimeRenderState& state, GLuint currentSourceTexture) +{ + const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; + for (unsigned index = 0; index < historyCap; ++index) + { + glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + index); + glBindTexture(GL_TEXTURE_2D, resolveHistoryTexture(mSourceHistoryRing, currentSourceTexture, index)); + } + + const GLuint temporalBase = kSourceHistoryTextureUnitBase + historyCap; + const HistoryRing* temporalRing = nullptr; + auto it = mPreLayerHistoryByLayerId.find(state.layerId); + if (it != mPreLayerHistoryByLayerId.end()) + temporalRing = &it->second; + + for (unsigned index = 0; index < historyCap; ++index) + { + glActiveTexture(GL_TEXTURE0 + temporalBase + index); + glBindTexture(GL_TEXTURE_2D, temporalRing ? resolveHistoryTexture(*temporalRing, currentSourceTexture, index) : currentSourceTexture); + } + glActiveTexture(GL_TEXTURE0); +} + void OpenGLComposite::renderEffect() { PollRuntimeChanges(); @@ -1143,15 +1489,25 @@ void OpenGLComposite::renderEffect() else { GLuint sourceTexture = mDecodedTexture; + GLuint sourceFrameBuffer = mDecodeFrameBuf; for (std::size_t index = 0; index < layerStates.size() && index < mLayerPrograms.size(); ++index) { const std::size_t remaining = layerStates.size() - index; const bool writeToMain = (remaining % 2) == 1; renderShaderProgram(sourceTexture, writeToMain ? mIdFrameBuf : mLayerTempFrameBuf, mLayerPrograms[index], layerStates[index]); + if (layerStates[index].temporalHistorySource == TemporalHistorySource::PreLayerInput) + { + auto historyIt = mPreLayerHistoryByLayerId.find(layerStates[index].layerId); + if (historyIt != mPreLayerHistoryByLayerId.end()) + pushFramebufferToHistoryRing(sourceFrameBuffer, historyIt->second); + } sourceTexture = writeToMain ? mFBOTexture : mLayerTempTexture; + sourceFrameBuffer = writeToMain ? mIdFrameBuf : mLayerTempFrameBuf; } } + pushFramebufferToHistoryRing(mDecodeFrameBuf, mSourceHistoryRing); + if (mFastTransferExtensionAvailable) VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); } @@ -1163,12 +1519,22 @@ void OpenGLComposite::renderShaderProgram(GLuint sourceTexture, GLuint destinati glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); glActiveTexture(GL_TEXTURE0 + kDecodedVideoTextureUnit); glBindTexture(GL_TEXTURE_2D, sourceTexture); + bindHistorySamplers(state, sourceTexture); glBindVertexArray(mFullscreenVAO); glUseProgram(layerProgram.program); - updateGlobalParamsBuffer(state); + updateGlobalParamsBuffer(state, sourceHistoryAvailableCount(), temporalHistoryAvailableCountForLayer(state.layerId)); glDrawArrays(GL_TRIANGLES, 0, 3); glUseProgram(0); glBindVertexArray(0); + const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; + for (unsigned index = 0; index < historyCap; ++index) + { + glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + index); + glBindTexture(GL_TEXTURE_2D, 0); + glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + historyCap + index); + glBindTexture(GL_TEXTURE_2D, 0); + } + glActiveTexture(GL_TEXTURE0 + kDecodedVideoTextureUnit); glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0); } @@ -1228,6 +1594,7 @@ bool OpenGLComposite::PollRuntimeChanges() return false; } + resetTemporalHistoryState(); broadcastRuntimeState(); return true; } @@ -1238,7 +1605,7 @@ void OpenGLComposite::broadcastRuntimeState() mControlServer->BroadcastState(); } -bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state) +bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength) { std::vector buffer; buffer.reserve(512); @@ -1249,6 +1616,14 @@ bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state) AppendStd140Float(buffer, static_cast(state.frameCount)); AppendStd140Float(buffer, static_cast(state.mixAmount)); AppendStd140Float(buffer, static_cast(state.bypass)); + const unsigned effectiveSourceHistoryLength = availableSourceHistoryLength < state.effectiveTemporalHistoryLength + ? availableSourceHistoryLength + : state.effectiveTemporalHistoryLength; + const unsigned effectiveTemporalHistoryLength = (state.temporalHistorySource == TemporalHistorySource::PreLayerInput) + ? (availableTemporalHistoryLength < state.effectiveTemporalHistoryLength ? availableTemporalHistoryLength : state.effectiveTemporalHistoryLength) + : 0u; + AppendStd140Int(buffer, static_cast(effectiveSourceHistoryLength)); + AppendStd140Int(buffer, static_cast(effectiveTemporalHistoryLength)); for (const ShaderParameterDefinition& definition : state.parameterDefinitions) { @@ -1323,6 +1698,7 @@ bool OpenGLComposite::AddLayer(const std::string& shaderId, std::string& error) return false; ReloadShader(); + resetTemporalHistoryState(); broadcastRuntimeState(); return true; } @@ -1333,6 +1709,7 @@ bool OpenGLComposite::RemoveLayer(const std::string& layerId, std::string& error return false; ReloadShader(); + resetTemporalHistoryState(); broadcastRuntimeState(); return true; } @@ -1343,6 +1720,7 @@ bool OpenGLComposite::MoveLayer(const std::string& layerId, int direction, std:: return false; ReloadShader(); + resetTemporalHistoryState(); broadcastRuntimeState(); return true; } @@ -1353,6 +1731,7 @@ bool OpenGLComposite::MoveLayerToIndex(const std::string& layerId, std::size_t t return false; ReloadShader(); + resetTemporalHistoryState(); broadcastRuntimeState(); return true; } @@ -1363,6 +1742,7 @@ bool OpenGLComposite::SetLayerBypass(const std::string& layerId, bool bypassed, return false; ReloadShader(); + resetTemporalHistoryState(); broadcastRuntimeState(); return true; } @@ -1373,6 +1753,7 @@ bool OpenGLComposite::SetLayerShader(const std::string& layerId, const std::stri return false; ReloadShader(); + resetTemporalHistoryState(); broadcastRuntimeState(); return true; } @@ -1414,6 +1795,7 @@ bool OpenGLComposite::LoadStackPreset(const std::string& presetName, std::string return false; ReloadShader(); + resetTemporalHistoryState(); broadcastRuntimeState(); return true; } diff --git a/apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.h b/apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.h index be0c9ec..1281fe2 100644 --- a/apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.h +++ b/apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.h @@ -110,6 +110,7 @@ private: // DeckLink IDeckLinkInput* mDLInput; IDeckLinkOutput* mDLOutput; + IDeckLinkKeyer* mDLKeyer; std::deque mDLOutputVideoFrameQueue; PinnedMemoryAllocator* mPlayoutAllocator; BMDTimeValue mFrameDuration; @@ -118,6 +119,12 @@ private: unsigned mFrameWidth; unsigned mFrameHeight; bool mHasNoInputSource; + std::string mDeckLinkOutputModelName; + bool mDeckLinkSupportsInternalKeying; + bool mDeckLinkSupportsExternalKeying; + bool mDeckLinkKeyerInterfaceAvailable; + bool mDeckLinkExternalKeyingActive; + std::string mDeckLinkStatusMessage; // OpenGL data bool mFastTransferExtensionAvailable; @@ -152,6 +159,25 @@ private: }; std::vector mLayerPrograms; + struct HistorySlot + { + GLuint texture = 0; + GLuint framebuffer = 0; + }; + + struct HistoryRing + { + std::vector slots; + std::size_t nextWriteIndex = 0; + std::size_t filledCount = 0; + unsigned effectiveLength = 0; + TemporalHistorySource historySource = TemporalHistorySource::None; + }; + + HistoryRing mSourceHistoryRing; + std::map mPreLayerHistoryByLayerId; + bool mTemporalHistoryNeedsReset; + bool InitOpenGLState(); bool compileLayerPrograms(int errorMessageSize, char* errorMessage); bool compileSingleLayerProgram(const RuntimeRenderState& state, LayerProgram& layerProgram, int errorMessageSize, char* errorMessage); @@ -164,7 +190,18 @@ private: void renderEffect(); bool PollRuntimeChanges(); void broadcastRuntimeState(); - bool updateGlobalParamsBuffer(const RuntimeRenderState& state); + bool updateGlobalParamsBuffer(const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength); + bool validateTemporalTextureUnitBudget(std::string& error) const; + bool ensureTemporalHistoryResources(const std::vector& layerStates, std::string& error); + bool createHistoryRing(HistoryRing& ring, unsigned effectiveLength, TemporalHistorySource historySource, std::string& error); + void destroyHistoryRing(HistoryRing& ring); + void destroyTemporalHistoryResources(); + void resetTemporalHistoryState(); + void pushFramebufferToHistoryRing(GLuint sourceFramebuffer, HistoryRing& ring); + void bindHistorySamplers(const RuntimeRenderState& state, GLuint currentSourceTexture); + GLuint resolveHistoryTexture(const HistoryRing& ring, GLuint fallbackTexture, std::size_t framesAgo) const; + unsigned sourceHistoryAvailableCount() const; + unsigned temporalHistoryAvailableCountForLayer(const std::string& layerId) const; }; //////////////////////////////////////////// diff --git a/apps/LoopThroughWithOpenGLCompositing/RuntimeHost.cpp b/apps/LoopThroughWithOpenGLCompositing/RuntimeHost.cpp index 8d986af..4a36101 100644 --- a/apps/LoopThroughWithOpenGLCompositing/RuntimeHost.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/RuntimeHost.cpp @@ -2,6 +2,7 @@ #include "RuntimeHost.h" #include +#include #include #include #include @@ -114,6 +115,21 @@ std::string SlangTypeForParameter(ShaderParameterType type) return "uniform float"; } +bool ParseTemporalHistorySource(const std::string& sourceName, TemporalHistorySource& source) +{ + if (sourceName == "source") + { + source = TemporalHistorySource::Source; + return true; + } + if (sourceName == "preLayerInput") + { + source = TemporalHistorySource::PreLayerInput; + return true; + } + return false; +} + bool ParseShaderParameterType(const std::string& typeName, ShaderParameterType& type) { if (typeName == "float") @@ -576,6 +592,19 @@ void RuntimeHost::SetSignalStatus(bool hasSignal, unsigned width, unsigned heigh mSignalModeName = modeName; } +void RuntimeHost::SetDeckLinkOutputStatus(const std::string& modelName, bool supportsInternalKeying, bool supportsExternalKeying, + bool keyerInterfaceAvailable, bool externalKeyingRequested, bool externalKeyingActive, const std::string& statusMessage) +{ + std::lock_guard lock(mMutex); + mDeckLinkOutputStatus.modelName = modelName; + mDeckLinkOutputStatus.supportsInternalKeying = supportsInternalKeying; + mDeckLinkOutputStatus.supportsExternalKeying = supportsExternalKeying; + mDeckLinkOutputStatus.keyerInterfaceAvailable = keyerInterfaceAvailable; + mDeckLinkOutputStatus.externalKeyingRequested = externalKeyingRequested; + mDeckLinkOutputStatus.externalKeyingActive = externalKeyingActive; + mDeckLinkOutputStatus.statusMessage = statusMessage; +} + void RuntimeHost::SetPerformanceStats(double frameBudgetMilliseconds, double renderMilliseconds) { std::lock_guard lock(mMutex); @@ -670,6 +699,10 @@ std::vector RuntimeHost::GetLayerRenderStates(unsigned outpu state.outputWidth = outputWidth; state.outputHeight = outputHeight; state.parameterDefinitions = shaderIt->second.parameters; + state.isTemporal = shaderIt->second.temporal.enabled; + state.temporalHistorySource = shaderIt->second.temporal.historySource; + state.requestedTemporalHistoryLength = shaderIt->second.temporal.requestedHistoryLength; + state.effectiveTemporalHistoryLength = shaderIt->second.temporal.effectiveHistoryLength; for (const ShaderParameterDefinition& definition : shaderIt->second.parameters) { @@ -716,6 +749,13 @@ bool RuntimeHost::LoadConfig(std::string& error) mConfig.serverPort = static_cast(serverPortValue->asNumber(mConfig.serverPort)); if (const JsonValue* autoReloadValue = configJson.find("autoReload")) mConfig.autoReload = autoReloadValue->asBoolean(mConfig.autoReload); + if (const JsonValue* maxTemporalHistoryFramesValue = configJson.find("maxTemporalHistoryFrames")) + { + const double configuredValue = maxTemporalHistoryFramesValue->asNumber(static_cast(mConfig.maxTemporalHistoryFrames)); + mConfig.maxTemporalHistoryFrames = configuredValue <= 0.0 ? 0u : static_cast(configuredValue); + } + if (const JsonValue* enableExternalKeyingValue = configJson.find("enableExternalKeying")) + mConfig.enableExternalKeying = enableExternalKeyingValue->asBoolean(mConfig.enableExternalKeying); mAutoReloadEnabled = mConfig.autoReload; return true; @@ -941,6 +981,51 @@ bool RuntimeHost::ParseShaderManifest(const std::filesystem::path& manifestPath, shaderPackage.shaderWriteTime = std::filesystem::last_write_time(shaderPackage.shaderPath); shaderPackage.manifestWriteTime = std::filesystem::last_write_time(shaderPackage.manifestPath); + if (const JsonValue* temporalValue = manifestJson.find("temporal")) + { + if (!temporalValue->isObject()) + { + error = "Shader manifest 'temporal' field must be an object in: " + manifestPath.string(); + return false; + } + + const JsonValue* enabledValue = temporalValue->find("enabled"); + if (enabledValue && enabledValue->asBoolean(false)) + { + const JsonValue* historySourceValue = temporalValue->find("historySource"); + const JsonValue* historyLengthValue = temporalValue->find("historyLength"); + if (!historySourceValue || Trim(historySourceValue->asString()).empty()) + { + error = "Temporal shader is missing required 'historySource' in: " + manifestPath.string(); + return false; + } + if (!historyLengthValue || !historyLengthValue->isNumber()) + { + error = "Temporal shader is missing required numeric 'historyLength' in: " + manifestPath.string(); + return false; + } + + TemporalHistorySource historySource = TemporalHistorySource::None; + if (!ParseTemporalHistorySource(historySourceValue->asString(), historySource)) + { + error = "Unsupported temporal historySource '" + historySourceValue->asString() + "' in: " + manifestPath.string(); + return false; + } + + const double requestedHistoryLength = historyLengthValue->asNumber(); + if (!IsFiniteNumber(requestedHistoryLength) || requestedHistoryLength <= 0.0 || std::floor(requestedHistoryLength) != requestedHistoryLength) + { + error = "Temporal shader 'historyLength' must be a positive integer in: " + manifestPath.string(); + return false; + } + + shaderPackage.temporal.enabled = true; + shaderPackage.temporal.historySource = historySource; + shaderPackage.temporal.requestedHistoryLength = static_cast(requestedHistoryLength); + shaderPackage.temporal.effectiveHistoryLength = std::min(shaderPackage.temporal.requestedHistoryLength, mConfig.maxTemporalHistoryFrames); + } + } + const JsonValue* parametersValue = manifestJson.find("parameters"); if (parametersValue && parametersValue->isArray()) { @@ -1181,6 +1266,8 @@ std::string RuntimeHost::BuildWrapperSlangSource(const ShaderPackage& shaderPack source << "\tfloat frameCount;\n"; source << "\tfloat mixAmount;\n"; source << "\tfloat bypass;\n"; + source << "\tint sourceHistoryLength;\n"; + source << "\tint temporalHistoryLength;\n"; source << "};\n\n"; source << "cbuffer GlobalParams\n"; source << "{\n"; @@ -1190,14 +1277,53 @@ std::string RuntimeHost::BuildWrapperSlangSource(const ShaderPackage& shaderPack source << "\tfloat gFrameCount;\n"; source << "\tfloat gMixAmount;\n"; source << "\tfloat gBypass;\n"; + source << "\tint gSourceHistoryLength;\n"; + source << "\tint gTemporalHistoryLength;\n"; for (const ShaderParameterDefinition& definition : shaderPackage.parameters) source << "\t" << SlangTypeForParameter(definition.type).substr(strlen("uniform ")) << " " << definition.id << ";\n"; source << "};\n\n"; - source << "Sampler2D gVideoInput;\n\n"; + source << "Sampler2D gVideoInput;\n"; + for (unsigned index = 0; index < mConfig.maxTemporalHistoryFrames; ++index) + source << "Sampler2D gSourceHistory" << index << ";\n"; + for (unsigned index = 0; index < mConfig.maxTemporalHistoryFrames; ++index) + source << "Sampler2D gTemporalHistory" << index << ";\n"; + source << "\n"; source << "float4 sampleVideo(float2 tc)\n"; source << "{\n"; source << "\treturn gVideoInput.Sample(tc);\n"; source << "}\n\n"; + source << "float4 sampleSourceHistory(int framesAgo, float2 tc)\n"; + source << "{\n"; + source << "\tif (gSourceHistoryLength <= 0)\n"; + source << "\t\treturn sampleVideo(tc);\n"; + source << "\tint clampedIndex = framesAgo;\n"; + source << "\tif (clampedIndex < 0)\n"; + source << "\t\tclampedIndex = 0;\n"; + source << "\tif (clampedIndex >= gSourceHistoryLength)\n"; + source << "\t\tclampedIndex = gSourceHistoryLength - 1;\n"; + source << "\tswitch (clampedIndex)\n"; + source << "\t{\n"; + for (unsigned index = 0; index < mConfig.maxTemporalHistoryFrames; ++index) + source << "\tcase " << index << ": return gSourceHistory" << index << ".Sample(tc);\n"; + source << "\tdefault: return sampleVideo(tc);\n"; + source << "\t}\n"; + source << "}\n\n"; + source << "float4 sampleTemporalHistory(int framesAgo, float2 tc)\n"; + source << "{\n"; + source << "\tif (gTemporalHistoryLength <= 0)\n"; + source << "\t\treturn sampleVideo(tc);\n"; + source << "\tint clampedIndex = framesAgo;\n"; + source << "\tif (clampedIndex < 0)\n"; + source << "\t\tclampedIndex = 0;\n"; + source << "\tif (clampedIndex >= gTemporalHistoryLength)\n"; + source << "\t\tclampedIndex = gTemporalHistoryLength - 1;\n"; + source << "\tswitch (clampedIndex)\n"; + source << "\t{\n"; + for (unsigned index = 0; index < mConfig.maxTemporalHistoryFrames; ++index) + source << "\tcase " << index << ": return gTemporalHistory" << index << ".Sample(tc);\n"; + source << "\tdefault: return sampleVideo(tc);\n"; + source << "\t}\n"; + source << "}\n\n"; source << "#include \"" << shaderPackage.shaderPath.generic_string() << "\"\n\n"; source << "[shader(\"fragment\")]\n"; source << "float4 fragmentMain(FragmentInput input) : SV_Target\n"; @@ -1211,6 +1337,8 @@ std::string RuntimeHost::BuildWrapperSlangSource(const ShaderPackage& shaderPack source << "\tcontext.frameCount = gFrameCount;\n"; source << "\tcontext.mixAmount = gMixAmount;\n"; source << "\tcontext.bypass = gBypass;\n"; + source << "\tcontext.sourceHistoryLength = gSourceHistoryLength;\n"; + source << "\tcontext.temporalHistoryLength = gTemporalHistoryLength;\n"; source << "\tfloat4 effectedColor = " << shaderPackage.entryPoint << "(context);\n"; source << "\tfloat mixValue = clamp(gBypass > 0.5 ? 0.0 : gMixAmount, 0.0, 1.0);\n"; source << "\treturn lerp(context.sourceColor, effectedColor, mixValue);\n"; @@ -1372,6 +1500,8 @@ JsonValue RuntimeHost::BuildStateValue() const JsonValue app = JsonValue::MakeObject(); app.set("serverPort", JsonValue(static_cast(mServerPort))); app.set("autoReload", JsonValue(mAutoReloadEnabled)); + app.set("maxTemporalHistoryFrames", JsonValue(static_cast(mConfig.maxTemporalHistoryFrames))); + app.set("enableExternalKeying", JsonValue(mConfig.enableExternalKeying)); root.set("app", app); JsonValue runtime = JsonValue::MakeObject(); @@ -1387,6 +1517,16 @@ JsonValue RuntimeHost::BuildStateValue() const video.set("modeName", JsonValue(mSignalModeName)); root.set("video", video); + JsonValue deckLink = JsonValue::MakeObject(); + deckLink.set("modelName", JsonValue(mDeckLinkOutputStatus.modelName)); + deckLink.set("supportsInternalKeying", JsonValue(mDeckLinkOutputStatus.supportsInternalKeying)); + deckLink.set("supportsExternalKeying", JsonValue(mDeckLinkOutputStatus.supportsExternalKeying)); + deckLink.set("keyerInterfaceAvailable", JsonValue(mDeckLinkOutputStatus.keyerInterfaceAvailable)); + deckLink.set("externalKeyingRequested", JsonValue(mDeckLinkOutputStatus.externalKeyingRequested)); + deckLink.set("externalKeyingActive", JsonValue(mDeckLinkOutputStatus.externalKeyingActive)); + deckLink.set("statusMessage", JsonValue(mDeckLinkOutputStatus.statusMessage)); + root.set("decklink", deckLink); + JsonValue performance = JsonValue::MakeObject(); performance.set("frameBudgetMs", JsonValue(mFrameBudgetMilliseconds)); performance.set("renderMs", JsonValue(mRenderMilliseconds)); @@ -1406,6 +1546,15 @@ JsonValue RuntimeHost::BuildStateValue() const shader.set("name", JsonValue(shaderIt->second.displayName)); shader.set("description", JsonValue(shaderIt->second.description)); shader.set("category", JsonValue(shaderIt->second.category)); + if (shaderIt->second.temporal.enabled) + { + JsonValue temporal = JsonValue::MakeObject(); + temporal.set("enabled", JsonValue(true)); + temporal.set("historySource", JsonValue(TemporalHistorySourceToString(shaderIt->second.temporal.historySource))); + temporal.set("requestedHistoryLength", JsonValue(static_cast(shaderIt->second.temporal.requestedHistoryLength))); + temporal.set("effectiveHistoryLength", JsonValue(static_cast(shaderIt->second.temporal.effectiveHistoryLength))); + shader.set("temporal", temporal); + } shaderLibrary.pushBack(shader); } root.set("shaders", shaderLibrary); @@ -1434,6 +1583,15 @@ JsonValue RuntimeHost::SerializeLayerStackLocked() const layerValue.set("shaderId", JsonValue(layer.shaderId)); layerValue.set("shaderName", JsonValue(shaderIt->second.displayName)); layerValue.set("bypass", JsonValue(layer.bypass)); + if (shaderIt->second.temporal.enabled) + { + JsonValue temporal = JsonValue::MakeObject(); + temporal.set("enabled", JsonValue(true)); + temporal.set("historySource", JsonValue(TemporalHistorySourceToString(shaderIt->second.temporal.historySource))); + temporal.set("requestedHistoryLength", JsonValue(static_cast(shaderIt->second.temporal.requestedHistoryLength))); + temporal.set("effectiveHistoryLength", JsonValue(static_cast(shaderIt->second.temporal.effectiveHistoryLength))); + layerValue.set("temporal", temporal); + } JsonValue parameters = JsonValue::MakeArray(); for (const ShaderParameterDefinition& definition : shaderIt->second.parameters) @@ -1615,6 +1773,20 @@ JsonValue RuntimeHost::SerializeParameterValue(const ShaderParameterDefinition& return JsonValue(); } +std::string RuntimeHost::TemporalHistorySourceToString(TemporalHistorySource source) const +{ + switch (source) + { + case TemporalHistorySource::Source: + return "source"; + case TemporalHistorySource::PreLayerInput: + return "preLayerInput"; + case TemporalHistorySource::None: + default: + return "none"; + } +} + RuntimeHost::LayerPersistentState* RuntimeHost::FindLayerById(const std::string& layerId) { auto it = std::find_if(mPersistentState.layers.begin(), mPersistentState.layers.end(), diff --git a/apps/LoopThroughWithOpenGLCompositing/RuntimeHost.h b/apps/LoopThroughWithOpenGLCompositing/RuntimeHost.h index 090d7ca..15041af 100644 --- a/apps/LoopThroughWithOpenGLCompositing/RuntimeHost.h +++ b/apps/LoopThroughWithOpenGLCompositing/RuntimeHost.h @@ -45,6 +45,21 @@ struct ShaderParameterValue std::string enumValue; }; +enum class TemporalHistorySource +{ + None, + Source, + PreLayerInput +}; + +struct TemporalSettings +{ + bool enabled = false; + TemporalHistorySource historySource = TemporalHistorySource::None; + unsigned requestedHistoryLength = 0; + unsigned effectiveHistoryLength = 0; +}; + struct ShaderPackage { std::string id; @@ -56,6 +71,7 @@ struct ShaderPackage std::filesystem::path shaderPath; std::filesystem::path manifestPath; std::vector parameters; + TemporalSettings temporal; std::filesystem::file_time_type shaderWriteTime; std::filesystem::file_time_type manifestWriteTime; }; @@ -74,6 +90,10 @@ struct RuntimeRenderState unsigned inputHeight = 0; unsigned outputWidth = 0; unsigned outputHeight = 0; + bool isTemporal = false; + TemporalHistorySource temporalHistorySource = TemporalHistorySource::None; + unsigned requestedTemporalHistoryLength = 0; + unsigned effectiveTemporalHistoryLength = 0; }; class RuntimeHost @@ -100,6 +120,8 @@ public: void SetCompileStatus(bool succeeded, const std::string& message); void SetSignalStatus(bool hasSignal, unsigned width, unsigned height, const std::string& modeName); + void SetDeckLinkOutputStatus(const std::string& modelName, bool supportsInternalKeying, bool supportsExternalKeying, + bool keyerInterfaceAvailable, bool externalKeyingRequested, bool externalKeyingActive, const std::string& statusMessage); void SetPerformanceStats(double frameBudgetMilliseconds, double renderMilliseconds); void AdvanceFrame(); @@ -111,6 +133,8 @@ public: const std::filesystem::path& GetUiRoot() const { return mUiRoot; } const std::filesystem::path& GetRuntimeRoot() const { return mRuntimeRoot; } unsigned short GetServerPort() const { return mServerPort; } + unsigned GetMaxTemporalHistoryFrames() const { return mConfig.maxTemporalHistoryFrames; } + bool ExternalKeyingEnabled() const { return mConfig.enableExternalKeying; } void SetServerPort(unsigned short port); bool AutoReloadEnabled() const { return mAutoReloadEnabled; } @@ -120,6 +144,19 @@ private: std::string shaderLibrary = "shaders"; unsigned short serverPort = 8080; bool autoReload = true; + unsigned maxTemporalHistoryFrames = 4; + bool enableExternalKeying = false; + }; + + struct DeckLinkOutputStatus + { + std::string modelName; + bool supportsInternalKeying = false; + bool supportsExternalKeying = false; + bool keyerInterfaceAvailable = false; + bool externalKeyingRequested = false; + bool externalKeyingActive = false; + std::string statusMessage; }; struct LayerPersistentState @@ -156,6 +193,7 @@ private: std::vector GetStackPresetNamesLocked() const; std::string MakeSafePresetFileStem(const std::string& presetName) const; JsonValue SerializeParameterValue(const ShaderParameterDefinition& definition, const ShaderParameterValue& value) const; + std::string TemporalHistorySourceToString(TemporalHistorySource source) const; LayerPersistentState* FindLayerById(const std::string& layerId); const LayerPersistentState* FindLayerById(const std::string& layerId) const; std::string GenerateLayerId(); @@ -186,6 +224,7 @@ private: double mFrameBudgetMilliseconds; double mRenderMilliseconds; double mSmoothedRenderMilliseconds; + DeckLinkOutputStatus mDeckLinkOutputStatus; unsigned short mServerPort; bool mAutoReloadEnabled; std::chrono::steady_clock::time_point mStartTime; diff --git a/config/runtime-host.json b/config/runtime-host.json index e3a315f..e9fd108 100644 --- a/config/runtime-host.json +++ b/config/runtime-host.json @@ -1,5 +1,6 @@ { "shaderLibrary": "shaders", "serverPort": 8080, - "autoReload": true + "autoReload": true, + "maxTemporalHistoryFrames": 12 } diff --git a/shaders/temporal-ghost-trail/shader.json b/shaders/temporal-ghost-trail/shader.json new file mode 100644 index 0000000..3b9a7e7 --- /dev/null +++ b/shaders/temporal-ghost-trail/shader.json @@ -0,0 +1,32 @@ +{ + "id": "temporal-ghost-trail", + "name": "Temporal Ghost Trail", + "description": "Blends older pre-layer input frames into the current layer input for a soft temporal trail.", + "category": "Built-in", + "entryPoint": "shadeVideo", + "temporal": { + "enabled": true, + "historySource": "preLayerInput", + "historyLength": 12 + }, + "parameters": [ + { + "id": "currentMix", + "label": "Current Mix", + "type": "float", + "default": 0.72, + "min": 0.0, + "max": 1.0, + "step": 0.01 + }, + { + "id": "trailMix", + "label": "Trail Mix", + "type": "float", + "default": 0.28, + "min": 0.0, + "max": 1.0, + "step": 0.01 + } + ] +} diff --git a/shaders/temporal-ghost-trail/shader.slang b/shaders/temporal-ghost-trail/shader.slang new file mode 100644 index 0000000..09e966b --- /dev/null +++ b/shaders/temporal-ghost-trail/shader.slang @@ -0,0 +1,11 @@ +float4 shadeVideo(ShaderContext context) +{ + float4 history0 = sampleTemporalHistory(1, context.uv); + float4 history1 = sampleTemporalHistory(3, context.uv); + float4 history2 = sampleTemporalHistory(5, context.uv); + float4 trail = history0 * 0.5 + history1 * 0.3 + history2 * 0.2; + float trailAmount = clamp(trailMix, 0.0, 1.0); + float currentAmount = clamp(currentMix, 0.0, 1.0); + float weightSum = max(0.0001, currentAmount + trailAmount); + return saturate((context.sourceColor * currentAmount + trail * trailAmount) / weightSum); +} diff --git a/shaders/temporal-low-fps/shader.json b/shaders/temporal-low-fps/shader.json new file mode 100644 index 0000000..d25ba51 --- /dev/null +++ b/shaders/temporal-low-fps/shader.json @@ -0,0 +1,32 @@ +{ + "id": "temporal-low-fps", + "name": "Temporal Low FPS", + "description": "Holds older source frames to create a deliberate choppy playback look.", + "category": "Built-in", + "entryPoint": "shadeVideo", + "temporal": { + "enabled": true, + "historySource": "source", + "historyLength": 8 + }, + "parameters": [ + { + "id": "holdFrames", + "label": "Hold Frames", + "type": "float", + "default": 3.0, + "min": 0.0, + "max": 7.0, + "step": 0.1 + }, + { + "id": "blendAmount", + "label": "Blend", + "type": "float", + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.01 + } + ] +} diff --git a/shaders/temporal-low-fps/shader.slang b/shaders/temporal-low-fps/shader.slang new file mode 100644 index 0000000..d10e289 --- /dev/null +++ b/shaders/temporal-low-fps/shader.slang @@ -0,0 +1,15 @@ +float4 shadeVideo(ShaderContext context) +{ + float clampedHoldFrames = clamp(holdFrames, 0.0, 7.0); + int lowerHoldLength = int(floor(clampedHoldFrames)) + 1; + int upperHoldLength = min(lowerHoldLength + 1, 8); + float fractional = frac(clampedHoldFrames); + + int lowerPhase = int(context.frameCount) % lowerHoldLength; + int upperPhase = int(context.frameCount) % upperHoldLength; + + float4 lowerHeld = lowerPhase == 0 ? context.sourceColor : sampleSourceHistory(lowerPhase - 1, context.uv); + float4 upperHeld = upperPhase == 0 ? context.sourceColor : sampleSourceHistory(upperPhase - 1, context.uv); + float4 held = lerp(lowerHeld, upperHeld, fractional); + return lerp(context.sourceColor, held, clamp(blendAmount, 0.0, 1.0)); +} diff --git a/ui/src/App.jsx b/ui/src/App.jsx index 3e8d599..ed3e65e 100644 --- a/ui/src/App.jsx +++ b/ui/src/App.jsx @@ -391,6 +391,20 @@ function LayerCard({ + {layer.temporal?.enabled ? ( +
+ +
+ {layer.temporal.historySource} history, requested {layer.temporal.requestedHistoryLength} frame{layer.temporal.requestedHistoryLength === 1 ? "" : "s"}, using {layer.temporal.effectiveHistoryLength} +
+
+ ) : ( +
+ +
Stateless shader
+
+ )} +

Parameters