shader validation checks

.gitea/workflows/ci.yml

@@ -59,7 +59,7 @@ jobs:
         shell: powershell
         run: cmake --build --preset build-debug
 
-      - name: Run Native Tests
+      - name: Run Native Tests And Shader Validation
         shell: powershell
         run: cmake --build --preset build-debug --target RUN_TESTS
 

CMakeLists.txt

@@ -31,7 +31,7 @@ if(NOT EXISTS "${SLANG_LICENSE_FILE}")
 endif()
 
 set(APP_SOURCES
-	"${APP_DIR}/DeckLinkAPI_i.c"
+	"${APP_DIR}/videoio/decklink/DeckLinkAPI_i.c"
 	"${APP_DIR}/control/ControlServer.cpp"
 	"${APP_DIR}/control/ControlServer.h"
 	"${APP_DIR}/control/OscServer.cpp"
@@ -40,48 +40,51 @@ set(APP_SOURCES
 	"${APP_DIR}/control/RuntimeControlBridge.h"
 	"${APP_DIR}/control/RuntimeServices.cpp"
 	"${APP_DIR}/control/RuntimeServices.h"
-	"${APP_DIR}/decklink/DeckLinkDisplayMode.cpp"
+	"${APP_DIR}/videoio/decklink/DeckLinkAPI_h.h"
-	"${APP_DIR}/decklink/DeckLinkDisplayMode.h"
+	"${APP_DIR}/videoio/decklink/DeckLinkDisplayMode.cpp"
-	"${APP_DIR}/decklink/DeckLinkFrameTransfer.cpp"
+	"${APP_DIR}/videoio/decklink/DeckLinkDisplayMode.h"
-	"${APP_DIR}/decklink/DeckLinkFrameTransfer.h"
+	"${APP_DIR}/videoio/decklink/DeckLinkFrameTransfer.cpp"
-	"${APP_DIR}/decklink/DeckLinkSession.cpp"
+	"${APP_DIR}/videoio/decklink/DeckLinkFrameTransfer.h"
-	"${APP_DIR}/decklink/DeckLinkSession.h"
+	"${APP_DIR}/videoio/decklink/DeckLinkSession.cpp"
-	"${APP_DIR}/decklink/VideoIOFormat.cpp"
+	"${APP_DIR}/videoio/decklink/DeckLinkSession.h"
-	"${APP_DIR}/decklink/VideoIOFormat.h"
+	"${APP_DIR}/videoio/decklink/DeckLinkVideoIOFormat.cpp"
-	"${APP_DIR}/gl/GLExtensions.cpp"
+	"${APP_DIR}/videoio/decklink/DeckLinkVideoIOFormat.h"
-	"${APP_DIR}/gl/GLExtensions.h"
+	"${APP_DIR}/gl/renderer/GLExtensions.cpp"
-	"${APP_DIR}/gl/GlobalParamsBuffer.cpp"
+	"${APP_DIR}/gl/renderer/GLExtensions.h"
-	"${APP_DIR}/gl/GlobalParamsBuffer.h"
+	"${APP_DIR}/gl/shader/GlobalParamsBuffer.cpp"
-	"${APP_DIR}/gl/GlRenderConstants.h"
+	"${APP_DIR}/gl/shader/GlobalParamsBuffer.h"
-	"${APP_DIR}/gl/GlScopedObjects.h"
+	"${APP_DIR}/gl/renderer/GlRenderConstants.h"
-	"${APP_DIR}/gl/GlShaderSources.cpp"
+	"${APP_DIR}/gl/renderer/GlScopedObjects.h"
-	"${APP_DIR}/gl/GlShaderSources.h"
+	"${APP_DIR}/gl/shader/GlShaderSources.cpp"
+	"${APP_DIR}/gl/shader/GlShaderSources.h"
 	"${APP_DIR}/gl/OpenGLComposite.cpp"
 	"${APP_DIR}/gl/OpenGLComposite.h"
 	"${APP_DIR}/gl/OpenGLCompositeRuntimeControls.cpp"
-	"${APP_DIR}/gl/OpenGLDeckLinkBridge.cpp"
+	"${APP_DIR}/gl/pipeline/OpenGLRenderPass.cpp"
-	"${APP_DIR}/gl/OpenGLDeckLinkBridge.h"
+	"${APP_DIR}/gl/pipeline/OpenGLRenderPass.h"
-	"${APP_DIR}/gl/OpenGLRenderPass.cpp"
+	"${APP_DIR}/gl/pipeline/OpenGLRenderPipeline.cpp"
-	"${APP_DIR}/gl/OpenGLRenderPass.h"
+	"${APP_DIR}/gl/pipeline/OpenGLRenderPipeline.h"
-	"${APP_DIR}/gl/OpenGLRenderer.cpp"
+	"${APP_DIR}/gl/renderer/OpenGLRenderer.cpp"
-	"${APP_DIR}/gl/OpenGLRenderer.h"
+	"${APP_DIR}/gl/renderer/OpenGLRenderer.h"
-	"${APP_DIR}/gl/OpenGLShaderPrograms.cpp"
+	"${APP_DIR}/gl/pipeline/OpenGLVideoIOBridge.cpp"
-	"${APP_DIR}/gl/OpenGLShaderPrograms.h"
+	"${APP_DIR}/gl/pipeline/OpenGLVideoIOBridge.h"
-	"${APP_DIR}/gl/PngScreenshotWriter.cpp"
+	"${APP_DIR}/gl/shader/OpenGLShaderPrograms.cpp"
-	"${APP_DIR}/gl/PngScreenshotWriter.h"
+	"${APP_DIR}/gl/shader/OpenGLShaderPrograms.h"
-	"${APP_DIR}/gl/ShaderProgramCompiler.cpp"
+	"${APP_DIR}/gl/pipeline/PngScreenshotWriter.cpp"
-	"${APP_DIR}/gl/ShaderProgramCompiler.h"
+	"${APP_DIR}/gl/pipeline/PngScreenshotWriter.h"
-	"${APP_DIR}/gl/ShaderBuildQueue.cpp"
+	"${APP_DIR}/gl/shader/ShaderProgramCompiler.cpp"
-	"${APP_DIR}/gl/ShaderBuildQueue.h"
+	"${APP_DIR}/gl/shader/ShaderProgramCompiler.h"
-	"${APP_DIR}/gl/ShaderTextureBindings.cpp"
+	"${APP_DIR}/gl/shader/ShaderBuildQueue.cpp"
-	"${APP_DIR}/gl/ShaderTextureBindings.h"
+	"${APP_DIR}/gl/shader/ShaderBuildQueue.h"
-	"${APP_DIR}/gl/Std140Buffer.h"
+	"${APP_DIR}/gl/shader/ShaderTextureBindings.cpp"
-	"${APP_DIR}/gl/TextRasterizer.cpp"
+	"${APP_DIR}/gl/shader/ShaderTextureBindings.h"
-	"${APP_DIR}/gl/TextRasterizer.h"
+	"${APP_DIR}/gl/shader/Std140Buffer.h"
-	"${APP_DIR}/gl/TextureAssetLoader.cpp"
+	"${APP_DIR}/gl/shader/TextRasterizer.cpp"
-	"${APP_DIR}/gl/TextureAssetLoader.h"
+	"${APP_DIR}/gl/shader/TextRasterizer.h"
-	"${APP_DIR}/gl/TemporalHistoryBuffers.cpp"
+	"${APP_DIR}/gl/shader/TextureAssetLoader.cpp"
-	"${APP_DIR}/gl/TemporalHistoryBuffers.h"
+	"${APP_DIR}/gl/shader/TextureAssetLoader.h"
+	"${APP_DIR}/gl/pipeline/TemporalHistoryBuffers.cpp"
+	"${APP_DIR}/gl/pipeline/TemporalHistoryBuffers.h"
 	"${APP_DIR}/LoopThroughWithOpenGLCompositing.cpp"
 	"${APP_DIR}/LoopThroughWithOpenGLCompositing.h"
 	"${APP_DIR}/LoopThroughWithOpenGLCompositing.rc"
@@ -104,6 +107,11 @@ set(APP_SOURCES
 	"${APP_DIR}/stdafx.cpp"
 	"${APP_DIR}/stdafx.h"
 	"${APP_DIR}/targetver.h"
+	"${APP_DIR}/videoio/VideoIOFormat.cpp"
+	"${APP_DIR}/videoio/VideoIOFormat.h"
+	"${APP_DIR}/videoio/VideoIOTypes.h"
+	"${APP_DIR}/videoio/VideoPlayoutScheduler.cpp"
+	"${APP_DIR}/videoio/VideoPlayoutScheduler.h"
 )
 
 add_executable(LoopThroughWithOpenGLCompositing WIN32 ${APP_SOURCES})
@@ -111,11 +119,15 @@ add_executable(LoopThroughWithOpenGLCompositing WIN32 ${APP_SOURCES})
 target_include_directories(LoopThroughWithOpenGLCompositing PRIVATE
 	"${APP_DIR}"
 	"${APP_DIR}/control"
-	"${APP_DIR}/decklink"
 	"${APP_DIR}/gl"
+	"${APP_DIR}/gl/pipeline"
+	"${APP_DIR}/gl/renderer"
+	"${APP_DIR}/gl/shader"
 	"${APP_DIR}/platform"
 	"${APP_DIR}/runtime"
 	"${APP_DIR}/shader"
+	"${APP_DIR}/videoio"
+	"${APP_DIR}/videoio/decklink"
 )
 
 target_link_libraries(LoopThroughWithOpenGLCompositing PRIVATE
@@ -196,6 +208,7 @@ add_executable(Std140BufferTests
 target_include_directories(Std140BufferTests PRIVATE
 	"${APP_DIR}"
 	"${APP_DIR}/gl"
+	"${APP_DIR}/gl/shader"
 )
 
 if(MSVC)
@@ -222,6 +235,29 @@ endif()
 
 add_test(NAME ShaderPackageRegistryTests COMMAND ShaderPackageRegistryTests)
 
+add_executable(ShaderSlangValidationTests
+	"${APP_DIR}/runtime/RuntimeJson.cpp"
+	"${APP_DIR}/shader/ShaderCompiler.cpp"
+	"${APP_DIR}/shader/ShaderPackageRegistry.cpp"
+	"${CMAKE_CURRENT_SOURCE_DIR}/tests/ShaderSlangValidationTests.cpp"
+)
+
+target_include_directories(ShaderSlangValidationTests PRIVATE
+	"${APP_DIR}"
+	"${APP_DIR}/platform"
+	"${APP_DIR}/runtime"
+	"${APP_DIR}/shader"
+)
+
+if(MSVC)
+	target_compile_options(ShaderSlangValidationTests PRIVATE /W3)
+endif()
+
+add_test(NAME ShaderSlangValidationTests COMMAND ShaderSlangValidationTests)
+set_tests_properties(ShaderSlangValidationTests PROPERTIES
+	ENVIRONMENT "SLANG_ROOT=${SLANG_ROOT}"
+)
+
 add_executable(OscServerTests
 	"${APP_DIR}/control/OscServer.cpp"
 	"${CMAKE_CURRENT_SOURCE_DIR}/tests/OscServerTests.cpp"
@@ -244,13 +280,15 @@ endif()
 add_test(NAME OscServerTests COMMAND OscServerTests)
 
 add_executable(VideoIOFormatTests
-	"${APP_DIR}/decklink/VideoIOFormat.cpp"
+	"${APP_DIR}/videoio/decklink/DeckLinkVideoIOFormat.cpp"
+	"${APP_DIR}/videoio/VideoIOFormat.cpp"
 	"${CMAKE_CURRENT_SOURCE_DIR}/tests/VideoIOFormatTests.cpp"
 )
 
 target_include_directories(VideoIOFormatTests PRIVATE
 	"${APP_DIR}"
-	"${APP_DIR}/decklink"
+	"${APP_DIR}/videoio"
+	"${APP_DIR}/videoio/decklink"
 )
 
 if(MSVC)
@@ -259,6 +297,40 @@ endif()
 
 add_test(NAME VideoIOFormatTests COMMAND VideoIOFormatTests)
 
+add_executable(VideoPlayoutSchedulerTests
+	"${APP_DIR}/videoio/VideoPlayoutScheduler.cpp"
+	"${CMAKE_CURRENT_SOURCE_DIR}/tests/VideoPlayoutSchedulerTests.cpp"
+)
+
+target_include_directories(VideoPlayoutSchedulerTests PRIVATE
+	"${APP_DIR}"
+	"${APP_DIR}/videoio"
+	"${APP_DIR}/videoio/decklink"
+)
+
+if(MSVC)
+	target_compile_options(VideoPlayoutSchedulerTests PRIVATE /W3)
+endif()
+
+add_test(NAME VideoPlayoutSchedulerTests COMMAND VideoPlayoutSchedulerTests)
+
+add_executable(VideoIODeviceFakeTests
+	"${APP_DIR}/videoio/VideoIOFormat.cpp"
+	"${CMAKE_CURRENT_SOURCE_DIR}/tests/VideoIODeviceFakeTests.cpp"
+)
+
+target_include_directories(VideoIODeviceFakeTests PRIVATE
+	"${APP_DIR}"
+	"${APP_DIR}/videoio"
+	"${APP_DIR}/videoio/decklink"
+)
+
+if(MSVC)
+	target_compile_options(VideoIODeviceFakeTests PRIVATE /W3)
+endif()
+
+add_test(NAME VideoIODeviceFakeTests COMMAND VideoIODeviceFakeTests)
+
 install(TARGETS LoopThroughWithOpenGLCompositing
 	RUNTIME DESTINATION "."
 )

README.md

@@ -1,6 +1,6 @@
 # Video Shader
 
-Native video shader host with an OpenGL/DeckLink render path, Slang shader packages, and a local React control UI.
+Native video shader host with an OpenGL render path, pluggable video I/O boundary, DeckLink backend, Slang shader packages, and a local React control UI.
 
 The app loads shader packages from `shaders/`, compiles Slang to GLSL at runtime, renders a configurable layer stack, and exposes a browser-based control surface over a local HTTP/WebSocket server.
 
@@ -15,12 +15,20 @@ The app loads shader packages from `shaders/`, compiles Slang to GLSL at runtime
 - `tests/`: focused native tests for pure runtime logic.
 - `.gitea/workflows/ci.yml`: Gitea Actions CI for Windows native tests and Ubuntu UI build.
 
+Native app internals are grouped by boundary:
+
+- `videoio/`: backend-neutral video I/O contracts, formats, and playout timing.
+- `videoio/decklink/`: DeckLink-specific device adapter, callbacks, and SDK bindings.
+- `gl/renderer/`: low-level OpenGL resources and extension helpers.
+- `gl/pipeline/`: frame pipeline, render passes, video I/O bridge, preview/readback, and screenshots.
+- `gl/shader/`: shader compilation, texture/text assets, UBO packing, and shader program ownership.
+
 ## Requirements
 
 - Windows with Visual Studio 2022 C++ tooling.
 - CMake 3.24 or newer.
 - Node.js and npm for the control UI.
-- Blackmagic Desktop Video drivers and a DeckLink device.
+- Blackmagic Desktop Video drivers and a DeckLink device for the current production video I/O backend.
 - Slang binary release with `slangc.exe`, `slang-compiler.dll`, `slang-glslang.dll`, and `LICENSE`.
 
 Default expected Slang path:
@@ -77,6 +85,8 @@ dist/VideoShader/
   shaders/
   3rdParty/slang/bin/
   ui/dist/
+  docs/
+  SHADER_CONTRACT.md
   runtime/templates/
   third_party_notices/
 ```
@@ -111,6 +121,8 @@ Current native test coverage includes:
 - JSON parsing and serialization.
 - Parameter normalization and preset filename safety.
 - Shader manifest parsing, temporal manifest validation, and package registry scanning.
+- Video I/O format helpers, v210 pack/unpack math, playout scheduler timing, and fake backend contract coverage.
+- OSC packet parsing.
 
 ## Runtime Configuration
 
@@ -131,7 +143,7 @@ Current native test coverage includes:
 }
 ```
 
-`inputVideoFormat`/`inputFrameRate` select the DeckLink capture mode. `outputVideoFormat`/`outputFrameRate` select the playout mode. The shader stack runs at input resolution and the final rendered frame is scaled once into the configured output mode. Common examples include `720p`/`50`, `720p`/`59.94`, `1080i`/`50`, `1080i`/`59.94`, `1080p`/`25`, `1080p`/`50`, `1080p`/`59.94`, and `2160p`/`59.94`, depending on card support.
+`inputVideoFormat`/`inputFrameRate` select the video capture mode. `outputVideoFormat`/`outputFrameRate` select the playout mode. With the current DeckLink backend, supported modes depend on the installed card and driver. The shader stack runs at input resolution and the final rendered frame is scaled once into the configured output mode. Common examples include `720p`/`50`, `720p`/`59.94`, `1080i`/`50`, `1080i`/`59.94`, `1080p`/`25`, `1080p`/`50`, `1080p`/`59.94`, and `2160p`/`59.94`.
 
 Legacy `videoFormat` and `frameRate` keys are still accepted and apply to both input and output unless the explicit input/output keys are present.
 
@@ -208,6 +220,7 @@ Runtime-generated files are intentionally ignored:
 - `runtime/shader_cache/active_shader.frag`
 - `runtime/runtime_state.json` autosaved latest stack and parameter state.
 - `runtime/stack_presets/*.json`
+- `runtime/screenshots/*.png` screenshots captured from the final output render target.
 
 Only `runtime/templates/` and `runtime/README.md` are tracked.
 
@@ -236,16 +249,15 @@ If `SLANG_ROOT` is not set, the workflow falls back to the repo-local default un
 
 - Audio.
 - Genlock.
-- Find a better UI library for react.
+- Find a better UI library for React.
 - Logs.
-- Continue source cleanup/refactoring. Pass 2 done
+- Add more video I/O backends now that the DeckLink path is behind `videoio/`.
 - Support a separate sound shader `.slang` file in shader packages. (https://www.shadertoy.com/view/XsBXWt)
 - Add WebView2
-- move to MSDF, typography rasterisation
+- MSDF typography rasterisation
-- better shader search UI, pass 1
+- More shader-library organisation and filtering as the built-in library grows.
-- More comprehensive greenscreen shader
 - linear compositing?
-- compute shaders or a small 1x1 or nx1 RGBA16f render target for abritary data store
+- compute shaders or a small 1x1 or nx1 RGBA16f render target for arbitrary data storage
 - allow shaders to read other shaders data store based on name? or output over OSC
-- Mipmappong for shader declared textures
+- Mipmapping for shader-declared textures
-- unwrap a fish eyelens and mirror it and map it to equirectangulr for environmnet map purposes
+- Multipass for shaders at request

SHADER_CONTRACT.md

@@ -163,7 +163,7 @@ Fields:
 - `uv`: normalized texture coordinates, usually `0..1`.
 - `sourceColor`: decoded RGBA source video at `uv`.
 - `inputResolution`: decoded input video resolution in pixels.
-- `outputResolution`: shader render resolution in pixels. The current pipeline renders the shader stack at input resolution, then scales the final frame to the configured DeckLink output mode.
+- `outputResolution`: shader render resolution in pixels. The current pipeline renders the shader stack at input resolution, then scales the final frame to the configured video I/O output mode.
 - `time`: elapsed runtime time in seconds.
 - `utcTimeSeconds`: current UTC time of day from the host PC clock, expressed as seconds since UTC midnight.
 - `utcOffsetSeconds`: host PC local UTC offset in seconds. Add this to `utcTimeSeconds` and wrap to `0..86400` to get local time of day.
@@ -177,8 +177,8 @@ Fields:
 Color/precision notes:
 
 - `context.sourceColor`, `sampleVideo()`, and temporal history samples are display-referred Rec.709-like RGB, not linear-light RGB.
-- The host prefers 10-bit DeckLink YUV capture and output when the card/mode supports it, with automatic 8-bit fallback.
+- The current DeckLink backend prefers 10-bit YUV capture and output when the card/mode supports it, with automatic 8-bit fallback.
-- Internal decoded, layer, composite, output, and temporal render targets are 16-bit floating point, so gradients and LUT work have more headroom than the packed DeckLink byte formats.
+- Internal decoded, layer, composite, output, and temporal render targets are 16-bit floating point, so gradients and LUT work have more headroom than packed byte video I/O formats.
 - Do not add extra Rec.709 or linear conversions unless the shader intentionally documents that behavior.
 
 ## Helper Functions

apps/LoopThroughWithOpenGLCompositing/LoopThroughWithOpenGLCompositing.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -89,7 +89,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>.;control;decklink;gl;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>.;control;gl;gl\pipeline;gl\renderer;gl\shader;videoio;videoio\decklink;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_DEBUG;_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -111,7 +111,7 @@
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>.;control;decklink;gl;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>.;control;gl;gl\pipeline;gl\renderer;gl\shader;videoio;videoio\decklink;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_DEBUG;_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -131,7 +131,7 @@
     <ClCompile>
       <Optimization>MaxSpeed</Optimization>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <AdditionalIncludeDirectories>.;control;decklink;gl;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>.;control;gl;gl\pipeline;gl\renderer;gl\shader;videoio;videoio\decklink;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <FunctionLevelLinking>true</FunctionLevelLinking>
@@ -156,7 +156,7 @@
     <ClCompile>
       <Optimization>MaxSpeed</Optimization>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <AdditionalIncludeDirectories>.;control;decklink;gl;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>.;control;gl;gl\pipeline;gl\renderer;gl\shader;videoio;videoio\decklink;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <FunctionLevelLinking>true</FunctionLevelLinking>
@@ -175,44 +175,53 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClCompile Include="gl\GLExtensions.cpp" />
+    <ClCompile Include="gl\renderer\GLExtensions.cpp" />
     <ClCompile Include="LoopThroughWithOpenGLCompositing.cpp" />
     <ClCompile Include="gl\OpenGLComposite.cpp" />
-    <ClCompile Include="gl\OpenGLRenderPass.cpp" />
+    <ClCompile Include="gl\pipeline\OpenGLRenderPass.cpp" />
-    <ClCompile Include="gl\OpenGLRenderer.cpp" />
+    <ClCompile Include="gl\pipeline\OpenGLRenderPipeline.cpp" />
-    <ClCompile Include="gl\OpenGLShaderPrograms.cpp" />
+    <ClCompile Include="gl\renderer\OpenGLRenderer.cpp" />
-    <ClCompile Include="gl\PngScreenshotWriter.cpp" />
+    <ClCompile Include="gl\shader\OpenGLShaderPrograms.cpp" />
-    <ClCompile Include="gl\ShaderBuildQueue.cpp" />
+    <ClCompile Include="gl\pipeline\PngScreenshotWriter.cpp" />
-    <ClCompile Include="gl\TemporalHistoryBuffers.cpp" />
+    <ClCompile Include="gl\shader\ShaderBuildQueue.cpp" />
+    <ClCompile Include="gl\pipeline\TemporalHistoryBuffers.cpp" />
+    <ClCompile Include="gl\pipeline\OpenGLVideoIOBridge.cpp" />
     <ClCompile Include="stdafx.cpp">
       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Create</PrecompiledHeader>
       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Create</PrecompiledHeader>
       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Create</PrecompiledHeader>
       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Create</PrecompiledHeader>
     </ClCompile>
-    <ClCompile Include="DeckLinkAPI_i.c" />
+    <ClCompile Include="videoio\decklink\DeckLinkAPI_i.c" />
     <ClCompile Include="control\RuntimeServices.cpp" />
-    <ClCompile Include="decklink\DeckLinkSession.cpp" />
+    <ClCompile Include="videoio\decklink\DeckLinkSession.cpp" />
-    <ClCompile Include="decklink\VideoIOFormat.cpp" />
+    <ClCompile Include="videoio\decklink\DeckLinkVideoIOFormat.cpp" />
     <ClCompile Include="runtime\RuntimeClock.cpp" />
+    <ClCompile Include="videoio\VideoIOFormat.cpp" />
+    <ClCompile Include="videoio\VideoPlayoutScheduler.cpp" />
   </ItemGroup>
   <ItemGroup>
-    <ClInclude Include="gl\GLExtensions.h" />
+    <ClInclude Include="gl\renderer\GLExtensions.h" />
     <ClInclude Include="LoopThroughWithOpenGLCompositing.h" />
     <ClInclude Include="gl\OpenGLComposite.h" />
-    <ClInclude Include="gl\OpenGLRenderPass.h" />
+    <ClInclude Include="gl\pipeline\OpenGLRenderPass.h" />
-    <ClInclude Include="gl\OpenGLRenderer.h" />
+    <ClInclude Include="gl\pipeline\OpenGLRenderPipeline.h" />
-    <ClInclude Include="gl\OpenGLShaderPrograms.h" />
+    <ClInclude Include="gl\renderer\OpenGLRenderer.h" />
-    <ClInclude Include="gl\PngScreenshotWriter.h" />
+    <ClInclude Include="gl\shader\OpenGLShaderPrograms.h" />
-    <ClInclude Include="gl\ShaderBuildQueue.h" />
+    <ClInclude Include="gl\pipeline\PngScreenshotWriter.h" />
-    <ClInclude Include="gl\TemporalHistoryBuffers.h" />
+    <ClInclude Include="gl\shader\ShaderBuildQueue.h" />
+    <ClInclude Include="gl\pipeline\TemporalHistoryBuffers.h" />
+    <ClInclude Include="gl\pipeline\OpenGLVideoIOBridge.h" />
     <ClInclude Include="resource.h" />
     <ClInclude Include="stdafx.h" />
     <ClInclude Include="targetver.h" />
     <ClInclude Include="control\RuntimeServices.h" />
-    <ClInclude Include="decklink\DeckLinkSession.h" />
+    <ClInclude Include="videoio\decklink\DeckLinkSession.h" />
-    <ClInclude Include="decklink\VideoIOFormat.h" />
+    <ClInclude Include="videoio\decklink\DeckLinkVideoIOFormat.h" />
     <ClInclude Include="runtime\RuntimeClock.h" />
+    <ClInclude Include="videoio\VideoIOFormat.h" />
+    <ClInclude Include="videoio\VideoIOTypes.h" />
+    <ClInclude Include="videoio\VideoPlayoutScheduler.h" />
   </ItemGroup>
   <ItemGroup>
     <Image Include="LoopThroughWithOpenGLCompositing.ico" />

apps/LoopThroughWithOpenGLCompositing/LoopThroughWithOpenGLCompositing.vcxproj.filters

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup>
     <Filter Include="Source Files">
@@ -18,7 +18,7 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="gl\GLExtensions.cpp">
+    <ClCompile Include="gl\renderer\GLExtensions.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
     <ClCompile Include="LoopThroughWithOpenGLCompositing.cpp">
@@ -27,45 +27,57 @@
     <ClCompile Include="gl\OpenGLComposite.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="gl\OpenGLRenderPass.cpp">
+    <ClCompile Include="gl\pipeline\OpenGLRenderPass.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="gl\OpenGLRenderer.cpp">
+    <ClCompile Include="gl\pipeline\OpenGLRenderPipeline.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="gl\OpenGLShaderPrograms.cpp">
+    <ClCompile Include="gl\renderer\OpenGLRenderer.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="gl\PngScreenshotWriter.cpp">
+    <ClCompile Include="gl\shader\OpenGLShaderPrograms.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="gl\ShaderBuildQueue.cpp">
+    <ClCompile Include="gl\pipeline\PngScreenshotWriter.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="gl\TemporalHistoryBuffers.cpp">
+    <ClCompile Include="gl\shader\ShaderBuildQueue.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="gl\pipeline\TemporalHistoryBuffers.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="gl\pipeline\OpenGLVideoIOBridge.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
     <ClCompile Include="stdafx.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="DeckLinkAPI_i.c">
+    <ClCompile Include="videoio\decklink\DeckLinkAPI_i.c">
       <Filter>DeckLink API</Filter>
     </ClCompile>
     <ClCompile Include="control\RuntimeServices.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="decklink\DeckLinkSession.cpp">
+    <ClCompile Include="videoio\decklink\DeckLinkSession.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="decklink\VideoIOFormat.cpp">
+    <ClCompile Include="videoio\decklink\DeckLinkVideoIOFormat.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
     <ClCompile Include="runtime\RuntimeClock.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="videoio\VideoIOFormat.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="videoio\VideoPlayoutScheduler.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
-    <ClInclude Include="gl\GLExtensions.h">
+    <ClInclude Include="gl\renderer\GLExtensions.h">
       <Filter>Header Files</Filter>
     </ClInclude>
     <ClInclude Include="LoopThroughWithOpenGLCompositing.h">
@@ -74,22 +86,28 @@
     <ClInclude Include="gl\OpenGLComposite.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="gl\OpenGLRenderPass.h">
+    <ClInclude Include="gl\pipeline\OpenGLRenderPass.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="gl\OpenGLRenderer.h">
+    <ClInclude Include="gl\pipeline\OpenGLRenderPipeline.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="gl\OpenGLShaderPrograms.h">
+    <ClInclude Include="gl\renderer\OpenGLRenderer.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="gl\PngScreenshotWriter.h">
+    <ClInclude Include="gl\shader\OpenGLShaderPrograms.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="gl\ShaderBuildQueue.h">
+    <ClInclude Include="gl\pipeline\PngScreenshotWriter.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="gl\TemporalHistoryBuffers.h">
+    <ClInclude Include="gl\shader\ShaderBuildQueue.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="gl\pipeline\TemporalHistoryBuffers.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="gl\pipeline\OpenGLVideoIOBridge.h">
       <Filter>Header Files</Filter>
     </ClInclude>
     <ClInclude Include="resource.h">
@@ -104,15 +122,24 @@
     <ClInclude Include="control\RuntimeServices.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="decklink\DeckLinkSession.h">
+    <ClInclude Include="videoio\decklink\DeckLinkSession.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="decklink\VideoIOFormat.h">
+    <ClInclude Include="videoio\decklink\DeckLinkVideoIOFormat.h">
       <Filter>Header Files</Filter>
     </ClInclude>
     <ClInclude Include="runtime\RuntimeClock.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="videoio\VideoIOFormat.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="videoio\VideoIOTypes.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="videoio\VideoPlayoutScheduler.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Image Include="LoopThroughWithOpenGLCompositing.ico">

apps/LoopThroughWithOpenGLCompositing/decklink/DeckLinkSession.h

@@ -1,88 +0,0 @@
-#pragma once
-
-#include "DeckLinkAPI_h.h"
-#include "DeckLinkDisplayMode.h"
-#include "DeckLinkFrameTransfer.h"
-#include "VideoIOFormat.h"
-
-#include <atlbase.h>
-#include <deque>
-#include <string>
-
-class OpenGLComposite;
-
-class DeckLinkSession
-{
-public:
-	DeckLinkSession() = default;
-	~DeckLinkSession();
-
-	void ReleaseResources();
-	bool DiscoverDevicesAndModes(const VideoFormatSelection& videoModes, std::string& error);
-	bool SelectPreferredFormats(const VideoFormatSelection& videoModes, std::string& error);
-	bool ConfigureInput(OpenGLComposite* owner, HDC hdc, HGLRC hglrc, const VideoFormat& inputVideoMode, std::string& error);
-	bool ConfigureOutput(OpenGLComposite* owner, HDC hdc, HGLRC hglrc, const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error);
-	bool Start();
-	bool Stop();
-
-	bool HasInputDevice() const { return input != nullptr; }
-	bool HasInputSource() const { return !hasNoInputSource; }
-	void SetInputSourceMissing(bool missing) { hasNoInputSource = missing; }
-	bool InputOutputDimensionsDiffer() const { return inputFrameSize != outputFrameSize; }
-	const FrameSize& InputFrameSize() const { return inputFrameSize; }
-	const FrameSize& OutputFrameSize() const { return outputFrameSize; }
-	unsigned InputFrameWidth() const { return inputFrameSize.width; }
-	unsigned InputFrameHeight() const { return inputFrameSize.height; }
-	unsigned OutputFrameWidth() const { return outputFrameSize.width; }
-	unsigned OutputFrameHeight() const { return outputFrameSize.height; }
-	VideoIOPixelFormat InputPixelFormat() const { return inputPixelFormat; }
-	VideoIOPixelFormat OutputPixelFormat() const { return outputPixelFormat; }
-	bool InputIsTenBit() const { return VideoIOPixelFormatIsTenBit(inputPixelFormat); }
-	bool OutputIsTenBit() const { return VideoIOPixelFormatIsTenBit(outputPixelFormat); }
-	unsigned InputFrameRowBytes() const { return inputFrameRowBytes; }
-	unsigned OutputFrameRowBytes() const { return outputFrameRowBytes; }
-	unsigned CaptureTextureWidth() const { return captureTextureWidth; }
-	unsigned OutputPackTextureWidth() const { return outputPackTextureWidth; }
-	const std::string& FormatStatusMessage() const { return formatStatusMessage; }
-	const std::string& InputDisplayModeName() const { return inputDisplayModeName; }
-	const std::string& OutputModelName() const { return outputModelName; }
-	bool SupportsInternalKeying() const { return supportsInternalKeying; }
-	bool SupportsExternalKeying() const { return supportsExternalKeying; }
-	bool KeyerInterfaceAvailable() const { return keyerInterfaceAvailable; }
-	bool ExternalKeyingActive() const { return externalKeyingActive; }
-	const std::string& StatusMessage() const { return statusMessage; }
-	void SetStatusMessage(const std::string& message) { statusMessage = message; }
-	double FrameBudgetMilliseconds() const;
-	IDeckLinkMutableVideoFrame* RotateOutputFrame();
-	void AccountForCompletionResult(BMDOutputFrameCompletionResult completionResult);
-	bool ScheduleOutputFrame(IDeckLinkMutableVideoFrame* outputVideoFrame);
-
-private:
-	CComPtr<CaptureDelegate>				captureDelegate;
-	CComPtr<PlayoutDelegate>				playoutDelegate;
-	CComPtr<IDeckLinkInput>					input;
-	CComPtr<IDeckLinkOutput>				output;
-	CComPtr<IDeckLinkKeyer>					keyer;
-	std::deque<CComPtr<IDeckLinkMutableVideoFrame>> outputVideoFrameQueue;
-	BMDTimeValue							frameDuration = 0;
-	BMDTimeScale							frameTimescale = 0;
-	unsigned								totalPlayoutFrames = 0;
-	FrameSize								inputFrameSize;
-	FrameSize								outputFrameSize;
-	VideoIOPixelFormat						inputPixelFormat = VideoIOPixelFormat::Uyvy8;
-	VideoIOPixelFormat						outputPixelFormat = VideoIOPixelFormat::Uyvy8;
-	unsigned								inputFrameRowBytes = 0;
-	unsigned								outputFrameRowBytes = 0;
-	unsigned								captureTextureWidth = 0;
-	unsigned								outputPackTextureWidth = 0;
-	std::string								inputDisplayModeName = "1080p59.94";
-	std::string								outputDisplayModeName = "1080p59.94";
-	bool									hasNoInputSource = true;
-	std::string								outputModelName;
-	bool									supportsInternalKeying = false;
-	bool									supportsExternalKeying = false;
-	bool									keyerInterfaceAvailable = false;
-	bool									externalKeyingActive = false;
-	std::string								statusMessage;
-	std::string								formatStatusMessage;
-};

apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp

@@ -3,9 +3,10 @@
 #include "OpenGLComposite.h"
 #include "GLExtensions.h"
 #include "GlRenderConstants.h"
-#include "OpenGLDeckLinkBridge.h"
 #include "OpenGLRenderPass.h"
+#include "OpenGLRenderPipeline.h"
 #include "OpenGLShaderPrograms.h"
+#include "OpenGLVideoIOBridge.h"
 #include "PngScreenshotWriter.h"
 #include "RuntimeServices.h"
 #include "ShaderBuildQueue.h"
@@ -22,23 +23,27 @@
 
 OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) :
 	hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC),
-	mDeckLink(std::make_unique<DeckLinkSession>()),
+	mVideoIO(std::make_unique<DeckLinkSession>()),
 	mRenderer(std::make_unique<OpenGLRenderer>()),
 	mUseCommittedLayerStates(false),
 	mScreenshotRequested(false)
 {
 	InitializeCriticalSection(&pMutex);
 	mRuntimeHost = std::make_unique<RuntimeHost>();
-	mDeckLinkBridge = std::make_unique<OpenGLDeckLinkBridge>(
+	mRenderPipeline = std::make_unique<OpenGLRenderPipeline>(
-		*mDeckLink,
 		*mRenderer,
 		*mRuntimeHost,
-		pMutex,
-		hGLDC,
-		hGLRC,
 		[this]() { renderEffect(); },
 		[this]() { ProcessScreenshotRequest(); },
 		[this]() { paintGL(); });
+	mVideoIOBridge = std::make_unique<OpenGLVideoIOBridge>(
+		*mVideoIO,
+		*mRenderer,
+		*mRenderPipeline,
+		*mRuntimeHost,
+		pMutex,
+		hGLDC,
+		hGLRC);
 	mRenderPass = std::make_unique<OpenGLRenderPass>(*mRenderer);
 	mShaderPrograms = std::make_unique<OpenGLShaderPrograms>(*mRenderer, *mRuntimeHost);
 	mShaderBuildQueue = std::make_unique<ShaderBuildQueue>(*mRuntimeHost);
@@ -51,13 +56,18 @@ OpenGLComposite::~OpenGLComposite()
 		mRuntimeServices->Stop();
 	if (mShaderBuildQueue)
 		mShaderBuildQueue->Stop();
-	mDeckLink->ReleaseResources();
+	mVideoIO->ReleaseResources();
 	mRenderer->DestroyResources();
 
 	DeleteCriticalSection(&pMutex);
 }
 
 bool OpenGLComposite::InitDeckLink()
+{
+	return InitVideoIO();
+}
+
+bool OpenGLComposite::InitVideoIO()
 {
 	VideoFormatSelection		videoModes;
 	std::string						initFailureReason;
@@ -87,7 +97,7 @@ bool OpenGLComposite::InitDeckLink()
 		}
 	}
 
-	if (!mDeckLink->DiscoverDevicesAndModes(videoModes, initFailureReason))
+	if (!mVideoIO->DiscoverDevicesAndModes(videoModes, initFailureReason))
 	{
 		const char* title = initFailureReason == "Please install the Blackmagic DeckLink drivers to use the features of this application."
 			? "This application requires the DeckLink drivers installed."
@@ -95,7 +105,7 @@ bool OpenGLComposite::InitDeckLink()
 		MessageBoxA(NULL, initFailureReason.c_str(), title, MB_OK | MB_ICONERROR);
 		return false;
 	}
-	if (!mDeckLink->SelectPreferredFormats(videoModes, initFailureReason))
+	if (!mVideoIO->SelectPreferredFormats(videoModes, initFailureReason))
 		goto error;
 
 	if (! CheckOpenGLExtensions())
@@ -110,38 +120,38 @@ bool OpenGLComposite::InitDeckLink()
 		goto error;
 	}
 
-	PublishDeckLinkOutputStatus(mDeckLink->OutputModelName().empty()
+	PublishVideoIOStatus(mVideoIO->OutputModelName().empty()
 		? "DeckLink output device selected."
-		: ("Selected output device: " + mDeckLink->OutputModelName()));
+		: ("Selected output device: " + mVideoIO->OutputModelName()));
 
 	// Resize window to match output video frame, but scale large formats down by half for viewing.
-	if (mDeckLink->OutputFrameWidth() < 1920)
+	if (mVideoIO->OutputFrameWidth() < 1920)
-		resizeWindow(mDeckLink->OutputFrameWidth(), mDeckLink->OutputFrameHeight());
+		resizeWindow(mVideoIO->OutputFrameWidth(), mVideoIO->OutputFrameHeight());
 	else
-		resizeWindow(mDeckLink->OutputFrameWidth() / 2, mDeckLink->OutputFrameHeight() / 2);
+		resizeWindow(mVideoIO->OutputFrameWidth() / 2, mVideoIO->OutputFrameHeight() / 2);
 
-	if (!mDeckLink->ConfigureInput(this, hGLDC, hGLRC, videoModes.input, initFailureReason))
+	if (!mVideoIO->ConfigureInput([this](const VideoIOFrame& frame) { mVideoIOBridge->VideoFrameArrived(frame); }, videoModes.input, initFailureReason))
 	{
 		goto error;
 	}
-	if (!mDeckLink->HasInputDevice() && mRuntimeHost)
+	if (!mVideoIO->HasInputDevice() && mRuntimeHost)
 	{
-		mRuntimeHost->SetSignalStatus(false, mDeckLink->InputFrameWidth(), mDeckLink->InputFrameHeight(), mDeckLink->InputDisplayModeName());
+		mRuntimeHost->SetSignalStatus(false, mVideoIO->InputFrameWidth(), mVideoIO->InputFrameHeight(), mVideoIO->InputDisplayModeName());
 	}
 
-	if (!mDeckLink->ConfigureOutput(this, hGLDC, hGLRC, videoModes.output, mRuntimeHost && mRuntimeHost->ExternalKeyingEnabled(), initFailureReason))
+	if (!mVideoIO->ConfigureOutput([this](const VideoIOCompletion& completion) { mVideoIOBridge->PlayoutFrameCompleted(completion); }, videoModes.output, mRuntimeHost && mRuntimeHost->ExternalKeyingEnabled(), initFailureReason))
 	{
 		goto error;
 	}
 
-	PublishDeckLinkOutputStatus(mDeckLink->StatusMessage());
+	PublishVideoIOStatus(mVideoIO->StatusMessage());
 
 	return true;
 
 error:
 	if (!initFailureReason.empty())
 		MessageBoxA(NULL, initFailureReason.c_str(), "DeckLink initialization failed", MB_OK | MB_ICONERROR);
-	mDeckLink->ReleaseResources();
+	mVideoIO->ReleaseResources();
 	return false;
 }
 
@@ -153,7 +163,7 @@ void OpenGLComposite::paintGL()
 		return;
 	}
 
-	mRenderer->PresentToWindow(hGLDC, mDeckLink->OutputFrameWidth(), mDeckLink->OutputFrameHeight());
+	mRenderer->PresentToWindow(hGLDC, mVideoIO->OutputFrameWidth(), mVideoIO->OutputFrameHeight());
 	ValidateRect(hGLWnd, NULL);
 	LeaveCriticalSection(&pMutex);
 }
@@ -174,22 +184,23 @@ void OpenGLComposite::resizeWindow(int width, int height)
 	}
 }
 
-void OpenGLComposite::PublishDeckLinkOutputStatus(const std::string& statusMessage)
+void OpenGLComposite::PublishVideoIOStatus(const std::string& statusMessage)
 {
 	if (!mRuntimeHost)
 		return;
 
 	if (!statusMessage.empty())
-		mDeckLink->SetStatusMessage(statusMessage);
+		mVideoIO->SetStatusMessage(statusMessage);
 
-	mRuntimeHost->SetDeckLinkOutputStatus(
+	mRuntimeHost->SetVideoIOStatus(
-		mDeckLink->OutputModelName(),
+		"decklink",
-		mDeckLink->SupportsInternalKeying(),
+		mVideoIO->OutputModelName(),
-		mDeckLink->SupportsExternalKeying(),
+		mVideoIO->SupportsInternalKeying(),
-		mDeckLink->KeyerInterfaceAvailable(),
+		mVideoIO->SupportsExternalKeying(),
+		mVideoIO->KeyerInterfaceAvailable(),
 		mRuntimeHost->ExternalKeyingEnabled(),
-		mDeckLink->ExternalKeyingActive(),
+		mVideoIO->ExternalKeyingActive(),
-		mDeckLink->StatusMessage());
+		mVideoIO->StatusMessage());
 }
 
 bool OpenGLComposite::InitOpenGLState()
@@ -223,7 +234,7 @@ bool OpenGLComposite::InitOpenGLState()
 		return false;
 	}
 
-	if (!mShaderPrograms->CompileLayerPrograms(mDeckLink->InputFrameWidth(), mDeckLink->InputFrameHeight(), sizeof(compilerErrorMessage), compilerErrorMessage))
+	if (!mShaderPrograms->CompileLayerPrograms(mVideoIO->InputFrameWidth(), mVideoIO->InputFrameHeight(), sizeof(compilerErrorMessage), compilerErrorMessage))
 	{
 		MessageBoxA(NULL, compilerErrorMessage, "OpenGL shader failed to load or compile", MB_OK);
 		return false;
@@ -234,12 +245,12 @@ bool OpenGLComposite::InitOpenGLState()
 
 	std::string rendererError;
 	if (!mRenderer->InitializeResources(
-		mDeckLink->InputFrameWidth(),
+		mVideoIO->InputFrameWidth(),
-		mDeckLink->InputFrameHeight(),
+		mVideoIO->InputFrameHeight(),
-		mDeckLink->CaptureTextureWidth(),
+		mVideoIO->CaptureTextureWidth(),
-		mDeckLink->OutputFrameWidth(),
+		mVideoIO->OutputFrameWidth(),
-		mDeckLink->OutputFrameHeight(),
+		mVideoIO->OutputFrameHeight(),
-		mDeckLink->OutputPackTextureWidth(),
+		mVideoIO->OutputPackTextureWidth(),
 		rendererError))
 	{
 		MessageBoxA(NULL, rendererError.c_str(), "OpenGL initialization error.", MB_OK);
@@ -251,20 +262,9 @@ bool OpenGLComposite::InitOpenGLState()
 	return true;
 }
 
-// DeckLink delegates still target OpenGLComposite; the bridge owns the per-frame work.
-void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource)
-{
-	mDeckLinkBridge->VideoFrameArrived(inputFrame, hasNoInputSource);
-}
-
-void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult)
-{
-	mDeckLinkBridge->PlayoutFrameCompleted(completedFrame, completionResult);
-}
-
 bool OpenGLComposite::Start()
 {
-	return mDeckLink->Start();
+	return mVideoIO->Start();
 }
 
 bool OpenGLComposite::Stop()
@@ -272,10 +272,10 @@ bool OpenGLComposite::Stop()
 	if (mRuntimeServices)
 		mRuntimeServices->Stop();
 
-	const bool wasExternalKeyingActive = mDeckLink->ExternalKeyingActive();
+	const bool wasExternalKeyingActive = mVideoIO->ExternalKeyingActive();
-	mDeckLink->Stop();
+	mVideoIO->Stop();
 	if (wasExternalKeyingActive)
-		PublishDeckLinkOutputStatus("External keying has been disabled.");
+		PublishVideoIOStatus("External keying has been disabled.");
 
 	return true;
 }
@@ -303,7 +303,7 @@ void OpenGLComposite::renderEffect()
 {
 	ProcessRuntimePollResults();
 
-	const bool hasInputSource = mDeckLink->HasInputSource();
+	const bool hasInputSource = mVideoIO->HasInputSource();
 	std::vector<RuntimeRenderState> layerStates;
 	if (mUseCommittedLayerStates)
 	{
@@ -313,7 +313,7 @@ void OpenGLComposite::renderEffect()
 	}
 	else if (mRuntimeHost)
 	{
-		if (mRuntimeHost->TryGetLayerRenderStates(mDeckLink->InputFrameWidth(), mDeckLink->InputFrameHeight(), layerStates))
+		if (mRuntimeHost->TryGetLayerRenderStates(mVideoIO->InputFrameWidth(), mVideoIO->InputFrameHeight(), layerStates))
 		{
 			mCachedLayerRenderStates = layerStates;
 		}
@@ -327,10 +327,10 @@ void OpenGLComposite::renderEffect()
 	mRenderPass->Render(
 		hasInputSource,
 		layerStates,
-		mDeckLink->InputFrameWidth(),
+		mVideoIO->InputFrameWidth(),
-		mDeckLink->InputFrameHeight(),
+		mVideoIO->InputFrameHeight(),
-		mDeckLink->CaptureTextureWidth(),
+		mVideoIO->CaptureTextureWidth(),
-		mDeckLink->InputPixelFormat(),
+		mVideoIO->InputPixelFormat(),
 		historyCap,
 		[this](const RuntimeRenderState& state, LayerProgram::TextBinding& textBinding, std::string& error) {
 			return mShaderPrograms->UpdateTextBindingTexture(state, textBinding, error);
@@ -345,8 +345,8 @@ void OpenGLComposite::ProcessScreenshotRequest()
 	if (!mScreenshotRequested.exchange(false))
 		return;
 
-	const unsigned width = mDeckLink ? mDeckLink->OutputFrameWidth() : 0;
+	const unsigned width = mVideoIO ? mVideoIO->OutputFrameWidth() : 0;
-	const unsigned height = mDeckLink ? mDeckLink->OutputFrameHeight() : 0;
+	const unsigned height = mVideoIO ? mVideoIO->OutputFrameHeight() : 0;
 	if (width == 0 || height == 0)
 		return;
 
@@ -426,7 +426,7 @@ bool OpenGLComposite::ProcessRuntimePollResults()
 			return true;
 
 		char compilerErrorMessage[1024] = {};
-		if (!mShaderPrograms->CommitPreparedLayerPrograms(readyBuild, mDeckLink->InputFrameWidth(), mDeckLink->InputFrameHeight(), sizeof(compilerErrorMessage), compilerErrorMessage))
+		if (!mShaderPrograms->CommitPreparedLayerPrograms(readyBuild, mVideoIO->InputFrameWidth(), mVideoIO->InputFrameHeight(), sizeof(compilerErrorMessage), compilerErrorMessage))
 		{
 			mRuntimeHost->SetCompileStatus(false, compilerErrorMessage);
 			mUseCommittedLayerStates = true;
@@ -449,13 +449,13 @@ bool OpenGLComposite::ProcessRuntimePollResults()
 
 void OpenGLComposite::RequestShaderBuild()
 {
-	if (!mShaderBuildQueue || !mDeckLink)
+	if (!mShaderBuildQueue || !mVideoIO)
 		return;
 
 	mUseCommittedLayerStates = true;
 	if (mRuntimeHost)
 		mRuntimeHost->ClearReloadRequest();
-	mShaderBuildQueue->RequestBuild(mDeckLink->InputFrameWidth(), mDeckLink->InputFrameHeight());
+	mShaderBuildQueue->RequestBuild(mVideoIO->InputFrameWidth(), mVideoIO->InputFrameHeight());
 }
 
 void OpenGLComposite::broadcastRuntimeState()

apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h

@@ -10,7 +10,6 @@
 #include <objbase.h>
 #include <atlbase.h>
 #include <comutil.h>
-#include "DeckLinkAPI_h.h"
 
 #include "GLExtensions.h"
 #include "OpenGLRenderer.h"
@@ -25,9 +24,10 @@
 #include <vector>
 #include <deque>
 
-class DeckLinkSession;
+class VideoIODevice;
-class OpenGLDeckLinkBridge;
+class OpenGLVideoIOBridge;
 class OpenGLRenderPass;
+class OpenGLRenderPipeline;
 class OpenGLShaderPrograms;
 class RuntimeServices;
 class ShaderBuildQueue;
@@ -40,6 +40,7 @@ public:
 	~OpenGLComposite();
 
 	bool InitDeckLink();
+	bool InitVideoIO();
 	bool Start();
 	bool Stop();
 	bool ReloadShader();
@@ -65,13 +66,10 @@ public:
 	void resizeGL(WORD width, WORD height);
 	void paintGL();
 
-	void VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource);
-	void PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult result);
-
 private:
 	void resizeWindow(int width, int height);
 	bool CheckOpenGLExtensions();
-	void PublishDeckLinkOutputStatus(const std::string& statusMessage);
+	void PublishVideoIOStatus(const std::string& statusMessage);
 	using LayerProgram = OpenGLRenderer::LayerProgram;
 
 	HWND									hGLWnd;
@@ -79,11 +77,12 @@ private:
 	HGLRC									hGLRC;
 	CRITICAL_SECTION						pMutex;
 
-	std::unique_ptr<DeckLinkSession>			mDeckLink;
+	std::unique_ptr<VideoIODevice>			mVideoIO;
 	std::unique_ptr<OpenGLRenderer>			mRenderer;
 	std::unique_ptr<RuntimeHost>				mRuntimeHost;
-	std::unique_ptr<OpenGLDeckLinkBridge>	mDeckLinkBridge;
+	std::unique_ptr<OpenGLVideoIOBridge>		mVideoIOBridge;
 	std::unique_ptr<OpenGLRenderPass>		mRenderPass;
+	std::unique_ptr<OpenGLRenderPipeline>	mRenderPipeline;
 	std::unique_ptr<OpenGLShaderPrograms>	mShaderPrograms;
 	std::unique_ptr<ShaderBuildQueue>		mShaderBuildQueue;
 	std::unique_ptr<RuntimeServices>		mRuntimeServices;

apps/LoopThroughWithOpenGLCompositing/gl/OpenGLDeckLinkBridge.cpp

@@ -1,209 +0,0 @@
-#include "OpenGLDeckLinkBridge.h"
-
-#include "DeckLinkSession.h"
-#include "OpenGLRenderer.h"
-#include "RuntimeHost.h"
-
-#include <chrono>
-#include <gl/gl.h>
-
-OpenGLDeckLinkBridge::OpenGLDeckLinkBridge(
-	DeckLinkSession& deckLink,
-	OpenGLRenderer& renderer,
-	RuntimeHost& runtimeHost,
-	CRITICAL_SECTION& mutex,
-	HDC hdc,
-	HGLRC hglrc,
-	RenderEffectCallback renderEffect,
-	OutputReadyCallback outputReady,
-	PaintCallback paint) :
-	mDeckLink(deckLink),
-	mRenderer(renderer),
-	mRuntimeHost(runtimeHost),
-	mMutex(mutex),
-	mHdc(hdc),
-	mHglrc(hglrc),
-	mRenderEffect(renderEffect),
-	mOutputReady(outputReady),
-	mPaint(paint)
-{
-}
-
-void OpenGLDeckLinkBridge::RecordFramePacing(BMDOutputFrameCompletionResult completionResult)
-{
-	const auto now = std::chrono::steady_clock::now();
-	if (mLastPlayoutCompletionTime != std::chrono::steady_clock::time_point())
-	{
-		mCompletionIntervalMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(now - mLastPlayoutCompletionTime).count();
-		if (mSmoothedCompletionIntervalMilliseconds <= 0.0)
-			mSmoothedCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds;
-		else
-			mSmoothedCompletionIntervalMilliseconds = mSmoothedCompletionIntervalMilliseconds * 0.9 + mCompletionIntervalMilliseconds * 0.1;
-		if (mCompletionIntervalMilliseconds > mMaxCompletionIntervalMilliseconds)
-			mMaxCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds;
-	}
-	mLastPlayoutCompletionTime = now;
-
-	if (completionResult == bmdOutputFrameDisplayedLate)
-		++mLateFrameCount;
-	else if (completionResult == bmdOutputFrameDropped)
-		++mDroppedFrameCount;
-	else if (completionResult == bmdOutputFrameFlushed)
-		++mFlushedFrameCount;
-
-	mRuntimeHost.TrySetFramePacingStats(
-		mCompletionIntervalMilliseconds,
-		mSmoothedCompletionIntervalMilliseconds,
-		mMaxCompletionIntervalMilliseconds,
-		mLateFrameCount,
-		mDroppedFrameCount,
-		mFlushedFrameCount);
-}
-
-void OpenGLDeckLinkBridge::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource)
-{
-	mDeckLink.SetInputSourceMissing(hasNoInputSource);
-	mRuntimeHost.TrySetSignalStatus(!hasNoInputSource, mDeckLink.InputFrameWidth(), mDeckLink.InputFrameHeight(), mDeckLink.InputDisplayModeName());
-
-	if (!mDeckLink.HasInputSource())
-		return;							// don't transfer texture when there's no input
-
-	long textureSize = inputFrame->GetRowBytes() * inputFrame->GetHeight();
-	IDeckLinkVideoBuffer* inputFrameBuffer = NULL;
-	void* videoPixels;
-
-	if (inputFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&inputFrameBuffer) != S_OK)
-		return;
-
-	if (inputFrameBuffer->StartAccess(bmdBufferAccessRead) != S_OK)
-	{
-		inputFrameBuffer->Release();
-		return;
-	}
-	inputFrameBuffer->GetBytes(&videoPixels);
-
-	EnterCriticalSection(&mMutex);
-
-	wglMakeCurrent(mHdc, mHglrc);		// make OpenGL context current in this thread
-
-	glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
-	glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
-	glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mRenderer.TextureUploadBuffer());
-	glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, videoPixels, GL_DYNAMIC_DRAW);
-	glBindTexture(GL_TEXTURE_2D, mRenderer.CaptureTexture());
-
-	// NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data.
-	if (mDeckLink.InputPixelFormat() == VideoIOPixelFormat::V210)
-		glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mDeckLink.CaptureTextureWidth(), mDeckLink.InputFrameHeight(), GL_RGBA, GL_UNSIGNED_BYTE, NULL);
-	else
-		glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mDeckLink.CaptureTextureWidth(), mDeckLink.InputFrameHeight(), GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
-
-	glBindTexture(GL_TEXTURE_2D, 0);
-	glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
-
-	wglMakeCurrent(NULL, NULL);
-
-	LeaveCriticalSection(&mMutex);
-
-	inputFrameBuffer->EndAccess(bmdBufferAccessRead);
-	inputFrameBuffer->Release();
-}
-
-void OpenGLDeckLinkBridge::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult)
-{
-	(void)completedFrame;
-
-	RecordFramePacing(completionResult);
-
-	EnterCriticalSection(&mMutex);
-
-	// Get the first frame from the queue
-	IDeckLinkMutableVideoFrame* outputVideoFrame = mDeckLink.RotateOutputFrame();
-
-	// make GL context current in this thread
-	wglMakeCurrent(mHdc, mHglrc);
-
-	// Draw the effect output to the off-screen framebuffer.
-	const auto renderStartTime = std::chrono::steady_clock::now();
-	glBindFramebuffer(GL_FRAMEBUFFER, mRenderer.CompositeFramebuffer());
-	mRenderEffect();
-	glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.CompositeFramebuffer());
-	glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer.OutputFramebuffer());
-	glBlitFramebuffer(0, 0, mDeckLink.InputFrameWidth(), mDeckLink.InputFrameHeight(), 0, 0, mDeckLink.OutputFrameWidth(), mDeckLink.OutputFrameHeight(), GL_COLOR_BUFFER_BIT, GL_LINEAR);
-	glBindFramebuffer(GL_FRAMEBUFFER, mRenderer.OutputFramebuffer());
-	if (mOutputReady)
-		mOutputReady();
-	if (mDeckLink.OutputIsTenBit())
-	{
-		glBindFramebuffer(GL_FRAMEBUFFER, mRenderer.OutputPackFramebuffer());
-		glViewport(0, 0, mDeckLink.OutputPackTextureWidth(), mDeckLink.OutputFrameHeight());
-		glDisable(GL_SCISSOR_TEST);
-		glDisable(GL_BLEND);
-		glDisable(GL_DEPTH_TEST);
-		glActiveTexture(GL_TEXTURE0);
-		glBindTexture(GL_TEXTURE_2D, mRenderer.OutputTexture());
-		glBindVertexArray(mRenderer.FullscreenVertexArray());
-		glUseProgram(mRenderer.OutputPackProgram());
-		const GLint outputResolutionLocation = glGetUniformLocation(mRenderer.OutputPackProgram(), "uOutputVideoResolution");
-		const GLint activeWordsLocation = glGetUniformLocation(mRenderer.OutputPackProgram(), "uActiveV210Words");
-		if (outputResolutionLocation >= 0)
-			glUniform2f(outputResolutionLocation, static_cast<float>(mDeckLink.OutputFrameWidth()), static_cast<float>(mDeckLink.OutputFrameHeight()));
-		if (activeWordsLocation >= 0)
-			glUniform1f(activeWordsLocation, static_cast<float>(ActiveV210WordsForWidth(mDeckLink.OutputFrameWidth())));
-		glDrawArrays(GL_TRIANGLES, 0, 3);
-		glUseProgram(0);
-		glBindVertexArray(0);
-		glBindTexture(GL_TEXTURE_2D, 0);
-	}
-	glFlush();
-	const auto renderEndTime = std::chrono::steady_clock::now();
-	const double frameBudgetMilliseconds = mDeckLink.FrameBudgetMilliseconds();
-	const double renderMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(renderEndTime - renderStartTime).count();
-	mRuntimeHost.TrySetPerformanceStats(frameBudgetMilliseconds, renderMilliseconds);
-	mRuntimeHost.TryAdvanceFrame();
-
-	IDeckLinkVideoBuffer* outputVideoFrameBuffer;
-	if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK)
-	{
-		wglMakeCurrent(NULL, NULL);
-		LeaveCriticalSection(&mMutex);
-		return;
-	}
-
-	if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK)
-	{
-		outputVideoFrameBuffer->Release();
-		wglMakeCurrent(NULL, NULL);
-		LeaveCriticalSection(&mMutex);
-		return;
-	}
-
-	void*	pFrame;
-	outputVideoFrameBuffer->GetBytes(&pFrame);
-
-	glPixelStorei(GL_PACK_ALIGNMENT, 4);
-	glPixelStorei(GL_PACK_ROW_LENGTH, 0);
-	if (mDeckLink.OutputIsTenBit())
-	{
-		glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.OutputPackFramebuffer());
-		glReadPixels(0, 0, mDeckLink.OutputPackTextureWidth(), mDeckLink.OutputFrameHeight(), GL_RGBA, GL_UNSIGNED_BYTE, pFrame);
-	}
-	else
-	{
-		glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.OutputFramebuffer());
-		glReadPixels(0, 0, mDeckLink.OutputFrameWidth(), mDeckLink.OutputFrameHeight(), GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame);
-	}
-	mPaint();
-
-	outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite);
-	outputVideoFrameBuffer->Release();
-
-	mDeckLink.AccountForCompletionResult(completionResult);
-
-	// Schedule the next frame for playout
-	mDeckLink.ScheduleOutputFrame(outputVideoFrame);
-
-	wglMakeCurrent(NULL, NULL);
-
-	LeaveCriticalSection(&mMutex);
-}

apps/LoopThroughWithOpenGLCompositing/gl/OpenGLDeckLinkBridge.h

@@ -1,55 +0,0 @@
-#pragma once
-
-#include "DeckLinkAPI_h.h"
-
-#include <windows.h>
-
-#include <chrono>
-#include <functional>
-#include <cstdint>
-
-class DeckLinkSession;
-class OpenGLRenderer;
-class RuntimeHost;
-
-class OpenGLDeckLinkBridge
-{
-public:
-	using RenderEffectCallback = std::function<void()>;
-	using OutputReadyCallback = std::function<void()>;
-	using PaintCallback = std::function<void()>;
-
-	OpenGLDeckLinkBridge(
-		DeckLinkSession& deckLink,
-		OpenGLRenderer& renderer,
-		RuntimeHost& runtimeHost,
-		CRITICAL_SECTION& mutex,
-		HDC hdc,
-		HGLRC hglrc,
-		RenderEffectCallback renderEffect,
-		OutputReadyCallback outputReady,
-		PaintCallback paint);
-
-	void VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource);
-	void PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult);
-
-private:
-	void RecordFramePacing(BMDOutputFrameCompletionResult completionResult);
-
-	DeckLinkSession& mDeckLink;
-	OpenGLRenderer& mRenderer;
-	RuntimeHost& mRuntimeHost;
-	CRITICAL_SECTION& mMutex;
-	HDC mHdc;
-	HGLRC mHglrc;
-	RenderEffectCallback mRenderEffect;
-	OutputReadyCallback mOutputReady;
-	PaintCallback mPaint;
-	std::chrono::steady_clock::time_point mLastPlayoutCompletionTime;
-	double mCompletionIntervalMilliseconds = 0.0;
-	double mSmoothedCompletionIntervalMilliseconds = 0.0;
-	double mMaxCompletionIntervalMilliseconds = 0.0;
-	uint64_t mLateFrameCount = 0;
-	uint64_t mDroppedFrameCount = 0;
-	uint64_t mFlushedFrameCount = 0;
-};

apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLRenderPipeline.cpp

@@ -0,0 +1,92 @@
+#include "OpenGLRenderPipeline.h"
+
+#include "OpenGLRenderer.h"
+#include "RuntimeHost.h"
+#include "VideoIOFormat.h"
+
+#include <chrono>
+#include <gl/gl.h>
+
+OpenGLRenderPipeline::OpenGLRenderPipeline(
+	OpenGLRenderer& renderer,
+	RuntimeHost& runtimeHost,
+	RenderEffectCallback renderEffect,
+	OutputReadyCallback outputReady,
+	PaintCallback paint) :
+	mRenderer(renderer),
+	mRuntimeHost(runtimeHost),
+	mRenderEffect(renderEffect),
+	mOutputReady(outputReady),
+	mPaint(paint)
+{
+}
+
+bool OpenGLRenderPipeline::RenderFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame)
+{
+	const VideoIOState& state = context.videoState;
+
+	const auto renderStartTime = std::chrono::steady_clock::now();
+	glBindFramebuffer(GL_FRAMEBUFFER, mRenderer.CompositeFramebuffer());
+	mRenderEffect();
+	glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.CompositeFramebuffer());
+	glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer.OutputFramebuffer());
+	glBlitFramebuffer(0, 0, state.inputFrameSize.width, state.inputFrameSize.height, 0, 0, state.outputFrameSize.width, state.outputFrameSize.height, GL_COLOR_BUFFER_BIT, GL_LINEAR);
+	glBindFramebuffer(GL_FRAMEBUFFER, mRenderer.OutputFramebuffer());
+	if (mOutputReady)
+		mOutputReady();
+	if (state.outputPixelFormat == VideoIOPixelFormat::V210)
+		PackOutputForV210(state);
+	glFlush();
+
+	const auto renderEndTime = std::chrono::steady_clock::now();
+	const double renderMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(renderEndTime - renderStartTime).count();
+	mRuntimeHost.TrySetPerformanceStats(state.frameBudgetMilliseconds, renderMilliseconds);
+	mRuntimeHost.TryAdvanceFrame();
+
+	ReadOutputFrame(state, outputFrame);
+	if (mPaint)
+		mPaint();
+
+	return true;
+}
+
+void OpenGLRenderPipeline::PackOutputForV210(const VideoIOState& state)
+{
+	glBindFramebuffer(GL_FRAMEBUFFER, mRenderer.OutputPackFramebuffer());
+	glViewport(0, 0, state.outputPackTextureWidth, state.outputFrameSize.height);
+	glDisable(GL_SCISSOR_TEST);
+	glDisable(GL_BLEND);
+	glDisable(GL_DEPTH_TEST);
+	glActiveTexture(GL_TEXTURE0);
+	glBindTexture(GL_TEXTURE_2D, mRenderer.OutputTexture());
+	glBindVertexArray(mRenderer.FullscreenVertexArray());
+	glUseProgram(mRenderer.OutputPackProgram());
+
+	const GLint outputResolutionLocation = glGetUniformLocation(mRenderer.OutputPackProgram(), "uOutputVideoResolution");
+	const GLint activeWordsLocation = glGetUniformLocation(mRenderer.OutputPackProgram(), "uActiveV210Words");
+	if (outputResolutionLocation >= 0)
+		glUniform2f(outputResolutionLocation, static_cast<float>(state.outputFrameSize.width), static_cast<float>(state.outputFrameSize.height));
+	if (activeWordsLocation >= 0)
+		glUniform1f(activeWordsLocation, static_cast<float>(ActiveV210WordsForWidth(state.outputFrameSize.width)));
+
+	glDrawArrays(GL_TRIANGLES, 0, 3);
+	glUseProgram(0);
+	glBindVertexArray(0);
+	glBindTexture(GL_TEXTURE_2D, 0);
+}
+
+void OpenGLRenderPipeline::ReadOutputFrame(const VideoIOState& state, VideoIOOutputFrame& outputFrame)
+{
+	glPixelStorei(GL_PACK_ALIGNMENT, 4);
+	glPixelStorei(GL_PACK_ROW_LENGTH, 0);
+	if (state.outputPixelFormat == VideoIOPixelFormat::V210)
+	{
+		glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.OutputPackFramebuffer());
+		glReadPixels(0, 0, state.outputPackTextureWidth, state.outputFrameSize.height, GL_RGBA, GL_UNSIGNED_BYTE, outputFrame.bytes);
+	}
+	else
+	{
+		glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.OutputFramebuffer());
+		glReadPixels(0, 0, state.outputFrameSize.width, state.outputFrameSize.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, outputFrame.bytes);
+	}
+}

apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLRenderPipeline.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "VideoIOTypes.h"
+
+#include <functional>
+
+class OpenGLRenderer;
+class RuntimeHost;
+
+struct RenderPipelineFrameContext
+{
+	VideoIOState videoState;
+	VideoIOCompletion completion;
+};
+
+class OpenGLRenderPipeline
+{
+public:
+	using RenderEffectCallback = std::function<void()>;
+	using OutputReadyCallback = std::function<void()>;
+	using PaintCallback = std::function<void()>;
+
+	OpenGLRenderPipeline(
+		OpenGLRenderer& renderer,
+		RuntimeHost& runtimeHost,
+		RenderEffectCallback renderEffect,
+		OutputReadyCallback outputReady,
+		PaintCallback paint);
+
+	bool RenderFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame);
+
+private:
+	void PackOutputForV210(const VideoIOState& state);
+	void ReadOutputFrame(const VideoIOState& state, VideoIOOutputFrame& outputFrame);
+
+	OpenGLRenderer& mRenderer;
+	RuntimeHost& mRuntimeHost;
+	RenderEffectCallback mRenderEffect;
+	OutputReadyCallback mOutputReady;
+	PaintCallback mPaint;
+};

apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.cpp

@@ -0,0 +1,124 @@
+#include "OpenGLVideoIOBridge.h"
+
+#include "OpenGLRenderer.h"
+#include "RuntimeHost.h"
+
+#include <chrono>
+#include <gl/gl.h>
+
+OpenGLVideoIOBridge::OpenGLVideoIOBridge(
+	VideoIODevice& videoIO,
+	OpenGLRenderer& renderer,
+	OpenGLRenderPipeline& renderPipeline,
+	RuntimeHost& runtimeHost,
+	CRITICAL_SECTION& mutex,
+	HDC hdc,
+	HGLRC hglrc) :
+	mVideoIO(videoIO),
+	mRenderer(renderer),
+	mRenderPipeline(renderPipeline),
+	mRuntimeHost(runtimeHost),
+	mMutex(mutex),
+	mHdc(hdc),
+	mHglrc(hglrc)
+{
+}
+
+void OpenGLVideoIOBridge::RecordFramePacing(VideoIOCompletionResult completionResult)
+{
+	const auto now = std::chrono::steady_clock::now();
+	if (mLastPlayoutCompletionTime != std::chrono::steady_clock::time_point())
+	{
+		mCompletionIntervalMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(now - mLastPlayoutCompletionTime).count();
+		if (mSmoothedCompletionIntervalMilliseconds <= 0.0)
+			mSmoothedCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds;
+		else
+			mSmoothedCompletionIntervalMilliseconds = mSmoothedCompletionIntervalMilliseconds * 0.9 + mCompletionIntervalMilliseconds * 0.1;
+		if (mCompletionIntervalMilliseconds > mMaxCompletionIntervalMilliseconds)
+			mMaxCompletionIntervalMilliseconds = mCompletionIntervalMilliseconds;
+	}
+	mLastPlayoutCompletionTime = now;
+
+	if (completionResult == VideoIOCompletionResult::DisplayedLate)
+		++mLateFrameCount;
+	else if (completionResult == VideoIOCompletionResult::Dropped)
+		++mDroppedFrameCount;
+	else if (completionResult == VideoIOCompletionResult::Flushed)
+		++mFlushedFrameCount;
+
+	mRuntimeHost.TrySetFramePacingStats(
+		mCompletionIntervalMilliseconds,
+		mSmoothedCompletionIntervalMilliseconds,
+		mMaxCompletionIntervalMilliseconds,
+		mLateFrameCount,
+		mDroppedFrameCount,
+		mFlushedFrameCount);
+}
+
+void OpenGLVideoIOBridge::VideoFrameArrived(const VideoIOFrame& inputFrame)
+{
+	const VideoIOState& state = mVideoIO.State();
+	mRuntimeHost.TrySetSignalStatus(!inputFrame.hasNoInputSource, state.inputFrameSize.width, state.inputFrameSize.height, state.inputDisplayModeName);
+
+	if (inputFrame.hasNoInputSource || inputFrame.bytes == nullptr)
+		return;							// don't transfer texture when there's no input
+
+	const long textureSize = inputFrame.rowBytes * static_cast<long>(inputFrame.height);
+
+	EnterCriticalSection(&mMutex);
+
+	wglMakeCurrent(mHdc, mHglrc);		// make OpenGL context current in this thread
+
+	glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
+	glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+	glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mRenderer.TextureUploadBuffer());
+	glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, inputFrame.bytes, GL_DYNAMIC_DRAW);
+	glBindTexture(GL_TEXTURE_2D, mRenderer.CaptureTexture());
+
+	// NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data.
+	if (inputFrame.pixelFormat == VideoIOPixelFormat::V210)
+		glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, state.captureTextureWidth, state.inputFrameSize.height, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
+	else
+		glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, state.captureTextureWidth, state.inputFrameSize.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
+
+	glBindTexture(GL_TEXTURE_2D, 0);
+	glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+
+	wglMakeCurrent(NULL, NULL);
+
+	LeaveCriticalSection(&mMutex);
+}
+
+void OpenGLVideoIOBridge::PlayoutFrameCompleted(const VideoIOCompletion& completion)
+{
+	RecordFramePacing(completion.result);
+
+	EnterCriticalSection(&mMutex);
+
+	VideoIOOutputFrame outputFrame;
+	if (!mVideoIO.BeginOutputFrame(outputFrame))
+	{
+		LeaveCriticalSection(&mMutex);
+		return;
+	}
+	const VideoIOState& state = mVideoIO.State();
+	RenderPipelineFrameContext frameContext;
+	frameContext.videoState = state;
+	frameContext.completion = completion;
+
+	// make GL context current in this thread
+	wglMakeCurrent(mHdc, mHglrc);
+
+	mRenderPipeline.RenderFrame(frameContext, outputFrame);
+
+	mVideoIO.EndOutputFrame(outputFrame);
+
+	mVideoIO.AccountForCompletionResult(completion.result);
+
+	// Schedule the next frame for playout
+	mVideoIO.ScheduleOutputFrame(outputFrame);
+
+	wglMakeCurrent(NULL, NULL);
+
+	LeaveCriticalSection(&mMutex);
+}

apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include "OpenGLRenderPipeline.h"
+
+#include <windows.h>
+
+#include <chrono>
+#include <cstdint>
+
+class RuntimeHost;
+
+class OpenGLVideoIOBridge
+{
+public:
+	OpenGLVideoIOBridge(
+		VideoIODevice& videoIO,
+		OpenGLRenderer& renderer,
+		OpenGLRenderPipeline& renderPipeline,
+		RuntimeHost& runtimeHost,
+		CRITICAL_SECTION& mutex,
+		HDC hdc,
+		HGLRC hglrc);
+
+	void VideoFrameArrived(const VideoIOFrame& inputFrame);
+	void PlayoutFrameCompleted(const VideoIOCompletion& completion);
+
+private:
+	void RecordFramePacing(VideoIOCompletionResult completionResult);
+
+	VideoIODevice& mVideoIO;
+	OpenGLRenderer& mRenderer;
+	OpenGLRenderPipeline& mRenderPipeline;
+	RuntimeHost& mRuntimeHost;
+	CRITICAL_SECTION& mMutex;
+	HDC mHdc;
+	HGLRC mHglrc;
+	std::chrono::steady_clock::time_point mLastPlayoutCompletionTime;
+	double mCompletionIntervalMilliseconds = 0.0;
+	double mSmoothedCompletionIntervalMilliseconds = 0.0;
+	double mMaxCompletionIntervalMilliseconds = 0.0;
+	uint64_t mLateFrameCount = 0;
+	uint64_t mDroppedFrameCount = 0;
+	uint64_t mFlushedFrameCount = 0;
+};

apps/LoopThroughWithOpenGLCompositing/runtime/RuntimeHost.cpp

@@ -1213,8 +1213,16 @@ void RuntimeHost::SetSignalStatusLocked(bool hasSignal, unsigned width, unsigned
 
 void RuntimeHost::SetDeckLinkOutputStatus(const std::string& modelName, bool supportsInternalKeying, bool supportsExternalKeying,
 	bool keyerInterfaceAvailable, bool externalKeyingRequested, bool externalKeyingActive, const std::string& statusMessage)
+{
+	SetVideoIOStatus("decklink", modelName, supportsInternalKeying, supportsExternalKeying, keyerInterfaceAvailable,
+		externalKeyingRequested, externalKeyingActive, statusMessage);
+}
+
+void RuntimeHost::SetVideoIOStatus(const std::string& backendName, const std::string& modelName, bool supportsInternalKeying, bool supportsExternalKeying,
+	bool keyerInterfaceAvailable, bool externalKeyingRequested, bool externalKeyingActive, const std::string& statusMessage)
 {
 	std::lock_guard<std::mutex> lock(mMutex);
+	mDeckLinkOutputStatus.backendName = backendName;
 	mDeckLinkOutputStatus.modelName = modelName;
 	mDeckLinkOutputStatus.supportsInternalKeying = supportsInternalKeying;
 	mDeckLinkOutputStatus.supportsExternalKeying = supportsExternalKeying;
@@ -1889,6 +1897,17 @@ JsonValue RuntimeHost::BuildStateValue() const
 	deckLink.set("statusMessage", JsonValue(mDeckLinkOutputStatus.statusMessage));
 	root.set("decklink", deckLink);
 
+	JsonValue videoIO = JsonValue::MakeObject();
+	videoIO.set("backend", JsonValue(mDeckLinkOutputStatus.backendName));
+	videoIO.set("modelName", JsonValue(mDeckLinkOutputStatus.modelName));
+	videoIO.set("supportsInternalKeying", JsonValue(mDeckLinkOutputStatus.supportsInternalKeying));
+	videoIO.set("supportsExternalKeying", JsonValue(mDeckLinkOutputStatus.supportsExternalKeying));
+	videoIO.set("keyerInterfaceAvailable", JsonValue(mDeckLinkOutputStatus.keyerInterfaceAvailable));
+	videoIO.set("externalKeyingRequested", JsonValue(mDeckLinkOutputStatus.externalKeyingRequested));
+	videoIO.set("externalKeyingActive", JsonValue(mDeckLinkOutputStatus.externalKeyingActive));
+	videoIO.set("statusMessage", JsonValue(mDeckLinkOutputStatus.statusMessage));
+	root.set("videoIO", videoIO);
+
 	JsonValue performance = JsonValue::MakeObject();
 	performance.set("frameBudgetMs", JsonValue(mFrameBudgetMilliseconds));
 	performance.set("renderMs", JsonValue(mRenderMilliseconds));

apps/LoopThroughWithOpenGLCompositing/runtime/RuntimeHost.h

@@ -39,6 +39,8 @@ public:
 	bool TrySetSignalStatus(bool hasSignal, unsigned width, unsigned height, const std::string& modeName);
 	void SetDeckLinkOutputStatus(const std::string& modelName, bool supportsInternalKeying, bool supportsExternalKeying,
 		bool keyerInterfaceAvailable, bool externalKeyingRequested, bool externalKeyingActive, const std::string& statusMessage);
+	void SetVideoIOStatus(const std::string& backendName, const std::string& modelName, bool supportsInternalKeying, bool supportsExternalKeying,
+		bool keyerInterfaceAvailable, bool externalKeyingRequested, bool externalKeyingActive, const std::string& statusMessage);
 	void SetPerformanceStats(double frameBudgetMilliseconds, double renderMilliseconds);
 	bool TrySetPerformanceStats(double frameBudgetMilliseconds, double renderMilliseconds);
 	void SetFramePacingStats(double completionIntervalMilliseconds, double smoothedCompletionIntervalMilliseconds,
@@ -86,6 +88,7 @@ private:
 
 	struct DeckLinkOutputStatus
 	{
+		std::string backendName = "decklink";
 		std::string modelName;
 		bool supportsInternalKeying = false;
 		bool supportsExternalKeying = false;

apps/LoopThroughWithOpenGLCompositing/shader/ShaderCompiler.cpp

@@ -190,6 +190,18 @@ bool ShaderCompiler::BuildWrapperSlangSource(const ShaderPackage& shaderPackage,
 
 bool ShaderCompiler::FindSlangCompiler(std::filesystem::path& compilerPath, std::string& error) const
 {
+	char slangRootBuffer[MAX_PATH] = {};
+	const DWORD slangRootLength = GetEnvironmentVariableA("SLANG_ROOT", slangRootBuffer, static_cast<DWORD>(sizeof(slangRootBuffer)));
+	if (slangRootLength > 0 && slangRootLength < sizeof(slangRootBuffer))
+	{
+		std::filesystem::path candidate = std::filesystem::path(slangRootBuffer) / "bin" / "slangc.exe";
+		if (std::filesystem::exists(candidate))
+		{
+			compilerPath = candidate;
+			return true;
+		}
+	}
+
 	std::filesystem::path thirdPartyRoot = mRepoRoot / "3rdParty";
 	if (!std::filesystem::exists(thirdPartyRoot))
 	{

apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOFormat.cpp

@@ -50,7 +50,16 @@ uint16_t Component(uint32_t word, unsigned index)
 
 const char* VideoIOPixelFormatName(VideoIOPixelFormat format)
 {
-	return format == VideoIOPixelFormat::V210 ? "10-bit YUV v210" : "8-bit YUV UYVY";
+	switch (format)
+	{
+	case VideoIOPixelFormat::V210:
+		return "10-bit YUV v210";
+	case VideoIOPixelFormat::Bgra8:
+		return "8-bit BGRA";
+	case VideoIOPixelFormat::Uyvy8:
+	default:
+		return "8-bit YUV UYVY";
+	}
 }
 
 bool VideoIOPixelFormatIsTenBit(VideoIOPixelFormat format)
@@ -58,21 +67,32 @@ bool VideoIOPixelFormatIsTenBit(VideoIOPixelFormat format)
 	return format == VideoIOPixelFormat::V210;
 }
 
-BMDPixelFormat DeckLinkPixelFormatForVideoIO(VideoIOPixelFormat format)
-{
-	return format == VideoIOPixelFormat::V210 ? bmdFormat10BitYUV : bmdFormat8BitYUV;
-}
-
-VideoIOPixelFormat VideoIOPixelFormatFromDeckLink(BMDPixelFormat format)
-{
-	return format == bmdFormat10BitYUV ? VideoIOPixelFormat::V210 : VideoIOPixelFormat::Uyvy8;
-}
-
 VideoIOPixelFormat ChoosePreferredVideoIOFormat(bool tenBitSupported)
 {
 	return tenBitSupported ? VideoIOPixelFormat::V210 : VideoIOPixelFormat::Uyvy8;
 }
 
+unsigned VideoIOBytesPerPixel(VideoIOPixelFormat format)
+{
+	switch (format)
+	{
+	case VideoIOPixelFormat::Uyvy8:
+		return 2u;
+	case VideoIOPixelFormat::Bgra8:
+		return 4u;
+	case VideoIOPixelFormat::V210:
+	default:
+		return 0u;
+	}
+}
+
+unsigned VideoIORowBytes(VideoIOPixelFormat format, unsigned frameWidth)
+{
+	if (format == VideoIOPixelFormat::V210)
+		return MinimumV210RowBytes(frameWidth);
+	return frameWidth * VideoIOBytesPerPixel(format);
+}
+
 unsigned PackedTextureWidthFromRowBytes(unsigned rowBytes)
 {
 	return (rowBytes + 3u) / 4u;

apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOFormat.h

@@ -1,14 +1,13 @@
 #pragma once
 
-#include "DeckLinkAPI_h.h"
-
 #include <array>
 #include <cstdint>
 
 enum class VideoIOPixelFormat
 {
 	Uyvy8,
-	V210
+	V210,
+	Bgra8
 };
 
 struct V210CodeValues
@@ -27,9 +26,9 @@ struct V210SixPixelBlock
 
 const char* VideoIOPixelFormatName(VideoIOPixelFormat format);
 bool VideoIOPixelFormatIsTenBit(VideoIOPixelFormat format);
-BMDPixelFormat DeckLinkPixelFormatForVideoIO(VideoIOPixelFormat format);
-VideoIOPixelFormat VideoIOPixelFormatFromDeckLink(BMDPixelFormat format);
 VideoIOPixelFormat ChoosePreferredVideoIOFormat(bool tenBitSupported);
+unsigned VideoIOBytesPerPixel(VideoIOPixelFormat format);
+unsigned VideoIORowBytes(VideoIOPixelFormat format, unsigned frameWidth);
 unsigned PackedTextureWidthFromRowBytes(unsigned rowBytes);
 unsigned MinimumV210RowBytes(unsigned frameWidth);
 unsigned ActiveV210WordsForWidth(unsigned frameWidth);

apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h

@@ -0,0 +1,137 @@
+#pragma once
+
+#include "DeckLinkDisplayMode.h"
+#include "VideoIOFormat.h"
+
+#include <cstdint>
+#include <functional>
+#include <string>
+
+enum class VideoIOBackend
+{
+	DeckLink
+};
+
+enum class VideoIOCompletionResult
+{
+	Completed,
+	DisplayedLate,
+	Dropped,
+	Flushed,
+	Unknown
+};
+
+struct VideoIOConfig
+{
+	VideoFormatSelection videoModes;
+	bool externalKeyingEnabled = false;
+	bool preferTenBit = true;
+};
+
+struct VideoIOState
+{
+	FrameSize inputFrameSize;
+	FrameSize outputFrameSize;
+	VideoIOPixelFormat inputPixelFormat = VideoIOPixelFormat::Uyvy8;
+	VideoIOPixelFormat outputPixelFormat = VideoIOPixelFormat::Bgra8;
+	unsigned inputFrameRowBytes = 0;
+	unsigned outputFrameRowBytes = 0;
+	unsigned captureTextureWidth = 0;
+	unsigned outputPackTextureWidth = 0;
+	std::string inputDisplayModeName = "1080p59.94";
+	std::string outputDisplayModeName = "1080p59.94";
+	std::string outputModelName;
+	std::string statusMessage;
+	std::string formatStatusMessage;
+	bool hasInputDevice = false;
+	bool hasInputSource = false;
+	bool supportsInternalKeying = false;
+	bool supportsExternalKeying = false;
+	bool keyerInterfaceAvailable = false;
+	bool externalKeyingActive = false;
+	double frameBudgetMilliseconds = 0.0;
+};
+
+struct VideoIOFrame
+{
+	void* bytes = nullptr;
+	long rowBytes = 0;
+	unsigned width = 0;
+	unsigned height = 0;
+	VideoIOPixelFormat pixelFormat = VideoIOPixelFormat::Uyvy8;
+	bool hasNoInputSource = false;
+};
+
+struct VideoIOOutputFrame
+{
+	void* bytes = nullptr;
+	long rowBytes = 0;
+	unsigned width = 0;
+	unsigned height = 0;
+	VideoIOPixelFormat pixelFormat = VideoIOPixelFormat::Bgra8;
+	void* nativeFrame = nullptr;
+	void* nativeBuffer = nullptr;
+};
+
+struct VideoIOCompletion
+{
+	VideoIOCompletionResult result = VideoIOCompletionResult::Completed;
+};
+
+struct VideoIOScheduleTime
+{
+	int64_t streamTime = 0;
+	int64_t duration = 0;
+	int64_t timeScale = 0;
+	uint64_t frameIndex = 0;
+};
+
+class VideoIODevice
+{
+public:
+	using InputFrameCallback = std::function<void(const VideoIOFrame&)>;
+	using OutputFrameCallback = std::function<void(const VideoIOCompletion&)>;
+
+	virtual ~VideoIODevice() = default;
+	virtual void ReleaseResources() = 0;
+	virtual bool DiscoverDevicesAndModes(const VideoFormatSelection& videoModes, std::string& error) = 0;
+	virtual bool SelectPreferredFormats(const VideoFormatSelection& videoModes, std::string& error) = 0;
+	virtual bool ConfigureInput(InputFrameCallback callback, const VideoFormat& inputVideoMode, std::string& error) = 0;
+	virtual bool ConfigureOutput(OutputFrameCallback callback, const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error) = 0;
+	virtual bool Start() = 0;
+	virtual bool Stop() = 0;
+	virtual const VideoIOState& State() const = 0;
+	virtual VideoIOState& MutableState() = 0;
+	virtual bool BeginOutputFrame(VideoIOOutputFrame& frame) = 0;
+	virtual void EndOutputFrame(VideoIOOutputFrame& frame) = 0;
+	virtual bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) = 0;
+	virtual void AccountForCompletionResult(VideoIOCompletionResult result) = 0;
+
+	bool HasInputDevice() const { return State().hasInputDevice; }
+	bool HasInputSource() const { return State().hasInputSource; }
+	bool InputOutputDimensionsDiffer() const { return State().inputFrameSize != State().outputFrameSize; }
+	const FrameSize& InputFrameSize() const { return State().inputFrameSize; }
+	const FrameSize& OutputFrameSize() const { return State().outputFrameSize; }
+	unsigned InputFrameWidth() const { return State().inputFrameSize.width; }
+	unsigned InputFrameHeight() const { return State().inputFrameSize.height; }
+	unsigned OutputFrameWidth() const { return State().outputFrameSize.width; }
+	unsigned OutputFrameHeight() const { return State().outputFrameSize.height; }
+	VideoIOPixelFormat InputPixelFormat() const { return State().inputPixelFormat; }
+	VideoIOPixelFormat OutputPixelFormat() const { return State().outputPixelFormat; }
+	bool InputIsTenBit() const { return VideoIOPixelFormatIsTenBit(State().inputPixelFormat); }
+	bool OutputIsTenBit() const { return VideoIOPixelFormatIsTenBit(State().outputPixelFormat); }
+	unsigned InputFrameRowBytes() const { return State().inputFrameRowBytes; }
+	unsigned OutputFrameRowBytes() const { return State().outputFrameRowBytes; }
+	unsigned CaptureTextureWidth() const { return State().captureTextureWidth; }
+	unsigned OutputPackTextureWidth() const { return State().outputPackTextureWidth; }
+	const std::string& FormatStatusMessage() const { return State().formatStatusMessage; }
+	const std::string& InputDisplayModeName() const { return State().inputDisplayModeName; }
+	const std::string& OutputModelName() const { return State().outputModelName; }
+	bool SupportsInternalKeying() const { return State().supportsInternalKeying; }
+	bool SupportsExternalKeying() const { return State().supportsExternalKeying; }
+	bool KeyerInterfaceAvailable() const { return State().keyerInterfaceAvailable; }
+	bool ExternalKeyingActive() const { return State().externalKeyingActive; }
+	const std::string& StatusMessage() const { return State().statusMessage; }
+	double FrameBudgetMilliseconds() const { return State().frameBudgetMilliseconds; }
+	void SetStatusMessage(const std::string& message) { MutableState().statusMessage = message; }
+};

apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.cpp

@@ -0,0 +1,37 @@
+#include "VideoPlayoutScheduler.h"
+
+void VideoPlayoutScheduler::Configure(int64_t frameDuration, int64_t timeScale)
+{
+	mFrameDuration = frameDuration;
+	mTimeScale = timeScale;
+	Reset();
+}
+
+void VideoPlayoutScheduler::Reset()
+{
+	mScheduledFrameIndex = 0;
+}
+
+VideoIOScheduleTime VideoPlayoutScheduler::NextScheduleTime()
+{
+	VideoIOScheduleTime time;
+	time.streamTime = static_cast<int64_t>(mScheduledFrameIndex) * mFrameDuration;
+	time.duration = mFrameDuration;
+	time.timeScale = mTimeScale;
+	time.frameIndex = mScheduledFrameIndex;
+	++mScheduledFrameIndex;
+	return time;
+}
+
+void VideoPlayoutScheduler::AccountForCompletionResult(VideoIOCompletionResult result)
+{
+	if (result == VideoIOCompletionResult::DisplayedLate || result == VideoIOCompletionResult::Dropped)
+		mScheduledFrameIndex += 2;
+}
+
+double VideoPlayoutScheduler::FrameBudgetMilliseconds() const
+{
+	return mTimeScale != 0
+		? (static_cast<double>(mFrameDuration) * 1000.0) / static_cast<double>(mTimeScale)
+		: 0.0;
+}

apps/LoopThroughWithOpenGLCompositing/videoio/VideoPlayoutScheduler.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "VideoIOTypes.h"
+
+#include <cstdint>
+
+class VideoPlayoutScheduler
+{
+public:
+	void Configure(int64_t frameDuration, int64_t timeScale);
+	void Reset();
+	VideoIOScheduleTime NextScheduleTime();
+	void AccountForCompletionResult(VideoIOCompletionResult result);
+	double FrameBudgetMilliseconds() const;
+	uint64_t ScheduledFrameIndex() const { return mScheduledFrameIndex; }
+	int64_t TimeScale() const { return mTimeScale; }
+
+private:
+	int64_t mFrameDuration = 0;
+	int64_t mTimeScale = 0;
+	uint64_t mScheduledFrameIndex = 0;
+};

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkFrameTransfer.cpp

@@ -1,11 +1,11 @@
 #include "DeckLinkFrameTransfer.h"
 
-#include "OpenGLComposite.h"
+#include "DeckLinkSession.h"
 
 ////////////////////////////////////////////
 // DeckLink Capture Delegate Class
 ////////////////////////////////////////////
-CaptureDelegate::CaptureDelegate(OpenGLComposite* pOwner) :
+CaptureDelegate::CaptureDelegate(DeckLinkSession* pOwner) :
 	m_pOwner(pOwner),
 	mRefCount(1)
 {
@@ -39,7 +39,7 @@ HRESULT CaptureDelegate::VideoInputFrameArrived(IDeckLinkVideoInputFrame* inputF
 	}
 
 	bool hasNoInputSource = (inputFrame->GetFlags() & bmdFrameHasNoInputSource) == bmdFrameHasNoInputSource;
-	m_pOwner->VideoFrameArrived(inputFrame, hasNoInputSource);
+	m_pOwner->HandleVideoInputFrame(inputFrame, hasNoInputSource);
 	return S_OK;
 }
 
@@ -51,7 +51,7 @@ HRESULT CaptureDelegate::VideoInputFormatChanged(BMDVideoInputFormatChangedEvent
 ////////////////////////////////////////////
 // DeckLink Playout Delegate Class
 ////////////////////////////////////////////
-PlayoutDelegate::PlayoutDelegate(OpenGLComposite* pOwner) :
+PlayoutDelegate::PlayoutDelegate(DeckLinkSession* pOwner) :
 	m_pOwner(pOwner),
 	mRefCount(1)
 {
@@ -94,7 +94,7 @@ HRESULT PlayoutDelegate::ScheduledFrameCompleted(IDeckLinkVideoFrame* completedF
 		OutputDebugStringA("ScheduledFrameCompleted() frame did not complete: Unknown error\n");
 	}
 
-	m_pOwner->PlayoutFrameCompleted(completedFrame, result);
+	m_pOwner->HandlePlayoutFrameCompleted(completedFrame, result);
 	return S_OK;
 }
 

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkFrameTransfer.h

@@ -6,18 +6,18 @@
 
 #include "DeckLinkAPI_h.h"
 
-class OpenGLComposite;
+class DeckLinkSession;
 
 ////////////////////////////////////////////
 // Capture Delegate Class
 ////////////////////////////////////////////
 class CaptureDelegate : public IDeckLinkInputCallback
 {
-	OpenGLComposite* m_pOwner;
+	DeckLinkSession* m_pOwner;
 	LONG mRefCount;
 
 public:
-	CaptureDelegate(OpenGLComposite* pOwner);
+	CaptureDelegate(DeckLinkSession* pOwner);
 
 	// IUnknown needs only a dummy implementation
 	virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID* ppv);
@@ -33,11 +33,11 @@ public:
 ////////////////////////////////////////////
 class PlayoutDelegate : public IDeckLinkVideoOutputCallback
 {
-	OpenGLComposite* m_pOwner;
+	DeckLinkSession* m_pOwner;
 	LONG mRefCount;
 
 public:
-	PlayoutDelegate(OpenGLComposite* pOwner);
+	PlayoutDelegate(DeckLinkSession* pOwner);
 
 	// IUnknown needs only a dummy implementation
 	virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID* ppv);

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp

@@ -7,6 +7,7 @@
 #include <cstring>
 #include <new>
 #include <sstream>
+#include <utility>
 #include <vector>
 
 namespace
@@ -82,7 +83,7 @@ void DeckLinkSession::ReleaseResources()
 	if (keyer != nullptr)
 	{
 		keyer->Disable();
-		externalKeyingActive = false;
+		mState.externalKeyingActive = false;
 	}
 	keyer.Release();
 
@@ -97,8 +98,8 @@ bool DeckLinkSession::DiscoverDevicesAndModes(const VideoFormatSelection& videoM
 	CComPtr<IDeckLinkDisplayMode> inputMode;
 	CComPtr<IDeckLinkDisplayMode> outputMode;
 
-	inputDisplayModeName = videoModes.input.displayName;
+	mState.inputDisplayModeName = videoModes.input.displayName;
-	outputDisplayModeName = videoModes.output.displayName;
+	mState.outputDisplayModeName = videoModes.output.displayName;
 
 	HRESULT result = CoCreateInstance(CLSID_CDeckLinkIterator, nullptr, CLSCTX_ALL, IID_IDeckLinkIterator, reinterpret_cast<void**>(&deckLinkIterator));
 	if (FAILED(result))
@@ -150,9 +151,9 @@ bool DeckLinkSession::DiscoverDevicesAndModes(const VideoFormatSelection& videoM
 				output.Release();
 			else
 			{
-				outputModelName = modelName;
+				mState.outputModelName = modelName;
-				supportsInternalKeying = deviceSupportsInternalKeying;
+				mState.supportsInternalKeying = deviceSupportsInternalKeying;
-				supportsExternalKeying = deviceSupportsExternalKeying;
+				mState.supportsExternalKeying = deviceSupportsExternalKeying;
 			}
 		}
 
@@ -200,18 +201,24 @@ bool DeckLinkSession::DiscoverDevicesAndModes(const VideoFormatSelection& videoM
 		return false;
 	}
 
-	outputFrameSize = { static_cast<unsigned>(outputMode->GetWidth()), static_cast<unsigned>(outputMode->GetHeight()) };
+	mState.outputFrameSize = { static_cast<unsigned>(outputMode->GetWidth()), static_cast<unsigned>(outputMode->GetHeight()) };
-	inputFrameSize = inputMode
+	mState.inputFrameSize = inputMode
 		? FrameSize{ static_cast<unsigned>(inputMode->GetWidth()), static_cast<unsigned>(inputMode->GetHeight()) }
-		: outputFrameSize;
+		: mState.outputFrameSize;
 	if (!input)
-		inputDisplayModeName = "No input - black frame";
+		mState.inputDisplayModeName = "No input - black frame";
+	BMDTimeValue frameDuration = 0;
+	BMDTimeScale frameTimescale = 0;
 	outputMode->GetFrameRate(&frameDuration, &frameTimescale);
+	mScheduler.Configure(frameDuration, frameTimescale);
+	mState.frameBudgetMilliseconds = mScheduler.FrameBudgetMilliseconds();
 
-	inputFrameRowBytes = inputFrameSize.width * 2u;
+	mState.inputFrameRowBytes = mState.inputFrameSize.width * 2u;
-	outputFrameRowBytes = outputFrameSize.width * 4u;
+	mState.outputFrameRowBytes = mState.outputFrameSize.width * 4u;
-	captureTextureWidth = inputFrameSize.width / 2u;
+	mState.captureTextureWidth = mState.inputFrameSize.width / 2u;
-	outputPackTextureWidth = outputFrameSize.width;
+	mState.outputPackTextureWidth = mState.outputFrameSize.width;
+	mState.hasInputDevice = input != nullptr;
+	mState.hasInputSource = false;
 
 	return true;
 }
@@ -224,95 +231,95 @@ bool DeckLinkSession::SelectPreferredFormats(const VideoFormatSelection& videoMo
 		return false;
 	}
 
-	formatStatusMessage.clear();
+	mState.formatStatusMessage.clear();
 
 	const bool inputTenBitSupported = input != nullptr && InputSupportsFormat(input, videoModes.input.displayMode, bmdFormat10BitYUV);
-	inputPixelFormat = input != nullptr ? ChoosePreferredVideoIOFormat(inputTenBitSupported) : VideoIOPixelFormat::Uyvy8;
+	mState.inputPixelFormat = input != nullptr ? ChoosePreferredVideoIOFormat(inputTenBitSupported) : VideoIOPixelFormat::Uyvy8;
 	if (input != nullptr && !inputTenBitSupported)
-		formatStatusMessage += "DeckLink input does not report 10-bit YUV support for the configured mode; using 8-bit capture. ";
+		mState.formatStatusMessage += "DeckLink input does not report 10-bit YUV support for the configured mode; using 8-bit capture. ";
 
 	const bool outputTenBitSupported = OutputSupportsFormat(output, videoModes.output.displayMode, bmdFormat10BitYUV);
-	outputPixelFormat = ChoosePreferredVideoIOFormat(outputTenBitSupported);
+	mState.outputPixelFormat = outputTenBitSupported ? VideoIOPixelFormat::V210 : VideoIOPixelFormat::Bgra8;
 	if (!outputTenBitSupported)
-		formatStatusMessage += "DeckLink output does not report 10-bit YUV support for the configured mode; using 8-bit BGRA output. ";
+		mState.formatStatusMessage += "DeckLink output does not report 10-bit YUV support for the configured mode; using 8-bit BGRA output. ";
 
 	int deckLinkOutputRowBytes = 0;
-	if (output->RowBytesForPixelFormat(OutputIsTenBit() ? bmdFormat10BitYUV : bmdFormat8BitBGRA, outputFrameSize.width, &deckLinkOutputRowBytes) != S_OK)
+	if (output->RowBytesForPixelFormat(DeckLinkPixelFormatForVideoIO(mState.outputPixelFormat), mState.outputFrameSize.width, &deckLinkOutputRowBytes) != S_OK)
 	{
 		error = "DeckLink output setup failed while calculating output row bytes.";
 		return false;
 	}
-	outputFrameRowBytes = static_cast<unsigned>(deckLinkOutputRowBytes);
+	mState.outputFrameRowBytes = static_cast<unsigned>(deckLinkOutputRowBytes);
-	outputPackTextureWidth = OutputIsTenBit()
+	mState.outputPackTextureWidth = OutputIsTenBit()
-		? PackedTextureWidthFromRowBytes(outputFrameRowBytes)
+		? PackedTextureWidthFromRowBytes(mState.outputFrameRowBytes)
-		: outputFrameSize.width;
+		: mState.outputFrameSize.width;
 
 	if (InputIsTenBit())
 	{
 		int deckLinkInputRowBytes = 0;
-		if (output->RowBytesForPixelFormat(bmdFormat10BitYUV, inputFrameSize.width, &deckLinkInputRowBytes) == S_OK)
+		if (output->RowBytesForPixelFormat(bmdFormat10BitYUV, mState.inputFrameSize.width, &deckLinkInputRowBytes) == S_OK)
-			inputFrameRowBytes = static_cast<unsigned>(deckLinkInputRowBytes);
+			mState.inputFrameRowBytes = static_cast<unsigned>(deckLinkInputRowBytes);
 		else
-			inputFrameRowBytes = MinimumV210RowBytes(inputFrameSize.width);
+			mState.inputFrameRowBytes = MinimumV210RowBytes(mState.inputFrameSize.width);
 	}
 	else
 	{
-		inputFrameRowBytes = inputFrameSize.width * 2u;
+		mState.inputFrameRowBytes = mState.inputFrameSize.width * 2u;
 	}
-	captureTextureWidth = InputIsTenBit()
+	mState.captureTextureWidth = InputIsTenBit()
-		? PackedTextureWidthFromRowBytes(inputFrameRowBytes)
+		? PackedTextureWidthFromRowBytes(mState.inputFrameRowBytes)
-		: inputFrameSize.width / 2u;
+		: mState.inputFrameSize.width / 2u;
 
 	std::ostringstream status;
-	status << "DeckLink formats: capture " << (input ? VideoIOPixelFormatName(inputPixelFormat) : "none")
+	status << "DeckLink formats: capture " << (input ? VideoIOPixelFormatName(mState.inputPixelFormat) : "none")
-		<< ", output " << (OutputIsTenBit() ? "10-bit YUV v210" : "8-bit BGRA") << ".";
+		<< ", output " << VideoIOPixelFormatName(mState.outputPixelFormat) << ".";
-	if (!formatStatusMessage.empty())
+	if (!mState.formatStatusMessage.empty())
-		status << " " << formatStatusMessage;
+		status << " " << mState.formatStatusMessage;
-	formatStatusMessage = status.str();
+	mState.formatStatusMessage = status.str();
 	return true;
 }
 
-bool DeckLinkSession::ConfigureInput(OpenGLComposite* owner, HDC hdc, HGLRC hglrc, const VideoFormat& inputVideoMode, std::string& error)
+bool DeckLinkSession::ConfigureInput(InputFrameCallback callback, const VideoFormat& inputVideoMode, std::string& error)
 {
-	(void)hdc;
+	mInputFrameCallback = std::move(callback);
-	(void)hglrc;
 
 	if (!input)
 	{
-		hasNoInputSource = true;
+		mState.hasInputSource = false;
-		inputDisplayModeName = "No input - black frame";
+		mState.inputDisplayModeName = "No input - black frame";
 		return true;
 	}
 
-	const BMDPixelFormat deckLinkInputPixelFormat = DeckLinkPixelFormatForVideoIO(inputPixelFormat);
+	const BMDPixelFormat deckLinkInputPixelFormat = DeckLinkPixelFormatForVideoIO(mState.inputPixelFormat);
 	if (input->EnableVideoInput(inputVideoMode.displayMode, deckLinkInputPixelFormat, bmdVideoInputFlagDefault) != S_OK)
 	{
-		if (inputPixelFormat == VideoIOPixelFormat::V210)
+		if (mState.inputPixelFormat == VideoIOPixelFormat::V210)
 		{
 			OutputDebugStringA("DeckLink 10-bit input could not be enabled; falling back to 8-bit capture.\n");
-			inputPixelFormat = VideoIOPixelFormat::Uyvy8;
+			mState.inputPixelFormat = VideoIOPixelFormat::Uyvy8;
-			inputFrameRowBytes = inputFrameSize.width * 2u;
+			mState.inputFrameRowBytes = mState.inputFrameSize.width * 2u;
-			captureTextureWidth = inputFrameSize.width / 2u;
+			mState.captureTextureWidth = mState.inputFrameSize.width / 2u;
 			if (input->EnableVideoInput(inputVideoMode.displayMode, bmdFormat8BitYUV, bmdVideoInputFlagDefault) == S_OK)
 			{
 				std::ostringstream status;
-				status << "DeckLink formats: capture " << VideoIOPixelFormatName(inputPixelFormat)
+				status << "DeckLink formats: capture " << VideoIOPixelFormatName(mState.inputPixelFormat)
-					<< ", output " << (OutputIsTenBit() ? "10-bit YUV v210" : "8-bit BGRA")
+					<< ", output " << VideoIOPixelFormatName(mState.outputPixelFormat)
 					<< ". DeckLink 10-bit input enable failed; using 8-bit capture.";
-				formatStatusMessage = status.str();
+				mState.formatStatusMessage = status.str();
 				goto input_enabled;
 			}
 		}
 
 		OutputDebugStringA("DeckLink input could not be enabled; continuing in output-only black-frame mode.\n");
 		input.Release();
-		hasNoInputSource = true;
+		mState.hasInputDevice = false;
-		inputDisplayModeName = "No input - black frame";
+		mState.hasInputSource = false;
+		mState.inputDisplayModeName = "No input - black frame";
 		return true;
 	}
 
 input_enabled:
-	captureDelegate.Attach(new (std::nothrow) CaptureDelegate(owner));
+	captureDelegate.Attach(new (std::nothrow) CaptureDelegate(this));
 	if (captureDelegate == nullptr)
 	{
 		error = "DeckLink input setup failed while creating the capture callback.";
@@ -327,10 +334,9 @@ input_enabled:
 	return true;
 }
 
-bool DeckLinkSession::ConfigureOutput(OpenGLComposite* owner, HDC hdc, HGLRC hglrc, const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error)
+bool DeckLinkSession::ConfigureOutput(OutputFrameCallback callback, const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error)
 {
-	(void)hdc;
+	mOutputFrameCallback = std::move(callback);
-	(void)hglrc;
 
 	if (output->EnableVideoOutput(outputVideoMode.displayMode, bmdVideoOutputFlagDefault) != S_OK)
 	{
@@ -339,39 +345,39 @@ bool DeckLinkSession::ConfigureOutput(OpenGLComposite* owner, HDC hdc, HGLRC hgl
 	}
 
 	if (output->QueryInterface(IID_IDeckLinkKeyer, (void**)&keyer) == S_OK && keyer != NULL)
-		keyerInterfaceAvailable = true;
+		mState.keyerInterfaceAvailable = true;
 
 	if (externalKeyingEnabled)
 	{
-		if (!supportsExternalKeying)
+		if (!mState.supportsExternalKeying)
 		{
-			statusMessage = "External keying was requested, but the selected DeckLink output does not report external keying support.";
+			mState.statusMessage = "External keying was requested, but the selected DeckLink output does not report external keying support.";
 		}
-		else if (!keyerInterfaceAvailable)
+		else if (!mState.keyerInterfaceAvailable)
 		{
-			statusMessage = "External keying was requested, but the selected DeckLink output does not expose the IDeckLinkKeyer interface.";
+			mState.statusMessage = "External keying was requested, but the selected DeckLink output does not expose the IDeckLinkKeyer interface.";
 		}
 		else if (keyer->Enable(TRUE) != S_OK || keyer->SetLevel(255) != S_OK)
 		{
-			statusMessage = "External keying was requested, but enabling the DeckLink keyer failed.";
+			mState.statusMessage = "External keying was requested, but enabling the DeckLink keyer failed.";
 		}
 		else
 		{
-			externalKeyingActive = true;
+			mState.externalKeyingActive = true;
-			statusMessage = "External keying is active on the selected DeckLink output.";
+			mState.statusMessage = "External keying is active on the selected DeckLink output.";
 		}
 	}
-	else if (supportsExternalKeying)
+	else if (mState.supportsExternalKeying)
 	{
-		statusMessage = "Selected DeckLink output supports external keying. Set enableExternalKeying to true in runtime-host.json to request it.";
+		mState.statusMessage = "Selected DeckLink output supports external keying. Set enableExternalKeying to true in runtime-host.json to request it.";
 	}
 
 	for (int i = 0; i < 10; i++)
 	{
 		CComPtr<IDeckLinkMutableVideoFrame> outputFrame;
 
-		const BMDPixelFormat deckLinkOutputPixelFormat = OutputIsTenBit() ? bmdFormat10BitYUV : bmdFormat8BitBGRA;
+		const BMDPixelFormat deckLinkOutputPixelFormat = DeckLinkPixelFormatForVideoIO(mState.outputPixelFormat);
-		if (output->CreateVideoFrame(outputFrameSize.width, outputFrameSize.height, outputFrameRowBytes, deckLinkOutputPixelFormat, bmdFrameFlagFlipVertical, &outputFrame) != S_OK)
+		if (output->CreateVideoFrame(mState.outputFrameSize.width, mState.outputFrameSize.height, mState.outputFrameRowBytes, deckLinkOutputPixelFormat, bmdFrameFlagFlipVertical, &outputFrame) != S_OK)
 		{
 			error = "DeckLink output setup failed while creating an output video frame.";
 			return false;
@@ -380,7 +386,7 @@ bool DeckLinkSession::ConfigureOutput(OpenGLComposite* owner, HDC hdc, HGLRC hgl
 		outputVideoFrameQueue.push_back(outputFrame);
 	}
 
-	playoutDelegate.Attach(new (std::nothrow) PlayoutDelegate(owner));
+	playoutDelegate.Attach(new (std::nothrow) PlayoutDelegate(this));
 	if (playoutDelegate == nullptr)
 	{
 		error = "DeckLink output setup failed while creating the playout callback.";
@@ -393,45 +399,73 @@ bool DeckLinkSession::ConfigureOutput(OpenGLComposite* owner, HDC hdc, HGLRC hgl
 		return false;
 	}
 
-	if (!formatStatusMessage.empty())
+	if (!mState.formatStatusMessage.empty())
-		statusMessage = statusMessage.empty() ? formatStatusMessage : formatStatusMessage + " " + statusMessage;
+		mState.statusMessage = mState.statusMessage.empty() ? mState.formatStatusMessage : mState.formatStatusMessage + " " + mState.statusMessage;
 
 	return true;
 }
 
 double DeckLinkSession::FrameBudgetMilliseconds() const
 {
-	return frameTimescale != 0
+	return mScheduler.FrameBudgetMilliseconds();
-		? (static_cast<double>(frameDuration) * 1000.0) / static_cast<double>(frameTimescale)
-		: 0.0;
 }
 
-IDeckLinkMutableVideoFrame* DeckLinkSession::RotateOutputFrame()
+bool DeckLinkSession::BeginOutputFrame(VideoIOOutputFrame& frame)
 {
 	CComPtr<IDeckLinkMutableVideoFrame> outputVideoFrame = outputVideoFrameQueue.front();
 	outputVideoFrameQueue.push_back(outputVideoFrame);
 	outputVideoFrameQueue.pop_front();
-	return outputVideoFrame.p;
-}
 
-void DeckLinkSession::AccountForCompletionResult(BMDOutputFrameCompletionResult completionResult)
+	CComPtr<IDeckLinkVideoBuffer> outputVideoFrameBuffer;
-{
+	if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK)
-	if (completionResult == bmdOutputFrameDisplayedLate || completionResult == bmdOutputFrameDropped)
+		return false;
-		totalPlayoutFrames += 2;
+
-}
+	if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK)
-
+		return false;
-bool DeckLinkSession::ScheduleOutputFrame(IDeckLinkMutableVideoFrame* outputVideoFrame)
+
-{
+	void* pFrame = nullptr;
-	if (output->ScheduleVideoFrame(outputVideoFrame, (totalPlayoutFrames * frameDuration), frameDuration, frameTimescale) != S_OK)
+	outputVideoFrameBuffer->GetBytes(&pFrame);
+
+	frame.bytes = pFrame;
+	frame.rowBytes = outputVideoFrame->GetRowBytes();
+	frame.width = mState.outputFrameSize.width;
+	frame.height = mState.outputFrameSize.height;
+	frame.pixelFormat = mState.outputPixelFormat;
+	frame.nativeFrame = outputVideoFrame.p;
+	frame.nativeBuffer = outputVideoFrameBuffer.Detach();
+	return true;
+}
+
+void DeckLinkSession::EndOutputFrame(VideoIOOutputFrame& frame)
+{
+	IDeckLinkVideoBuffer* outputVideoFrameBuffer = static_cast<IDeckLinkVideoBuffer*>(frame.nativeBuffer);
+	if (outputVideoFrameBuffer != nullptr)
+	{
+		outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite);
+		outputVideoFrameBuffer->Release();
+	}
+	frame.nativeBuffer = nullptr;
+	frame.bytes = nullptr;
+}
+
+void DeckLinkSession::AccountForCompletionResult(VideoIOCompletionResult completionResult)
+{
+	mScheduler.AccountForCompletionResult(completionResult);
+}
+
+bool DeckLinkSession::ScheduleOutputFrame(const VideoIOOutputFrame& frame)
+{
+	IDeckLinkMutableVideoFrame* outputVideoFrame = static_cast<IDeckLinkMutableVideoFrame*>(frame.nativeFrame);
+	const VideoIOScheduleTime scheduleTime = mScheduler.NextScheduleTime();
+	if (outputVideoFrame == nullptr || output->ScheduleVideoFrame(outputVideoFrame, scheduleTime.streamTime, scheduleTime.duration, scheduleTime.timeScale) != S_OK)
 		return false;
 
-	totalPlayoutFrames++;
 	return true;
 }
 
 bool DeckLinkSession::Start()
 {
-	totalPlayoutFrames = 0;
+	mScheduler.Reset();
 	if (!output)
 	{
 		MessageBoxA(NULL, "Cannot start playout because no DeckLink output device is available.", "DeckLink start failed", MB_OK | MB_ICONERROR);
@@ -462,19 +496,18 @@ bool DeckLinkSession::Start()
 			return false;
 		}
 
-		void* pFrame;
+		void* pFrame = nullptr;
 		outputVideoFrameBuffer->GetBytes((void**)&pFrame);
-		memset(pFrame, 0, outputVideoFrame->GetRowBytes() * outputFrameSize.height);
+		memset(pFrame, 0, outputVideoFrame->GetRowBytes() * mState.outputFrameSize.height);
 
 		outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite);
 
-		if (output->ScheduleVideoFrame(outputVideoFrame, (totalPlayoutFrames * frameDuration), frameDuration, frameTimescale) != S_OK)
+		const VideoIOScheduleTime scheduleTime = mScheduler.NextScheduleTime();
+		if (output->ScheduleVideoFrame(outputVideoFrame, scheduleTime.streamTime, scheduleTime.duration, scheduleTime.timeScale) != S_OK)
 		{
 			MessageBoxA(NULL, "Could not schedule a preroll output frame.", "DeckLink start failed", MB_OK | MB_ICONERROR);
 			return false;
 		}
-
-		totalPlayoutFrames++;
 	}
 
 	if (input)
@@ -485,7 +518,7 @@ bool DeckLinkSession::Start()
 			return false;
 		}
 	}
-	if (output->StartScheduledPlayback(0, frameTimescale, 1.0) != S_OK)
+	if (output->StartScheduledPlayback(0, mScheduler.TimeScale(), 1.0) != S_OK)
 	{
 		MessageBoxA(NULL, "Could not start DeckLink scheduled playback.", "DeckLink start failed", MB_OK | MB_ICONERROR);
 		return false;
@@ -499,7 +532,7 @@ bool DeckLinkSession::Stop()
 	if (keyer != nullptr)
 	{
 		keyer->Disable();
-		externalKeyingActive = false;
+		mState.externalKeyingActive = false;
 	}
 
 	if (input)
@@ -516,3 +549,66 @@ bool DeckLinkSession::Stop()
 
 	return true;
 }
+
+void DeckLinkSession::HandleVideoInputFrame(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource)
+{
+	mState.hasInputSource = !hasNoInputSource;
+	if (hasNoInputSource || mInputFrameCallback == nullptr)
+	{
+		VideoIOFrame frame;
+		frame.width = mState.inputFrameSize.width;
+		frame.height = mState.inputFrameSize.height;
+		frame.pixelFormat = mState.inputPixelFormat;
+		frame.hasNoInputSource = hasNoInputSource;
+		if (mInputFrameCallback)
+			mInputFrameCallback(frame);
+		return;
+	}
+
+	CComPtr<IDeckLinkVideoBuffer> inputFrameBuffer;
+	void* videoPixels = nullptr;
+	if (inputFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&inputFrameBuffer) != S_OK)
+		return;
+	if (inputFrameBuffer->StartAccess(bmdBufferAccessRead) != S_OK)
+		return;
+
+	inputFrameBuffer->GetBytes(&videoPixels);
+
+	VideoIOFrame frame;
+	frame.bytes = videoPixels;
+	frame.rowBytes = inputFrame->GetRowBytes();
+	frame.width = static_cast<unsigned>(inputFrame->GetWidth());
+	frame.height = static_cast<unsigned>(inputFrame->GetHeight());
+	frame.pixelFormat = mState.inputPixelFormat;
+	frame.hasNoInputSource = hasNoInputSource;
+	mInputFrameCallback(frame);
+
+	inputFrameBuffer->EndAccess(bmdBufferAccessRead);
+}
+
+void DeckLinkSession::HandlePlayoutFrameCompleted(IDeckLinkVideoFrame*, BMDOutputFrameCompletionResult completionResult)
+{
+	if (!mOutputFrameCallback)
+		return;
+
+	VideoIOCompletion completion;
+	switch (completionResult)
+	{
+	case bmdOutputFrameDisplayedLate:
+		completion.result = VideoIOCompletionResult::DisplayedLate;
+		break;
+	case bmdOutputFrameDropped:
+		completion.result = VideoIOCompletionResult::Dropped;
+		break;
+	case bmdOutputFrameFlushed:
+		completion.result = VideoIOCompletionResult::Flushed;
+		break;
+	case bmdOutputFrameCompleted:
+		completion.result = VideoIOCompletionResult::Completed;
+		break;
+	default:
+		completion.result = VideoIOCompletionResult::Unknown;
+		break;
+	}
+	mOutputFrameCallback(completion);
+}

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h

@@ -0,0 +1,79 @@
+#pragma once
+
+#include "DeckLinkAPI_h.h"
+#include "DeckLinkDisplayMode.h"
+#include "DeckLinkFrameTransfer.h"
+#include "DeckLinkVideoIOFormat.h"
+#include "VideoIOFormat.h"
+#include "VideoIOTypes.h"
+#include "VideoPlayoutScheduler.h"
+
+#include <atlbase.h>
+#include <deque>
+#include <string>
+
+class OpenGLComposite;
+
+class DeckLinkSession : public VideoIODevice
+{
+public:
+	DeckLinkSession() = default;
+	~DeckLinkSession();
+
+	void ReleaseResources() override;
+	bool DiscoverDevicesAndModes(const VideoFormatSelection& videoModes, std::string& error) override;
+	bool SelectPreferredFormats(const VideoFormatSelection& videoModes, std::string& error) override;
+	bool ConfigureInput(InputFrameCallback callback, const VideoFormat& inputVideoMode, std::string& error) override;
+	bool ConfigureOutput(OutputFrameCallback callback, const VideoFormat& outputVideoMode, bool externalKeyingEnabled, std::string& error) override;
+	bool Start() override;
+	bool Stop() override;
+
+	bool HasInputDevice() const { return mState.hasInputDevice; }
+	bool HasInputSource() const { return mState.hasInputSource; }
+	void SetInputSourceMissing(bool missing) { mState.hasInputSource = !missing; }
+	bool InputOutputDimensionsDiffer() const { return mState.inputFrameSize != mState.outputFrameSize; }
+	const FrameSize& InputFrameSize() const { return mState.inputFrameSize; }
+	const FrameSize& OutputFrameSize() const { return mState.outputFrameSize; }
+	unsigned InputFrameWidth() const { return mState.inputFrameSize.width; }
+	unsigned InputFrameHeight() const { return mState.inputFrameSize.height; }
+	unsigned OutputFrameWidth() const { return mState.outputFrameSize.width; }
+	unsigned OutputFrameHeight() const { return mState.outputFrameSize.height; }
+	VideoIOPixelFormat InputPixelFormat() const { return mState.inputPixelFormat; }
+	VideoIOPixelFormat OutputPixelFormat() const { return mState.outputPixelFormat; }
+	bool InputIsTenBit() const { return VideoIOPixelFormatIsTenBit(mState.inputPixelFormat); }
+	bool OutputIsTenBit() const { return VideoIOPixelFormatIsTenBit(mState.outputPixelFormat); }
+	unsigned InputFrameRowBytes() const { return mState.inputFrameRowBytes; }
+	unsigned OutputFrameRowBytes() const { return mState.outputFrameRowBytes; }
+	unsigned CaptureTextureWidth() const { return mState.captureTextureWidth; }
+	unsigned OutputPackTextureWidth() const { return mState.outputPackTextureWidth; }
+	const std::string& FormatStatusMessage() const { return mState.formatStatusMessage; }
+	const std::string& InputDisplayModeName() const { return mState.inputDisplayModeName; }
+	const std::string& OutputModelName() const { return mState.outputModelName; }
+	bool SupportsInternalKeying() const { return mState.supportsInternalKeying; }
+	bool SupportsExternalKeying() const { return mState.supportsExternalKeying; }
+	bool KeyerInterfaceAvailable() const { return mState.keyerInterfaceAvailable; }
+	bool ExternalKeyingActive() const { return mState.externalKeyingActive; }
+	const std::string& StatusMessage() const { return mState.statusMessage; }
+	void SetStatusMessage(const std::string& message) { mState.statusMessage = message; }
+	const VideoIOState& State() const override { return mState; }
+	VideoIOState& MutableState() override { return mState; }
+	double FrameBudgetMilliseconds() const;
+	void AccountForCompletionResult(VideoIOCompletionResult completionResult) override;
+	bool BeginOutputFrame(VideoIOOutputFrame& frame) override;
+	void EndOutputFrame(VideoIOOutputFrame& frame) override;
+	bool ScheduleOutputFrame(const VideoIOOutputFrame& frame) override;
+	void HandleVideoInputFrame(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource);
+	void HandlePlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult);
+
+private:
+	CComPtr<CaptureDelegate>				captureDelegate;
+	CComPtr<PlayoutDelegate>				playoutDelegate;
+	CComPtr<IDeckLinkInput>					input;
+	CComPtr<IDeckLinkOutput>				output;
+	CComPtr<IDeckLinkKeyer>					keyer;
+	std::deque<CComPtr<IDeckLinkMutableVideoFrame>> outputVideoFrameQueue;
+	VideoIOState							mState;
+	VideoPlayoutScheduler					mScheduler;
+	InputFrameCallback						mInputFrameCallback;
+	OutputFrameCallback						mOutputFrameCallback;
+};

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkVideoIOFormat.cpp

@@ -0,0 +1,24 @@
+#include "DeckLinkVideoIOFormat.h"
+
+BMDPixelFormat DeckLinkPixelFormatForVideoIO(VideoIOPixelFormat format)
+{
+	switch (format)
+	{
+	case VideoIOPixelFormat::V210:
+		return bmdFormat10BitYUV;
+	case VideoIOPixelFormat::Bgra8:
+		return bmdFormat8BitBGRA;
+	case VideoIOPixelFormat::Uyvy8:
+	default:
+		return bmdFormat8BitYUV;
+	}
+}
+
+VideoIOPixelFormat VideoIOPixelFormatFromDeckLink(BMDPixelFormat format)
+{
+	if (format == bmdFormat10BitYUV)
+		return VideoIOPixelFormat::V210;
+	if (format == bmdFormat8BitBGRA)
+		return VideoIOPixelFormat::Bgra8;
+	return VideoIOPixelFormat::Uyvy8;
+}

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkVideoIOFormat.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "DeckLinkAPI_h.h"
+#include "VideoIOFormat.h"
+
+BMDPixelFormat DeckLinkPixelFormatForVideoIO(VideoIOPixelFormat format);
+VideoIOPixelFormat VideoIOPixelFormatFromDeckLink(BMDPixelFormat format);

docs/openapi.yaml

@@ -358,6 +358,8 @@ components:
           $ref: "#/components/schemas/VideoStatus"
         decklink:
           $ref: "#/components/schemas/DeckLinkStatus"
+        videoIO:
+          $ref: "#/components/schemas/VideoIOStatus"
         performance:
           $ref: "#/components/schemas/PerformanceStatus"
         shaders:
@@ -415,6 +417,8 @@ components:
           type: string
     DeckLinkStatus:
       type: object
+      deprecated: true
+      description: Legacy DeckLink-specific status object. Prefer `videoIO` for new clients.
       properties:
         modelName:
           type: string
@@ -430,6 +434,26 @@ components:
           type: boolean
         statusMessage:
           type: string
+    VideoIOStatus:
+      type: object
+      properties:
+        backend:
+          type: string
+          example: decklink
+        modelName:
+          type: string
+        supportsInternalKeying:
+          type: boolean
+        supportsExternalKeying:
+          type: boolean
+        keyerInterfaceAvailable:
+          type: boolean
+        externalKeyingRequested:
+          type: boolean
+        externalKeyingActive:
+          type: boolean
+        statusMessage:
+          type: string
     PerformanceStatus:
       type: object
       properties:

runtime/README.md

@@ -19,6 +19,7 @@ Generated files:
 - `shader_cache/active_shader.frag`: patched GLSL consumed by the OpenGL path.
 - `runtime_state.json`: autosaved latest layer stack, layer order, bypass state, shader assignments, and parameter values. The host reloads this file on startup.
 - `stack_presets/*.json`: user-saved layer stack presets.
+- `screenshots/*.png`: screenshots captured from the final output render target through the control UI/API.
 
 Git policy:
 

shaders/greenscreen-key/shader.json

@@ -1,7 +1,7 @@
 {
   "id": "greenscreen-key",
   "name": "Greenscreen Key",
-  "description": "Keys out a green screen background and outputs transparent alpha for compositing.",
+  "description": "Production-style green/blue screen keyer with matte refinement, despill, edge treatment, and debug views.",
   "category": "Keying",
   "entryPoint": "shadeVideo",
   "parameters": [
@@ -16,7 +16,7 @@
     },
     {
       "id": "threshold",
-      "label": "Threshold",
+      "label": "Screen Gain",
       "type": "float",
       "default": 0.24,
       "min": 0.01,
@@ -27,20 +27,29 @@
       "id": "softness",
       "label": "Softness",
       "type": "float",
-      "default": 0.12,
+      "default": 0.16,
       "min": 0.001,
       "max": 0.5,
       "step": 0.005
     },
     {
-      "id": "edgeSoftness",
+      "id": "screenBalance",
-      "label": "Edge Softness",
+      "label": "Screen Balance",
       "type": "float",
-      "default": 0.08,
+      "default": 0.5,
       "min": 0.0,
-      "max": 0.4,
+      "max": 1.0,
       "step": 0.005
     },
+    {
+      "id": "screenPreBlur",
+      "label": "Screen PreBlur",
+      "type": "float",
+      "default": 1.0,
+      "min": 0.0,
+      "max": 8.0,
+      "step": 0.1
+    },
     {
       "id": "erodeDilate",
       "label": "Erode/Dilate",
@@ -50,6 +59,51 @@
       "max": 0.3,
       "step": 0.005
     },
+    {
+      "id": "matteBlur",
+      "label": "Matte Blur",
+      "type": "float",
+      "default": 1.25,
+      "min": 0.0,
+      "max": 6.0,
+      "step": 0.1
+    },
+    {
+      "id": "matteGamma",
+      "label": "Matte Gamma",
+      "type": "float",
+      "default": 1.0,
+      "min": 0.25,
+      "max": 4.0,
+      "step": 0.01
+    },
+    {
+      "id": "matteContrast",
+      "label": "Matte Contrast",
+      "type": "float",
+      "default": 1.0,
+      "min": 0.25,
+      "max": 4.0,
+      "step": 0.01
+    },
+    {
+      "id": "blackCleanup",
+      "label": "Black Cleanup",
+      "type": "float",
+      "default": 0.0,
+      "min": 0.0,
+      "max": 1.0,
+      "step": 0.005
+    },
+    {
+      "id": "whiteCleanup",
+      "label": "White Cleanup",
+      "type": "float",
+      "default": 0.0,
+      "min": 0.0,
+      "max": 1.0,
+      "step": 0.005
+    },
     {
       "id": "despill",
       "label": "Despill",
@@ -60,14 +114,41 @@
       "step": 0.01
     },
     {
-      "id": "edgeBoost",
+      "id": "despillBias",
-      "label": "Edge Boost",
+      "label": "Despill Bias",
       "type": "float",
-      "default": 0.08,
+      "default": 0.0,
-      "min": -0.2,
+      "min": -0.5,
-      "max": 0.3,
+      "max": 0.5,
       "step": 0.005
     },
+    {
+      "id": "spillTint",
+      "label": "Spill Tint",
+      "type": "color",
+      "default": [1.0, 1.0, 1.0, 1.0],
+      "min": [0.0, 0.0, 0.0, 0.0],
+      "max": [1.0, 1.0, 1.0, 1.0],
+      "step": [0.01, 0.01, 0.01, 0.01]
+    },
+    {
+      "id": "edgeRecover",
+      "label": "Edge Recover",
+      "type": "float",
+      "default": 0.18,
+      "min": 0.0,
+      "max": 1.0,
+      "step": 0.005
+    },
+    {
+      "id": "edgeColor",
+      "label": "Edge Color",
+      "type": "color",
+      "default": [1.0, 1.0, 1.0, 1.0],
+      "min": [0.0, 0.0, 0.0, 0.0],
+      "max": [1.0, 1.0, 1.0, 1.0],
+      "step": [0.01, 0.01, 0.01, 0.01]
+    },
     {
       "id": "clipBlack",
       "label": "Clip Black",
@@ -85,6 +166,19 @@
       "min": 0.5,
       "max": 1.0,
       "step": 0.005
+    },
+    {
+      "id": "viewMode",
+      "label": "View",
+      "type": "enum",
+      "default": "composite",
+      "options": [
+        { "value": "composite", "label": "Composite" },
+        { "value": "matte", "label": "Matte" },
+        { "value": "spill", "label": "Spill" },
+        { "value": "despill", "label": "Despill" },
+        { "value": "status", "label": "Status" }
+      ]
     }
   ]
 }

shaders/greenscreen-key/shader.slang

@@ -9,31 +9,167 @@ float luma709(float3 color)
 	return dot(color, float3(0.2126, 0.7152, 0.0722));
 }
 
+float2 chroma709(float3 color)
+{
+	float y = luma709(color);
+	return float2((color.b - y) * 0.5647, (color.r - y) * 0.7132);
+}
+
+float3 matteSampleColor(float2 uv, ShaderContext context)
+{
+	float2 pixel = 1.0 / max(context.outputResolution, float2(1.0, 1.0));
+	float blur = max(screenPreBlur, 0.0);
+	float3 center = saturate(sampleVideo(saturate(uv)).rgb);
+	if (blur <= 0.0001)
+		return center;
+
+	float2 radius = pixel * blur;
+	float3 color = center * 0.36;
+	color += saturate(sampleVideo(saturate(uv + float2(radius.x, 0.0))).rgb) * 0.16;
+	color += saturate(sampleVideo(saturate(uv - float2(radius.x, 0.0))).rgb) * 0.16;
+	color += saturate(sampleVideo(saturate(uv + float2(0.0, radius.y))).rgb) * 0.16;
+	color += saturate(sampleVideo(saturate(uv - float2(0.0, radius.y))).rgb) * 0.16;
+	return color;
+}
+
+float keyDistanceAt(float2 uv, ShaderContext context)
+{
+	float3 color = matteSampleColor(uv, context);
+	float3 keyColor = saturate(screenColor.rgb);
+	float chromaDistance = distance(chroma709(color), chroma709(keyColor)) * 2.65;
+	float directionDistance = length(safeNormalize(max(color, float3(0.0001, 0.0001, 0.0001))) - safeNormalize(max(keyColor, float3(0.0001, 0.0001, 0.0001)))) * 0.55;
+	return lerp(directionDistance, chromaDistance, saturate(screenBalance));
+}
+
+float rawAlphaAt(float2 uv, ShaderContext context)
+{
+	float keyDistance = keyDistanceAt(uv, context);
+	float matteCenter = threshold + erodeDilate;
+	float matteFeather = max(softness, 0.0005);
+	float alpha = smoothstep(matteCenter - matteFeather, matteCenter + matteFeather, keyDistance);
+	return saturate(alpha);
+}
+
+float refinedAlphaAt(float2 uv, ShaderContext context)
+{
+	float2 pixel = 1.0 / max(context.outputResolution, float2(1.0, 1.0));
+	float blur = max(matteBlur, 0.0);
+	float aaRadius = max(blur, 0.65);
+	float centerAlpha = rawAlphaAt(uv, context);
+	float alpha = centerAlpha * 0.30;
+
+	if (aaRadius > 0.0001)
+	{
+		float2 radius = pixel * aaRadius;
+		float2 halfRadius = radius * 0.5;
+		float alphaMin = centerAlpha;
+		float alphaMax = centerAlpha;
+		float sampleAlpha = rawAlphaAt(uv + float2(halfRadius.x, 0.0), context);
+		alpha += sampleAlpha * 0.065;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv - float2(halfRadius.x, 0.0), context);
+		alpha += sampleAlpha * 0.065;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv + float2(0.0, halfRadius.y), context);
+		alpha += sampleAlpha * 0.065;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv - float2(0.0, halfRadius.y), context);
+		alpha += sampleAlpha * 0.065;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv + float2(radius.x, 0.0), context);
+		alpha += sampleAlpha * 0.06;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv - float2(radius.x, 0.0), context);
+		alpha += sampleAlpha * 0.06;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv + float2(0.0, radius.y), context);
+		alpha += sampleAlpha * 0.06;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv - float2(0.0, radius.y), context);
+		alpha += sampleAlpha * 0.06;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv + radius, context);
+		alpha += sampleAlpha * 0.05;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv - radius, context);
+		alpha += sampleAlpha * 0.05;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv + float2(radius.x, -radius.y), context);
+		alpha += sampleAlpha * 0.05;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+		sampleAlpha = rawAlphaAt(uv + float2(-radius.x, radius.y), context);
+		alpha += sampleAlpha * 0.05;
+		alphaMin = min(alphaMin, sampleAlpha);
+		alphaMax = max(alphaMax, sampleAlpha);
+
+		alpha = lerp(alpha, alphaMin, saturate(blackCleanup));
+		alpha = lerp(alpha, alphaMax, saturate(whiteCleanup));
+	}
+	else
+	{
+		alpha = rawAlphaAt(uv, context);
+	}
+
+	alpha = saturate((alpha - clipBlack) / max(clipWhite - clipBlack, 0.0001));
+	alpha = saturate((alpha - 0.5) * max(matteContrast, 0.0001) + 0.5);
+	alpha = pow(max(alpha, 0.0), max(matteGamma, 0.0001));
+	return saturate(alpha);
+}
+
+float spillAmountForColor(float3 color)
+{
+	float3 keyColor = saturate(screenColor.rgb);
+	float keyComponent = dot(color, safeNormalize(max(keyColor, float3(0.0001, 0.0001, 0.0001))));
+	float opposingComponent = max(max(color.r * (1.0 - keyColor.r), color.g * (1.0 - keyColor.g)), color.b * (1.0 - keyColor.b));
+	return saturate(keyComponent - opposingComponent + despillBias);
+}
+
+float3 despillColor(float3 color, float alpha)
+{
+	float3 keyColor = safeNormalize(max(screenColor.rgb, float3(0.0001, 0.0001, 0.0001)));
+	float spill = spillAmountForColor(color) * despill * (1.0 - alpha * 0.35);
+	float neutral = luma709(color);
+	float3 neutralized = color - keyColor * spill;
+	neutralized = max(neutralized, float3(0.0, 0.0, 0.0));
+	neutralized = lerp(neutralized, float3(neutral, neutral, neutral), spill * 0.18);
+	neutralized = lerp(neutralized, neutralized * saturate(spillTint.rgb), saturate(spill));
+	return saturate(neutralized);
+}
+
 float4 shadeVideo(ShaderContext context)
 {
 	float4 src = context.sourceColor;
 	float3 color = saturate(src.rgb);
+	float alpha = refinedAlphaAt(context.uv, context);
+	float spill = spillAmountForColor(color);
+	float3 despilled = despillColor(color, alpha);
 
-	float3 keyColor = safeNormalize(max(screenColor.rgb, float3(0.0001, 0.0001, 0.0001)));
+	float edgeAmount = saturate(1.0 - abs(alpha * 2.0 - 1.0));
-	float3 sampleColor = safeNormalize(max(color, float3(0.0001, 0.0001, 0.0001)));
+	despilled = lerp(despilled, despilled * saturate(edgeColor.rgb), edgeAmount * saturate(edgeRecover));
+	alpha = saturate(lerp(alpha, rawAlphaAt(context.uv, context), edgeAmount * saturate(edgeRecover) * 0.35));
 
-	float chromaDistance = length(sampleColor - keyColor);
+	if (viewMode == 1)
-	float matteCenter = threshold - erodeDilate;
+		return float4(alpha, alpha, alpha, 1.0);
-	float matteFeather = max(softness + edgeSoftness, 0.0005);
+	if (viewMode == 2)
-	float alpha = smoothstep(matteCenter - matteFeather, matteCenter + matteFeather, chromaDistance);
+		return float4(spill, spill * 0.55, 0.0, 1.0);
-
+	if (viewMode == 3)
-	alpha = saturate((alpha - clipBlack) / max(clipWhite - clipBlack, 0.0001));
+		return float4(despilled, 1.0);
-	alpha = saturate(alpha + edgeBoost);
+	if (viewMode == 4)
-
+	{
-	float greenExcess = max(0.0, color.g - max(color.r, color.b));
+		float rawAlpha = rawAlphaAt(context.uv, context);
-	float spillReduction = greenExcess * despill;
+		return float4(rawAlpha, alpha, spill, 1.0);
-
+	}
-	float3 despilled = color;
-	despilled.g = max(0.0, despilled.g - spillReduction);
-
-	float neutral = luma709(despilled);
-	despilled.rb += spillReduction * 0.25;
-	despilled = lerp(float3(neutral, neutral, neutral), despilled, 0.92);
 
 	float3 premultiplied = saturate(despilled) * alpha;
 	return float4(premultiplied, alpha);

tests/ShaderSlangValidationTests.cpp

@@ -0,0 +1,106 @@
+#include "ShaderCompiler.h"
+#include "ShaderPackageRegistry.h"
+
+#include <filesystem>
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace
+{
+int gFailures = 0;
+
+void Fail(const std::string& message)
+{
+	std::cerr << "FAIL: " << message << "\n";
+	++gFailures;
+}
+
+std::filesystem::path FindRepoRoot()
+{
+	std::filesystem::path current = std::filesystem::current_path();
+	for (;;)
+	{
+		if (std::filesystem::exists(current / "shaders") &&
+			std::filesystem::exists(current / "runtime" / "templates" / "shader_wrapper.slang.in"))
+		{
+			return current;
+		}
+
+		if (!current.has_parent_path() || current.parent_path() == current)
+			return std::filesystem::path();
+		current = current.parent_path();
+	}
+}
+
+bool IsExpectedInvalidPackage(const ShaderPackageStatus& status)
+{
+	return status.id.find("broken-shader-example") != std::string::npos &&
+		status.error.find("Unsupported parameter type") != std::string::npos;
+}
+}
+
+int main()
+{
+	const std::filesystem::path repoRoot = FindRepoRoot();
+	if (repoRoot.empty())
+	{
+		Fail("Could not locate repository root from current working directory.");
+		return 1;
+	}
+
+	std::map<std::string, ShaderPackage> packagesById;
+	std::vector<std::string> packageOrder;
+	std::vector<ShaderPackageStatus> packageStatuses;
+	std::string error;
+
+	ShaderPackageRegistry registry(12);
+	if (!registry.Scan(repoRoot / "shaders", packagesById, packageOrder, packageStatuses, error))
+	{
+		Fail("Shader package scan failed: " + error);
+		return 1;
+	}
+
+	for (const ShaderPackageStatus& status : packageStatuses)
+	{
+		if (status.available || IsExpectedInvalidPackage(status))
+			continue;
+		Fail("Unexpected invalid shader package '" + status.id + "': " + status.error);
+	}
+
+	const std::filesystem::path validationRoot = repoRoot / "runtime" / "shader_validation";
+	const std::filesystem::path wrapperPath = validationRoot / "validation_shader_wrapper.slang";
+	const std::filesystem::path generatedGlslPath = validationRoot / "validation_shader.raw.frag";
+	const std::filesystem::path patchedGlslPath = validationRoot / "validation_shader.frag";
+
+	ShaderCompiler compiler(repoRoot, wrapperPath, generatedGlslPath, patchedGlslPath, 12);
+	for (const std::string& packageId : packageOrder)
+	{
+		auto packageIt = packagesById.find(packageId);
+		if (packageIt == packagesById.end())
+			continue;
+
+		std::string fragmentShaderSource;
+		std::string compileError;
+		if (!compiler.BuildLayerFragmentShaderSource(packageIt->second, fragmentShaderSource, compileError))
+		{
+			Fail("Shader package '" + packageId + "' failed Slang validation: " + compileError);
+			continue;
+		}
+		if (fragmentShaderSource.find("#version 430 core") == std::string::npos)
+			Fail("Shader package '" + packageId + "' generated GLSL without the expected patched GLSL version header.");
+	}
+
+	std::error_code removeError;
+	std::filesystem::remove_all(validationRoot, removeError);
+
+	if (gFailures != 0)
+	{
+		std::cerr << gFailures << " shader Slang validation failure(s).\n";
+		return 1;
+	}
+
+	std::cout << "Validated " << packagesById.size() << " shader package(s) through Slang.\n";
+	return 0;
+}

tests/VideoIODeviceFakeTests.cpp

@@ -0,0 +1,151 @@
+#include "VideoIOTypes.h"
+
+#include <array>
+#include <iostream>
+
+namespace
+{
+int gFailures = 0;
+
+void Expect(bool condition, const char* message)
+{
+	if (condition)
+		return;
+
+	std::cerr << "FAIL: " << message << "\n";
+	++gFailures;
+}
+
+class FakeVideoIODevice : public VideoIODevice
+{
+public:
+	void ReleaseResources() override {}
+
+	bool DiscoverDevicesAndModes(const VideoFormatSelection&, std::string&) override
+	{
+		mState.inputFrameSize = { 1920, 1080 };
+		mState.outputFrameSize = { 1920, 1080 };
+		mState.inputDisplayModeName = "fake 1080p";
+		mState.outputModelName = "Fake Video IO";
+		mState.hasInputDevice = true;
+		return true;
+	}
+
+	bool SelectPreferredFormats(const VideoFormatSelection&, std::string&) override
+	{
+		mState.inputPixelFormat = VideoIOPixelFormat::Uyvy8;
+		mState.outputPixelFormat = VideoIOPixelFormat::Bgra8;
+		mState.inputFrameRowBytes = VideoIORowBytes(mState.inputPixelFormat, mState.inputFrameSize.width);
+		mState.outputFrameRowBytes = VideoIORowBytes(mState.outputPixelFormat, mState.outputFrameSize.width);
+		mState.captureTextureWidth = PackedTextureWidthFromRowBytes(mState.inputFrameRowBytes);
+		mState.outputPackTextureWidth = mState.outputFrameSize.width;
+		return true;
+	}
+
+	bool ConfigureInput(InputFrameCallback callback, const VideoFormat&, std::string&) override
+	{
+		mInputCallback = callback;
+		return true;
+	}
+
+	bool ConfigureOutput(OutputFrameCallback callback, const VideoFormat&, bool, std::string&) override
+	{
+		mOutputCallback = callback;
+		return true;
+	}
+
+	bool Start() override
+	{
+		mState.hasInputSource = true;
+		VideoIOFrame input;
+		input.bytes = mInputBytes.data();
+		input.rowBytes = static_cast<long>(mState.inputFrameRowBytes);
+		input.width = mState.inputFrameSize.width;
+		input.height = mState.inputFrameSize.height;
+		input.pixelFormat = mState.inputPixelFormat;
+		if (mInputCallback)
+			mInputCallback(input);
+		if (mOutputCallback)
+			mOutputCallback(VideoIOCompletion{ VideoIOCompletionResult::Completed });
+		return true;
+	}
+
+	bool Stop() override { return true; }
+	const VideoIOState& State() const override { return mState; }
+	VideoIOState& MutableState() override { return mState; }
+
+	bool BeginOutputFrame(VideoIOOutputFrame& frame) override
+	{
+		frame.bytes = mOutputBytes.data();
+		frame.rowBytes = static_cast<long>(mState.outputFrameRowBytes);
+		frame.width = mState.outputFrameSize.width;
+		frame.height = mState.outputFrameSize.height;
+		frame.pixelFormat = mState.outputPixelFormat;
+		return true;
+	}
+
+	void EndOutputFrame(VideoIOOutputFrame&) override {}
+
+	bool ScheduleOutputFrame(const VideoIOOutputFrame&) override
+	{
+		++mScheduledFrames;
+		return true;
+	}
+
+	void AccountForCompletionResult(VideoIOCompletionResult result) override
+	{
+		mLastCompletion = result;
+	}
+
+	unsigned ScheduledFrames() const { return mScheduledFrames; }
+	VideoIOCompletionResult LastCompletion() const { return mLastCompletion; }
+
+private:
+	VideoIOState mState;
+	InputFrameCallback mInputCallback;
+	OutputFrameCallback mOutputCallback;
+	std::array<unsigned char, 3840> mInputBytes = {};
+	std::array<unsigned char, 7680> mOutputBytes = {};
+	unsigned mScheduledFrames = 0;
+	VideoIOCompletionResult mLastCompletion = VideoIOCompletionResult::Unknown;
+};
+}
+
+int main()
+{
+	FakeVideoIODevice device;
+	VideoFormatSelection selection;
+	std::string error;
+	bool inputSeen = false;
+	bool outputSeen = false;
+
+	Expect(device.DiscoverDevicesAndModes(selection, error), "fake discovery succeeds");
+	Expect(device.SelectPreferredFormats(selection, error), "fake format selection succeeds");
+	Expect(device.ConfigureInput([&](const VideoIOFrame& frame) {
+		inputSeen = frame.bytes != nullptr && frame.width == 1920 && frame.pixelFormat == VideoIOPixelFormat::Uyvy8;
+	}, selection.input, error), "fake input config succeeds");
+	Expect(device.ConfigureOutput([&](const VideoIOCompletion& completion) {
+		outputSeen = completion.result == VideoIOCompletionResult::Completed;
+	}, selection.output, false, error), "fake output config succeeds");
+	Expect(device.Start(), "fake device starts");
+
+	VideoIOOutputFrame outputFrame;
+	Expect(device.BeginOutputFrame(outputFrame), "fake output frame can be acquired");
+	device.EndOutputFrame(outputFrame);
+	device.AccountForCompletionResult(VideoIOCompletionResult::Completed);
+	Expect(device.ScheduleOutputFrame(outputFrame), "fake output frame can be scheduled");
+
+	Expect(inputSeen, "fake input callback emits generic frame");
+	Expect(outputSeen, "fake output callback emits generic completion");
+	Expect(device.ScheduledFrames() == 1, "fake backend schedules one frame");
+	Expect(device.LastCompletion() == VideoIOCompletionResult::Completed, "fake backend records generic completion");
+
+	if (gFailures != 0)
+	{
+		std::cerr << gFailures << " VideoIODevice fake test failure(s).\n";
+		return 1;
+	}
+
+	std::cout << "VideoIODevice fake tests passed.\n";
+	return 0;
+}

tests/VideoIOFormatTests.cpp

@@ -1,4 +1,5 @@
 #include "VideoIOFormat.h"
+#include "DeckLinkVideoIOFormat.h"
 
 #include <cmath>
 #include <iostream>
@@ -22,6 +23,7 @@ void TestPreferredFormatSelection()
 	Expect(ChoosePreferredVideoIOFormat(false) == VideoIOPixelFormat::Uyvy8, "8-bit is used as fallback");
 	Expect(DeckLinkPixelFormatForVideoIO(VideoIOPixelFormat::V210) == bmdFormat10BitYUV, "v210 maps to DeckLink 10-bit YUV");
 	Expect(DeckLinkPixelFormatForVideoIO(VideoIOPixelFormat::Uyvy8) == bmdFormat8BitYUV, "UYVY maps to DeckLink 8-bit YUV");
+	Expect(DeckLinkPixelFormatForVideoIO(VideoIOPixelFormat::Bgra8) == bmdFormat8BitBGRA, "BGRA maps to DeckLink 8-bit BGRA");
 }
 
 void TestRowByteHelpers()
@@ -31,6 +33,9 @@ void TestRowByteHelpers()
 	Expect(MinimumV210RowBytes(3840) == 10240, "3840-wide v210 active row bytes");
 	Expect(PackedTextureWidthFromRowBytes(5120) == 1280, "packed texture width is row bytes divided into RGBA byte texels");
 	Expect(ActiveV210WordsForWidth(1920) == 1280, "active v210 words match 1920 width");
+	Expect(VideoIORowBytes(VideoIOPixelFormat::Uyvy8, 1920) == 3840, "UYVY row bytes");
+	Expect(VideoIORowBytes(VideoIOPixelFormat::Bgra8, 1920) == 7680, "BGRA row bytes");
+	Expect(VideoIORowBytes(VideoIOPixelFormat::V210, 1920) == 5120, "v210 row bytes");
 }
 
 void TestV210PackUnpack()

tests/VideoPlayoutSchedulerTests.cpp

@@ -0,0 +1,81 @@
+#include "VideoPlayoutScheduler.h"
+
+#include <cmath>
+#include <iostream>
+
+namespace
+{
+int gFailures = 0;
+
+void Expect(bool condition, const char* message)
+{
+	if (condition)
+		return;
+
+	std::cerr << "FAIL: " << message << "\n";
+	++gFailures;
+}
+
+void ExpectNear(double actual, double expected, double tolerance, const char* message)
+{
+	Expect(std::fabs(actual - expected) <= tolerance, message);
+}
+
+void TestScheduleAdvancesFromZero()
+{
+	VideoPlayoutScheduler scheduler;
+	scheduler.Configure(1001, 60000);
+
+	const VideoIOScheduleTime first = scheduler.NextScheduleTime();
+	const VideoIOScheduleTime second = scheduler.NextScheduleTime();
+	const VideoIOScheduleTime third = scheduler.NextScheduleTime();
+
+	Expect(first.streamTime == 0, "first frame starts at stream time zero");
+	Expect(first.duration == 1001, "duration is preserved");
+	Expect(first.timeScale == 60000, "time scale is preserved");
+	Expect(second.streamTime == 1001, "second frame advances by one duration");
+	Expect(third.streamTime == 2002, "third frame advances by two durations");
+}
+
+void TestLateAndDroppedSkipAhead()
+{
+	VideoPlayoutScheduler scheduler;
+	scheduler.Configure(1000, 50000);
+
+	(void)scheduler.NextScheduleTime();
+	scheduler.AccountForCompletionResult(VideoIOCompletionResult::DisplayedLate);
+	Expect(scheduler.NextScheduleTime().streamTime == 3000, "late completion preserves the existing two-frame skip policy");
+
+	scheduler.AccountForCompletionResult(VideoIOCompletionResult::Dropped);
+	Expect(scheduler.NextScheduleTime().streamTime == 6000, "dropped completion preserves the existing two-frame skip policy");
+}
+
+void TestFrameBudgets()
+{
+	VideoPlayoutScheduler scheduler;
+	scheduler.Configure(1000, 50000);
+	ExpectNear(scheduler.FrameBudgetMilliseconds(), 20.0, 0.0001, "50 fps budget");
+
+	scheduler.Configure(1001, 60000);
+	ExpectNear(scheduler.FrameBudgetMilliseconds(), 16.6833, 0.0001, "59.94 fps budget");
+
+	scheduler.Configure(1, 60);
+	ExpectNear(scheduler.FrameBudgetMilliseconds(), 16.6667, 0.0001, "60 fps budget");
+}
+}
+
+int main()
+{
+	TestScheduleAdvancesFromZero();
+	TestLateAndDroppedSkipAhead();
+	TestFrameBudgets();
+
+	if (gFailures != 0)
+	{
+		std::cerr << gFailures << " VideoPlayoutScheduler test failure(s).\n";
+		return 1;
+	}
+
+	std::cout << "VideoPlayoutScheduler tests passed.\n";
+	return 0;
+}