Improvement

apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLRenderPipeline.cpp

@@ -25,7 +25,8 @@ OpenGLRenderPipeline::OpenGLRenderPipeline(
 	mRenderEffect(renderEffect),
 	mOutputReady(outputReady),
 	mPaint(paint),
-	mOutputReadbackMode(ReadOutputReadbackModeFromEnvironment())
+	mOutputReadbackMode(ReadOutputReadbackModeFromEnvironment()),
+	mAsyncReadbackDepth(ReadAsyncReadbackDepthFromEnvironment())
 {
 }
 
@@ -47,7 +48,9 @@ bool OpenGLRenderPipeline::RenderFrame(const RenderPipelineFrameContext& context
 	glBindFramebuffer(GL_FRAMEBUFFER, mRenderer.OutputFramebuffer());
 	if (mOutputReady)
 		mOutputReady();
-	if (state.outputPixelFormat == VideoIOPixelFormat::V210 || state.outputPixelFormat == VideoIOPixelFormat::Yuva10)
+	if (state.outputPixelFormat == VideoIOPixelFormat::Bgra8)
+		PackOutputForBgra8(state);
+	else if (state.outputPixelFormat == VideoIOPixelFormat::V210 || state.outputPixelFormat == VideoIOPixelFormat::Yuva10)
 		PackOutputFor10Bit(state);
 	glFlush();
 
@@ -76,6 +79,24 @@ bool OpenGLRenderPipeline::RenderFrame(const RenderPipelineFrameContext& context
 	return true;
 }
 
+void OpenGLRenderPipeline::PackOutputForBgra8(const VideoIOState& state)
+{
+	glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.OutputFramebuffer());
+	glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer.OutputPackFramebuffer());
+	glBlitFramebuffer(
+		0,
+		0,
+		state.outputFrameSize.width,
+		state.outputFrameSize.height,
+		0,
+		0,
+		state.outputFrameSize.width,
+		state.outputFrameSize.height,
+		GL_COLOR_BUFFER_BIT,
+		GL_NEAREST);
+	glBindFramebuffer(GL_FRAMEBUFFER, mRenderer.OutputPackFramebuffer());
+}
+
 void OpenGLRenderPipeline::PackOutputFor10Bit(const VideoIOState& state)
 {
 	glBindFramebuffer(GL_FRAMEBUFFER, mRenderer.OutputPackFramebuffer());
@@ -109,11 +130,17 @@ bool OpenGLRenderPipeline::EnsureAsyncReadbackBuffers(std::size_t requiredBytes)
 	if (requiredBytes == 0)
 		return false;
 
-	if (mAsyncReadbackBytes == requiredBytes && mAsyncReadbackSlots[0].pixelPackBuffer != 0)
+	if (mAsyncReadbackBytes == requiredBytes &&
+		mAsyncReadbackSlots.size() == mAsyncReadbackDepth &&
+		!mAsyncReadbackSlots.empty() &&
+		mAsyncReadbackSlots[0].pixelPackBuffer != 0)
+	{
 		return true;
+	}
 
 	ResetAsyncReadbackState();
 	mAsyncReadbackBytes = requiredBytes;
+	mAsyncReadbackSlots.resize(mAsyncReadbackDepth);
 	for (AsyncReadbackSlot& slot : mAsyncReadbackSlots)
 	{
 		glGenBuffers(1, &slot.pixelPackBuffer);
@@ -134,7 +161,7 @@ void OpenGLRenderPipeline::ResetAsyncReadbackState()
 	for (AsyncReadbackSlot& slot : mAsyncReadbackSlots)
 		slot.sizeBytes = 0;
 
-	if (mAsyncReadbackSlots[0].pixelPackBuffer != 0)
+	if (!mAsyncReadbackSlots.empty() && mAsyncReadbackSlots[0].pixelPackBuffer != 0)
 	{
 		for (AsyncReadbackSlot& slot : mAsyncReadbackSlots)
 		{
@@ -149,6 +176,7 @@ void OpenGLRenderPipeline::ResetAsyncReadbackState()
 	mAsyncReadbackWriteIndex = 0;
 	mAsyncReadbackReadIndex = 0;
 	mAsyncReadbackBytes = 0;
+	mAsyncReadbackSlots.clear();
 }
 
 void OpenGLRenderPipeline::FlushAsyncReadbackPipeline()
@@ -170,12 +198,14 @@ void OpenGLRenderPipeline::FlushAsyncReadbackPipeline()
 bool OpenGLRenderPipeline::QueueAsyncReadback(const VideoIOState& state, OutputReadbackTiming& timing)
 {
 	const auto queueStartTime = std::chrono::steady_clock::now();
-	const bool usePackedOutput = state.outputPixelFormat == VideoIOPixelFormat::V210 || state.outputPixelFormat == VideoIOPixelFormat::Yuva10;
+	const bool useTenBitPackedOutput = state.outputPixelFormat == VideoIOPixelFormat::V210 ||
+		state.outputPixelFormat == VideoIOPixelFormat::Yuva10;
+	const bool usePackFramebuffer = state.outputPixelFormat == VideoIOPixelFormat::Bgra8 || useTenBitPackedOutput;
 	const std::size_t requiredBytes = static_cast<std::size_t>(state.outputFrameRowBytes) * state.outputFrameSize.height;
-	const GLenum format = usePackedOutput ? GL_RGBA : GL_BGRA;
-	const GLenum type = usePackedOutput ? GL_UNSIGNED_BYTE : GL_UNSIGNED_INT_8_8_8_8_REV;
-	const GLuint framebuffer = usePackedOutput ? mRenderer.OutputPackFramebuffer() : mRenderer.OutputFramebuffer();
-	const GLsizei readWidth = static_cast<GLsizei>(usePackedOutput ? state.outputPackTextureWidth : state.outputFrameSize.width);
+	const GLenum format = useTenBitPackedOutput ? GL_RGBA : GL_BGRA;
+	const GLenum type = useTenBitPackedOutput ? GL_UNSIGNED_BYTE : GL_UNSIGNED_INT_8_8_8_8_REV;
+	const GLuint framebuffer = usePackFramebuffer ? mRenderer.OutputPackFramebuffer() : mRenderer.OutputFramebuffer();
+	const GLsizei readWidth = static_cast<GLsizei>(useTenBitPackedOutput ? state.outputPackTextureWidth : state.outputFrameSize.width);
 	const GLsizei readHeight = static_cast<GLsizei>(state.outputFrameSize.height);
 
 	const auto finishTiming = [&timing, queueStartTime]() {
@@ -204,6 +234,12 @@ bool OpenGLRenderPipeline::QueueAsyncReadback(const VideoIOState& state, OutputR
 		}
 	}
 
+	if (mAsyncReadbackSlots.empty())
+	{
+		finishTiming();
+		return false;
+	}
+
 	AsyncReadbackSlot& slot = mAsyncReadbackSlots[mAsyncReadbackWriteIndex];
 	if (slot.inFlight)
 	{
@@ -321,13 +357,17 @@ void OpenGLRenderPipeline::ReadOutputFrameSynchronously(const VideoIOState& stat
 {
 	const auto readStartTime = std::chrono::steady_clock::now();
 	const bool usePackedOutput = state.outputPixelFormat == VideoIOPixelFormat::V210 || state.outputPixelFormat == VideoIOPixelFormat::Yuva10;
+	const bool usePackFramebuffer = state.outputPixelFormat == VideoIOPixelFormat::Bgra8 || usePackedOutput;
 
 	glPixelStorei(GL_PACK_ALIGNMENT, 4);
 	glPixelStorei(GL_PACK_ROW_LENGTH, 0);
-	if (usePackedOutput)
+	if (usePackFramebuffer)
 	{
 		glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.OutputPackFramebuffer());
-		glReadPixels(0, 0, state.outputPackTextureWidth, state.outputFrameSize.height, GL_RGBA, GL_UNSIGNED_BYTE, destinationBytes);
+		if (usePackedOutput)
+			glReadPixels(0, 0, state.outputPackTextureWidth, state.outputFrameSize.height, GL_RGBA, GL_UNSIGNED_BYTE, destinationBytes);
+		else
+			glReadPixels(0, 0, state.outputFrameSize.width, state.outputFrameSize.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, destinationBytes);
 	}
 	else
 	{
@@ -366,15 +406,19 @@ OpenGLRenderPipeline::OutputReadbackTiming OpenGLRenderPipeline::ReadOutputFrame
 		return timing;
 	}
 
-	if (TryConsumeAsyncReadback(outputFrame, 500000, timing))
+	if (TryConsumeAsyncReadback(outputFrame, 0, timing))
 	{
 		(void)QueueAsyncReadback(state, timing);
 		return timing;
 	}
 
+	const bool queued = QueueAsyncReadback(state, timing);
+
+	if (queued && TryConsumeAsyncReadback(outputFrame, 0, timing))
+		return timing;
+
 	if (TryCopyCachedOutputFrame(outputFrame, timing))
 	{
-		(void)QueueAsyncReadback(state, timing);
 		return timing;
 	}
 
@@ -386,8 +430,8 @@ OpenGLRenderPipeline::OutputReadbackTiming OpenGLRenderPipeline::ReadOutputFrame
 		CacheOutputFrame(outputFrame);
 	}
 
-	FlushAsyncReadbackPipeline();
-	(void)QueueAsyncReadback(state, timing);
+	if (!queued)
+		(void)QueueAsyncReadback(state, timing);
 	return timing;
 }
 
@@ -409,3 +453,27 @@ OpenGLRenderPipeline::OutputReadbackMode OpenGLRenderPipeline::ReadOutputReadbac
 
 	return OutputReadbackMode::AsyncPbo;
 }
+
+std::size_t OpenGLRenderPipeline::ReadAsyncReadbackDepthFromEnvironment()
+{
+	char* depthValue = nullptr;
+	std::size_t depthValueSize = 0;
+	if (_dupenv_s(&depthValue, &depthValueSize, "VST_OUTPUT_READBACK_DEPTH") != 0 || depthValue == nullptr)
+		return 6;
+
+	const std::string value(depthValue);
+	std::free(depthValue);
+	try
+	{
+		const unsigned long requestedDepth = std::stoul(value);
+		if (requestedDepth < 3)
+			return 3;
+		if (requestedDepth > 12)
+			return 12;
+		return static_cast<std::size_t>(requestedDepth);
+	}
+	catch (...)
+	{
+		return 6;
+	}
+}

apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLRenderPipeline.h

@@ -3,7 +3,6 @@
 #include "GLExtensions.h"
 #include "VideoIOTypes.h"
 
-#include <array>
 #include <functional>
 #include <vector>
 
@@ -76,9 +75,11 @@ private:
 	void CacheOutputFrame(const VideoIOOutputFrame& outputFrame);
 	bool TryCopyCachedOutputFrame(VideoIOOutputFrame& outputFrame, OutputReadbackTiming& timing) const;
 	void ReadOutputFrameSynchronously(const VideoIOState& state, void* destinationBytes, OutputReadbackTiming& timing);
+	void PackOutputForBgra8(const VideoIOState& state);
 	void PackOutputFor10Bit(const VideoIOState& state);
 	OutputReadbackTiming ReadOutputFrame(const VideoIOState& state, VideoIOOutputFrame& outputFrame);
 	static OutputReadbackMode ReadOutputReadbackModeFromEnvironment();
+	static std::size_t ReadAsyncReadbackDepthFromEnvironment();
 
 	OpenGLRenderer& mRenderer;
 	RuntimeSnapshotProvider& mRuntimeSnapshotProvider;
@@ -87,7 +88,8 @@ private:
 	OutputReadyCallback mOutputReady;
 	PaintCallback mPaint;
 	OutputReadbackMode mOutputReadbackMode = OutputReadbackMode::AsyncPbo;
-	std::array<AsyncReadbackSlot, 3> mAsyncReadbackSlots;
+	std::vector<AsyncReadbackSlot> mAsyncReadbackSlots;
+	std::size_t mAsyncReadbackDepth = 0;
 	std::size_t mAsyncReadbackWriteIndex = 0;
 	std::size_t mAsyncReadbackReadIndex = 0;
 	std::size_t mAsyncReadbackBytes = 0;

apps/LoopThroughWithOpenGLCompositing/runtime/presentation/RuntimeStatePresenter.cpp

@@ -91,6 +91,16 @@ JsonValue RuntimeStatePresenter::BuildRuntimeStateValue(const RuntimeStore& runt
 	readyQueue.set("droppedCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.readyQueueDroppedCount)));
 	readyQueue.set("underrunCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.readyQueueUnderrunCount)));
 
+	JsonValue systemMemory = JsonValue::MakeObject();
+	systemMemory.set("freeFrameCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.systemFramePoolFree)));
+	systemMemory.set("readyFrameCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.systemFramePoolReady)));
+	systemMemory.set("scheduledFrameCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.systemFramePoolScheduled)));
+	systemMemory.set("underrunCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.systemFrameUnderrunCount)));
+	systemMemory.set("repeatCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.systemFrameRepeatCount)));
+	systemMemory.set("dropCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.systemFrameDropCount)));
+	systemMemory.set("ageAtScheduleMs", JsonValue(telemetrySnapshot.backendPlayout.systemFrameAgeAtScheduleMilliseconds));
+	systemMemory.set("ageAtCompletionMs", JsonValue(telemetrySnapshot.backendPlayout.systemFrameAgeAtCompletionMilliseconds));
+
 	JsonValue outputRender = JsonValue::MakeObject();
 	outputRender.set("renderMs", JsonValue(telemetrySnapshot.backendPlayout.outputRenderMilliseconds));
 	outputRender.set("smoothedRenderMs", JsonValue(telemetrySnapshot.backendPlayout.smoothedOutputRenderMilliseconds));
@@ -132,6 +142,7 @@ JsonValue RuntimeStatePresenter::BuildRuntimeStateValue(const RuntimeStore& runt
 	backendPlayout.set("droppedFrameCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.droppedFrameCount)));
 	backendPlayout.set("flushedFrameCount", JsonValue(static_cast<double>(telemetrySnapshot.backendPlayout.flushedFrameCount)));
 	backendPlayout.set("readyQueue", readyQueue);
+	backendPlayout.set("systemMemory", systemMemory);
 	backendPlayout.set("outputRender", outputRender);
 	backendPlayout.set("recovery", recovery);
 	root.set("backendPlayout", backendPlayout);

apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.cpp

@@ -313,6 +313,40 @@ bool HealthTelemetry::TryRecordOutputRenderQueueWait(double queueWaitMillisecond
 	return true;
 }
 
+void HealthTelemetry::RecordSystemMemoryPlayoutStats(std::size_t freeFrameCount, std::size_t readyFrameCount,
+	std::size_t scheduledFrameCount, uint64_t underrunCount, uint64_t repeatCount, uint64_t dropCount,
+	double frameAgeAtScheduleMilliseconds, double frameAgeAtCompletionMilliseconds)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mBackendPlayout.systemFramePoolFree = freeFrameCount;
+	mBackendPlayout.systemFramePoolReady = readyFrameCount;
+	mBackendPlayout.systemFramePoolScheduled = scheduledFrameCount;
+	mBackendPlayout.systemFrameUnderrunCount = underrunCount;
+	mBackendPlayout.systemFrameRepeatCount = repeatCount;
+	mBackendPlayout.systemFrameDropCount = dropCount;
+	mBackendPlayout.systemFrameAgeAtScheduleMilliseconds = std::max(frameAgeAtScheduleMilliseconds, 0.0);
+	mBackendPlayout.systemFrameAgeAtCompletionMilliseconds = std::max(frameAgeAtCompletionMilliseconds, 0.0);
+}
+
+bool HealthTelemetry::TryRecordSystemMemoryPlayoutStats(std::size_t freeFrameCount, std::size_t readyFrameCount,
+	std::size_t scheduledFrameCount, uint64_t underrunCount, uint64_t repeatCount, uint64_t dropCount,
+	double frameAgeAtScheduleMilliseconds, double frameAgeAtCompletionMilliseconds)
+{
+	std::unique_lock<std::mutex> lock(mMutex, std::try_to_lock);
+	if (!lock.owns_lock())
+		return false;
+
+	mBackendPlayout.systemFramePoolFree = freeFrameCount;
+	mBackendPlayout.systemFramePoolReady = readyFrameCount;
+	mBackendPlayout.systemFramePoolScheduled = scheduledFrameCount;
+	mBackendPlayout.systemFrameUnderrunCount = underrunCount;
+	mBackendPlayout.systemFrameRepeatCount = repeatCount;
+	mBackendPlayout.systemFrameDropCount = dropCount;
+	mBackendPlayout.systemFrameAgeAtScheduleMilliseconds = std::max(frameAgeAtScheduleMilliseconds, 0.0);
+	mBackendPlayout.systemFrameAgeAtCompletionMilliseconds = std::max(frameAgeAtCompletionMilliseconds, 0.0);
+	return true;
+}
+
 void HealthTelemetry::RecordOutputRenderPipelineTiming(
 	double drawMilliseconds,
 	double fenceWaitMilliseconds,

apps/LoopThroughWithOpenGLCompositing/runtime/telemetry/HealthTelemetry.h

@@ -94,6 +94,14 @@ public:
 		uint64_t readyQueuePoppedCount = 0;
 		uint64_t readyQueueDroppedCount = 0;
 		uint64_t readyQueueUnderrunCount = 0;
+		std::size_t systemFramePoolFree = 0;
+		std::size_t systemFramePoolReady = 0;
+		std::size_t systemFramePoolScheduled = 0;
+		uint64_t systemFrameUnderrunCount = 0;
+		uint64_t systemFrameRepeatCount = 0;
+		uint64_t systemFrameDropCount = 0;
+		double systemFrameAgeAtScheduleMilliseconds = 0.0;
+		double systemFrameAgeAtCompletionMilliseconds = 0.0;
 		double outputRenderMilliseconds = 0.0;
 		double smoothedOutputRenderMilliseconds = 0.0;
 		double maxOutputRenderMilliseconds = 0.0;
@@ -198,6 +206,13 @@ public:
 	void RecordOutputRenderQueueWait(double queueWaitMilliseconds);
 	bool TryRecordOutputRenderQueueWait(double queueWaitMilliseconds);
 
+	void RecordSystemMemoryPlayoutStats(std::size_t freeFrameCount, std::size_t readyFrameCount,
+		std::size_t scheduledFrameCount, uint64_t underrunCount, uint64_t repeatCount, uint64_t dropCount,
+		double frameAgeAtScheduleMilliseconds, double frameAgeAtCompletionMilliseconds);
+	bool TryRecordSystemMemoryPlayoutStats(std::size_t freeFrameCount, std::size_t readyFrameCount,
+		std::size_t scheduledFrameCount, uint64_t underrunCount, uint64_t repeatCount, uint64_t dropCount,
+		double frameAgeAtScheduleMilliseconds, double frameAgeAtCompletionMilliseconds);
+
 	void RecordOutputRenderPipelineTiming(
 		double drawMilliseconds,
 		double fenceWaitMilliseconds,

apps/LoopThroughWithOpenGLCompositing/videoio/RenderOutputQueue.cpp

@@ -11,6 +11,7 @@ void RenderOutputQueue::Configure(const VideoPlayoutPolicy& policy)
 	mPolicy = NormalizeVideoPlayoutPolicy(policy);
 	while (mReadyFrames.size() > CapacityLocked())
 	{
+		ReleaseFrame(mReadyFrames.front());
 		mReadyFrames.pop_front();
 		++mDroppedCount;
 	}
@@ -21,6 +22,7 @@ bool RenderOutputQueue::Push(RenderOutputFrame frame)
 	std::lock_guard<std::mutex> lock(mMutex);
 	if (mReadyFrames.size() >= CapacityLocked())
 	{
+		ReleaseFrame(mReadyFrames.front());
 		mReadyFrames.pop_front();
 		++mDroppedCount;
 	}
@@ -48,6 +50,8 @@ bool RenderOutputQueue::TryPop(RenderOutputFrame& frame)
 void RenderOutputQueue::Clear()
 {
 	std::lock_guard<std::mutex> lock(mMutex);
+	for (RenderOutputFrame& frame : mReadyFrames)
+		ReleaseFrame(frame);
 	mReadyFrames.clear();
 }
 
@@ -68,3 +72,10 @@ std::size_t RenderOutputQueue::CapacityLocked() const
 {
 	return static_cast<std::size_t>(mPolicy.maxReadyFrames);
 }
+
+void RenderOutputQueue::ReleaseFrame(RenderOutputFrame& frame)
+{
+	if (frame.releaseFrame)
+		frame.releaseFrame(frame.frame);
+	frame.releaseFrame = {};
+}

apps/LoopThroughWithOpenGLCompositing/videoio/RenderOutputQueue.h

@@ -5,6 +5,7 @@
 
 #include <cstdint>
 #include <deque>
+#include <functional>
 #include <mutex>
 
 struct RenderOutputFrame
@@ -12,6 +13,7 @@ struct RenderOutputFrame
 	VideoIOOutputFrame frame;
 	uint64_t frameIndex = 0;
 	bool stale = false;
+	std::function<void(VideoIOOutputFrame& frame)> releaseFrame;
 };
 
 struct RenderOutputQueueMetrics
@@ -37,6 +39,7 @@ public:
 
 private:
 	std::size_t CapacityLocked() const;
+	static void ReleaseFrame(RenderOutputFrame& frame);
 
 	mutable std::mutex mMutex;
 	VideoPlayoutPolicy mPolicy;

apps/LoopThroughWithOpenGLCompositing/videoio/SystemOutputFramePool.cpp

@@ -0,0 +1,253 @@
+#include "SystemOutputFramePool.h"
+
+#include <algorithm>
+
+namespace
+{
+SystemOutputFramePoolConfig NormalizeConfig(SystemOutputFramePoolConfig config)
+{
+	if (config.rowBytes == 0)
+		config.rowBytes = VideoIORowBytes(config.pixelFormat, config.width);
+	return config;
+}
+}
+
+SystemOutputFramePool::SystemOutputFramePool(const SystemOutputFramePoolConfig& config)
+{
+	Configure(config);
+}
+
+void SystemOutputFramePool::Configure(const SystemOutputFramePoolConfig& config)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mConfig = NormalizeConfig(config);
+	mReadySlots.clear();
+	mSlots.clear();
+	mSlots.resize(mConfig.capacity);
+
+	const std::size_t byteCount = FrameByteCount();
+	for (StoredSlot& slot : mSlots)
+	{
+		slot.bytes.resize(byteCount);
+		slot.state = OutputFrameSlotState::Free;
+		++slot.generation;
+	}
+
+	mAcquireMissCount = 0;
+	mReadyUnderrunCount = 0;
+}
+
+SystemOutputFramePoolConfig SystemOutputFramePool::Config() const
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	return mConfig;
+}
+
+bool SystemOutputFramePool::AcquireFreeSlot(OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	for (std::size_t index = 0; index < mSlots.size(); ++index)
+	{
+		if (mSlots[index].state != OutputFrameSlotState::Free)
+			continue;
+
+		mSlots[index].state = OutputFrameSlotState::Acquired;
+		++mSlots[index].generation;
+		FillOutputSlotLocked(index, slot);
+		return true;
+	}
+
+	slot = OutputFrameSlot();
+	++mAcquireMissCount;
+	return false;
+}
+
+bool SystemOutputFramePool::PublishReadySlot(const OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (!TransitionSlotLocked(slot, OutputFrameSlotState::Acquired, OutputFrameSlotState::Ready))
+		return false;
+
+	mReadySlots.push_back(slot.index);
+	return true;
+}
+
+bool SystemOutputFramePool::ConsumeReadySlot(OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	while (!mReadySlots.empty())
+	{
+		const std::size_t index = mReadySlots.front();
+		mReadySlots.pop_front();
+		if (index >= mSlots.size() || mSlots[index].state != OutputFrameSlotState::Ready)
+			continue;
+
+		mSlots[index].state = OutputFrameSlotState::Consumed;
+		FillOutputSlotLocked(index, slot);
+		return true;
+	}
+
+	slot = OutputFrameSlot();
+	++mReadyUnderrunCount;
+	return false;
+}
+
+bool SystemOutputFramePool::MarkScheduled(const OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (!IsValidSlotLocked(slot))
+		return false;
+	if (mSlots[slot.index].state != OutputFrameSlotState::Ready &&
+		mSlots[slot.index].state != OutputFrameSlotState::Consumed)
+	{
+		return false;
+	}
+
+	RemoveReadyIndexLocked(slot.index);
+	mSlots[slot.index].state = OutputFrameSlotState::Scheduled;
+	return true;
+}
+
+bool SystemOutputFramePool::MarkScheduledByBuffer(void* bytes)
+{
+	if (bytes == nullptr)
+		return false;
+
+	std::lock_guard<std::mutex> lock(mMutex);
+	for (std::size_t index = 0; index < mSlots.size(); ++index)
+	{
+		if (mSlots[index].bytes.empty() || mSlots[index].bytes.data() != bytes)
+			continue;
+		if (mSlots[index].state != OutputFrameSlotState::Ready &&
+			mSlots[index].state != OutputFrameSlotState::Consumed)
+		{
+			return false;
+		}
+
+		RemoveReadyIndexLocked(index);
+		mSlots[index].state = OutputFrameSlotState::Scheduled;
+		return true;
+	}
+	return false;
+}
+
+bool SystemOutputFramePool::ReleaseSlot(const OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (!IsValidSlotLocked(slot) || mSlots[slot.index].state == OutputFrameSlotState::Free)
+		return false;
+
+	return ReleaseSlotByIndexLocked(slot.index);
+}
+
+bool SystemOutputFramePool::ReleaseScheduledSlot(const OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	return TransitionSlotLocked(slot, OutputFrameSlotState::Scheduled, OutputFrameSlotState::Free);
+}
+
+bool SystemOutputFramePool::ReleaseSlotByBuffer(void* bytes)
+{
+	if (bytes == nullptr)
+		return false;
+
+	std::lock_guard<std::mutex> lock(mMutex);
+	for (std::size_t index = 0; index < mSlots.size(); ++index)
+	{
+		if (!mSlots[index].bytes.empty() && mSlots[index].bytes.data() == bytes)
+			return ReleaseSlotByIndexLocked(index);
+	}
+	return false;
+}
+
+void SystemOutputFramePool::Clear()
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mReadySlots.clear();
+	for (StoredSlot& slot : mSlots)
+	{
+		slot.state = OutputFrameSlotState::Free;
+		++slot.generation;
+	}
+}
+
+SystemOutputFramePoolMetrics SystemOutputFramePool::GetMetrics() const
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	SystemOutputFramePoolMetrics metrics;
+	metrics.capacity = mSlots.size();
+	metrics.readyCount = mReadySlots.size();
+	metrics.acquireMissCount = mAcquireMissCount;
+	metrics.readyUnderrunCount = mReadyUnderrunCount;
+
+	for (const StoredSlot& slot : mSlots)
+	{
+		switch (slot.state)
+		{
+		case OutputFrameSlotState::Free:
+			++metrics.freeCount;
+			break;
+		case OutputFrameSlotState::Acquired:
+			++metrics.acquiredCount;
+			break;
+		case OutputFrameSlotState::Ready:
+			break;
+		case OutputFrameSlotState::Consumed:
+			++metrics.consumedCount;
+			break;
+		case OutputFrameSlotState::Scheduled:
+			++metrics.scheduledCount;
+			break;
+		}
+	}
+
+	return metrics;
+}
+
+bool SystemOutputFramePool::IsValidSlotLocked(const OutputFrameSlot& slot) const
+{
+	return slot.index < mSlots.size() && mSlots[slot.index].generation == slot.generation;
+}
+
+bool SystemOutputFramePool::TransitionSlotLocked(const OutputFrameSlot& slot, OutputFrameSlotState expectedState, OutputFrameSlotState nextState)
+{
+	if (!IsValidSlotLocked(slot) || mSlots[slot.index].state != expectedState)
+		return false;
+
+	mSlots[slot.index].state = nextState;
+	return true;
+}
+
+void SystemOutputFramePool::FillOutputSlotLocked(std::size_t index, OutputFrameSlot& slot)
+{
+	StoredSlot& storedSlot = mSlots[index];
+	slot.index = index;
+	slot.generation = storedSlot.generation;
+	slot.frame.bytes = storedSlot.bytes.empty() ? nullptr : storedSlot.bytes.data();
+	slot.frame.rowBytes = static_cast<long>(mConfig.rowBytes);
+	slot.frame.width = mConfig.width;
+	slot.frame.height = mConfig.height;
+	slot.frame.pixelFormat = mConfig.pixelFormat;
+	slot.frame.nativeFrame = nullptr;
+	slot.frame.nativeBuffer = slot.frame.bytes;
+}
+
+void SystemOutputFramePool::RemoveReadyIndexLocked(std::size_t index)
+{
+	mReadySlots.erase(std::remove(mReadySlots.begin(), mReadySlots.end(), index), mReadySlots.end());
+}
+
+bool SystemOutputFramePool::ReleaseSlotByIndexLocked(std::size_t index)
+{
+	if (index >= mSlots.size() || mSlots[index].state == OutputFrameSlotState::Free)
+		return false;
+
+	RemoveReadyIndexLocked(index);
+	mSlots[index].state = OutputFrameSlotState::Free;
+	return true;
+}
+
+std::size_t SystemOutputFramePool::FrameByteCount() const
+{
+	return static_cast<std::size_t>(mConfig.rowBytes) * static_cast<std::size_t>(mConfig.height);
+}

apps/LoopThroughWithOpenGLCompositing/videoio/SystemOutputFramePool.h

@@ -0,0 +1,90 @@
+#pragma once
+
+#include "VideoIOTypes.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <deque>
+#include <mutex>
+#include <vector>
+
+enum class OutputFrameSlotState
+{
+	Free,
+	Acquired,
+	Ready,
+	Consumed,
+	Scheduled
+};
+
+struct SystemOutputFramePoolConfig
+{
+	unsigned width = 0;
+	unsigned height = 0;
+	VideoIOPixelFormat pixelFormat = VideoIOPixelFormat::Bgra8;
+	unsigned rowBytes = 0;
+	std::size_t capacity = 0;
+};
+
+struct OutputFrameSlot
+{
+	VideoIOOutputFrame frame;
+	std::size_t index = 0;
+	uint64_t generation = 0;
+};
+
+struct SystemOutputFramePoolMetrics
+{
+	std::size_t capacity = 0;
+	std::size_t freeCount = 0;
+	std::size_t acquiredCount = 0;
+	std::size_t readyCount = 0;
+	std::size_t consumedCount = 0;
+	std::size_t scheduledCount = 0;
+	uint64_t acquireMissCount = 0;
+	uint64_t readyUnderrunCount = 0;
+};
+
+class SystemOutputFramePool
+{
+public:
+	SystemOutputFramePool() = default;
+	explicit SystemOutputFramePool(const SystemOutputFramePoolConfig& config);
+
+	void Configure(const SystemOutputFramePoolConfig& config);
+	SystemOutputFramePoolConfig Config() const;
+
+	bool AcquireFreeSlot(OutputFrameSlot& slot);
+	bool PublishReadySlot(const OutputFrameSlot& slot);
+	bool ConsumeReadySlot(OutputFrameSlot& slot);
+	bool MarkScheduled(const OutputFrameSlot& slot);
+	bool MarkScheduledByBuffer(void* bytes);
+	bool ReleaseSlot(const OutputFrameSlot& slot);
+	bool ReleaseScheduledSlot(const OutputFrameSlot& slot);
+	bool ReleaseSlotByBuffer(void* bytes);
+	void Clear();
+
+	SystemOutputFramePoolMetrics GetMetrics() const;
+
+private:
+	struct StoredSlot
+	{
+		std::vector<unsigned char> bytes;
+		OutputFrameSlotState state = OutputFrameSlotState::Free;
+		uint64_t generation = 1;
+	};
+
+	bool IsValidSlotLocked(const OutputFrameSlot& slot) const;
+	bool TransitionSlotLocked(const OutputFrameSlot& slot, OutputFrameSlotState expectedState, OutputFrameSlotState nextState);
+	void FillOutputSlotLocked(std::size_t index, OutputFrameSlot& slot);
+	void RemoveReadyIndexLocked(std::size_t index);
+	bool ReleaseSlotByIndexLocked(std::size_t index);
+	std::size_t FrameByteCount() const;
+
+	mutable std::mutex mMutex;
+	SystemOutputFramePoolConfig mConfig;
+	std::vector<StoredSlot> mSlots;
+	std::deque<std::size_t> mReadySlots;
+	uint64_t mAcquireMissCount = 0;
+	uint64_t mReadyUnderrunCount = 0;
+};

apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp

@@ -34,6 +34,7 @@ void VideoBackend::ReleaseResources()
 	mReadyOutputQueue.Clear();
 	if (mVideoIODevice)
 		mVideoIODevice->ReleaseResources();
+	mSystemOutputFramePool.Clear();
 	if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped))
 		ApplyLifecycleFailure("Video backend resources released before lifecycle completed.");
 	ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend resources released.");
@@ -95,6 +96,14 @@ bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool exte
 		ApplyLifecycleFailure(error.empty() ? "Video backend output configuration failed." : error);
 		return false;
 	}
+	SystemOutputFramePoolConfig poolConfig;
+	poolConfig.width = mVideoIODevice->OutputFrameWidth();
+	poolConfig.height = mVideoIODevice->OutputFrameHeight();
+	poolConfig.pixelFormat = mVideoIODevice->OutputPixelFormat();
+	poolConfig.rowBytes = mVideoIODevice->OutputFrameRowBytes();
+	poolConfig.capacity = mPlayoutPolicy.outputFramePoolSize;
+	mSystemOutputFramePool.Configure(poolConfig);
+	RecordSystemMemoryPlayoutStats();
 	return ApplyLifecycleTransition(VideoBackendLifecycleState::Configured, "Video backend configured.");
 }
 
@@ -460,6 +469,8 @@ std::chrono::milliseconds VideoBackend::OutputProducerWakeInterval() const
 
 void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion)
 {
+	if (completion.outputFrameBuffer != nullptr)
+		mSystemOutputFramePool.ReleaseSlotByBuffer(completion.outputFrameBuffer);
 	RecordFramePacing(completion.result);
 	PublishOutputFrameCompleted(completion);
 	const RenderOutputQueueMetrics initialQueueMetrics = mReadyOutputQueue.GetMetrics();
@@ -483,6 +494,7 @@ void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& complet
 	}
 	NotifyOutputProducer();
 	RecordBackendPlayoutHealth(completion.result, recoveryDecision);
+	RecordSystemMemoryPlayoutStats();
 }
 
 void VideoBackend::RecordBackendPlayoutHealth(VideoIOCompletionResult result, const VideoPlayoutRecoveryDecision& recoveryDecision)
@@ -582,10 +594,12 @@ OutputProductionPressure VideoBackend::BuildOutputProductionPressure(const Rende
 bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion)
 {
 	const auto renderStart = std::chrono::steady_clock::now();
+	OutputFrameSlot outputSlot;
 	VideoIOOutputFrame outputFrame;
 	const auto acquireStart = std::chrono::steady_clock::now();
-	if (!BeginOutputFrame(outputFrame))
+	if (!mSystemOutputFramePool.AcquireFreeSlot(outputSlot))
 		return false;
+	outputFrame = outputSlot.frame;
 	const auto acquireEnd = std::chrono::steady_clock::now();
 
 	bool rendered = true;
@@ -595,7 +609,7 @@ bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const Video
 	const auto renderRequestEnd = std::chrono::steady_clock::now();
 
 	const auto endAccessStart = std::chrono::steady_clock::now();
-	EndOutputFrame(outputFrame);
+	const bool publishedReady = mSystemOutputFramePool.PublishReadySlot(outputSlot);
 	const auto endAccessEnd = std::chrono::steady_clock::now();
 	const double acquireMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(acquireEnd - acquireStart).count();
 	const double renderRequestMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(renderRequestEnd - renderRequestStart).count();
@@ -603,15 +617,17 @@ bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const Video
 
 	if (!rendered)
 	{
+		mSystemOutputFramePool.ReleaseSlot(outputSlot);
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame.");
 		const double renderMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(
 			std::chrono::steady_clock::now() - renderStart).count();
 		RecordOutputRenderDuration(renderMilliseconds, acquireMilliseconds, renderRequestMilliseconds, endAccessMilliseconds);
-		if (outputFrame.nativeFrame != nullptr)
-		{
-			static_cast<IUnknown*>(outputFrame.nativeFrame)->Release();
-			outputFrame.nativeFrame = nullptr;
-		}
+		return false;
+	}
+
+	if (!publishedReady)
+	{
+		mSystemOutputFramePool.ReleaseSlot(outputSlot);
 		return false;
 	}
 
@@ -622,12 +638,13 @@ bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const Video
 	RenderOutputFrame readyFrame;
 	readyFrame.frame = outputFrame;
 	readyFrame.frameIndex = ++mNextReadyOutputFrameIndex;
+	readyFrame.releaseFrame = [this](VideoIOOutputFrame& frame) {
+		mSystemOutputFramePool.ReleaseSlotByBuffer(frame.bytes);
+	};
 	const bool pushed = mReadyOutputQueue.Push(readyFrame);
-	if (!pushed && outputFrame.nativeFrame != nullptr)
-	{
-		static_cast<IUnknown*>(outputFrame.nativeFrame)->Release();
-		outputFrame.nativeFrame = nullptr;
-	}
+	if (!pushed)
+		mSystemOutputFramePool.ReleaseSlot(outputSlot);
+	RecordSystemMemoryPlayoutStats();
 	return pushed;
 }
 
@@ -638,10 +655,21 @@ bool VideoBackend::ScheduleReadyOutputFrame()
 		return false;
 	RecordReadyQueueDepthSample(mReadyOutputQueue.GetMetrics());
 
-	if (!ScheduleOutputFrame(readyFrame.frame))
+	if (!mSystemOutputFramePool.MarkScheduledByBuffer(readyFrame.frame.bytes))
+	{
+		if (readyFrame.releaseFrame)
+			readyFrame.releaseFrame(readyFrame.frame);
 		return false;
+	}
+
+	if (!ScheduleOutputFrame(readyFrame.frame))
+	{
+		mSystemOutputFramePool.ReleaseSlotByBuffer(readyFrame.frame.bytes);
+		return false;
+	}
 
 	PublishOutputFrameScheduled(readyFrame.frame);
+	RecordSystemMemoryPlayoutStats();
 	return true;
 }
 
@@ -721,6 +749,21 @@ void VideoBackend::RecordReadyQueueDepthSample(const RenderOutputQueueMetrics& m
 		++mReadyQueueZeroDepthCount;
 }
 
+void VideoBackend::RecordSystemMemoryPlayoutStats()
+{
+	const SystemOutputFramePoolMetrics poolMetrics = mSystemOutputFramePool.GetMetrics();
+	const RenderOutputQueueMetrics queueMetrics = mReadyOutputQueue.GetMetrics();
+	mHealthTelemetry.TryRecordSystemMemoryPlayoutStats(
+		poolMetrics.freeCount,
+		poolMetrics.readyCount,
+		poolMetrics.scheduledCount,
+		poolMetrics.readyUnderrunCount,
+		0,
+		queueMetrics.droppedCount,
+		0.0,
+		0.0);
+}
+
 void VideoBackend::RecordOutputRenderDuration(double renderMilliseconds, double acquireMilliseconds, double renderRequestMilliseconds, double endAccessMilliseconds)
 {
 	std::lock_guard<std::mutex> lock(mOutputMetricsMutex);

apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h

@@ -2,6 +2,7 @@
 
 #include "OutputProductionController.h"
 #include "RenderOutputQueue.h"
+#include "SystemOutputFramePool.h"
 #include "VideoBackendLifecycle.h"
 #include "VideoIOTypes.h"
 #include "VideoPlayoutPolicy.h"
@@ -84,6 +85,7 @@ private:
 	bool ScheduleBlackUnderrunFrame();
 	void RecordFramePacing(VideoIOCompletionResult completionResult);
 	void RecordReadyQueueDepthSample(const RenderOutputQueueMetrics& metrics);
+	void RecordSystemMemoryPlayoutStats();
 	void RecordOutputRenderDuration(double renderMilliseconds, double acquireMilliseconds, double renderRequestMilliseconds, double endAccessMilliseconds);
 	bool ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message);
 	bool ApplyLifecycleFailure(const std::string& message);
@@ -102,6 +104,7 @@ private:
 	VideoPlayoutPolicy mPlayoutPolicy;
 	OutputProductionController mOutputProductionController;
 	RenderOutputQueue mReadyOutputQueue;
+	SystemOutputFramePool mSystemOutputFramePool;
 	std::unique_ptr<VideoIODevice> mVideoIODevice;
 	std::unique_ptr<OpenGLVideoIOBridge> mBridge;
 	std::mutex mOutputCompletionMutex;

apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h

@@ -76,6 +76,7 @@ struct VideoIOOutputFrame
 struct VideoIOCompletion
 {
 	VideoIOCompletionResult result = VideoIOCompletionResult::Completed;
+	void* outputFrameBuffer = nullptr;
 };
 
 struct VideoIOScheduleTime

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp

@@ -1,6 +1,7 @@
 #include "DeckLinkSession.h"
 
 #include <atlbase.h>
+#include <atomic>
 #include <cstdio>
 #include <cstring>
 #include <new>
@@ -10,6 +11,75 @@
 
 namespace
 {
+class SystemMemoryDeckLinkVideoBuffer : public IDeckLinkVideoBuffer
+{
+public:
+	SystemMemoryDeckLinkVideoBuffer(void* bytes, unsigned long long sizeBytes) :
+		mBytes(bytes),
+		mSizeBytes(sizeBytes),
+		mRefCount(1)
+	{
+	}
+
+	HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID* ppv) override
+	{
+		if (ppv == nullptr)
+			return E_POINTER;
+		if (iid == IID_IUnknown || iid == IID_IDeckLinkVideoBuffer)
+		{
+			*ppv = static_cast<IDeckLinkVideoBuffer*>(this);
+			AddRef();
+			return S_OK;
+		}
+		*ppv = nullptr;
+		return E_NOINTERFACE;
+	}
+
+	ULONG STDMETHODCALLTYPE AddRef() override
+	{
+		return ++mRefCount;
+	}
+
+	ULONG STDMETHODCALLTYPE Release() override
+	{
+		const ULONG refCount = --mRefCount;
+		if (refCount == 0)
+			delete this;
+		return refCount;
+	}
+
+	HRESULT STDMETHODCALLTYPE GetBytes(void** buffer) override
+	{
+		if (buffer == nullptr)
+			return E_POINTER;
+		*buffer = mBytes;
+		return mBytes != nullptr ? S_OK : E_FAIL;
+	}
+
+	HRESULT STDMETHODCALLTYPE GetSize(unsigned long long* bufferSize) override
+	{
+		if (bufferSize == nullptr)
+			return E_POINTER;
+		*bufferSize = mSizeBytes;
+		return S_OK;
+	}
+
+	HRESULT STDMETHODCALLTYPE StartAccess(BMDBufferAccessFlags) override
+	{
+		return S_OK;
+	}
+
+	HRESULT STDMETHODCALLTYPE EndAccess(BMDBufferAccessFlags) override
+	{
+		return S_OK;
+	}
+
+private:
+	void* mBytes = nullptr;
+	unsigned long long mSizeBytes = 0;
+	std::atomic<ULONG> mRefCount;
+};
+
 std::string BstrToUtf8(BSTR value)
 {
 	if (value == nullptr)
@@ -460,6 +530,48 @@ bool DeckLinkSession::ScheduleFrame(IDeckLinkMutableVideoFrame* outputVideoFrame
 		output->ScheduleVideoFrame(outputVideoFrame, scheduleTime.streamTime, scheduleTime.duration, scheduleTime.timeScale) == S_OK;
 }
 
+bool DeckLinkSession::ScheduleSystemMemoryFrame(const VideoIOOutputFrame& frame)
+{
+	if (output == nullptr || frame.bytes == nullptr || frame.rowBytes <= 0 || frame.height == 0)
+		return false;
+
+	CComPtr<IDeckLinkVideoBuffer> videoBuffer;
+	videoBuffer.Attach(new (std::nothrow) SystemMemoryDeckLinkVideoBuffer(
+		frame.bytes,
+		static_cast<unsigned long long>(frame.rowBytes) * static_cast<unsigned long long>(frame.height)));
+	if (videoBuffer == nullptr)
+		return false;
+
+	CComPtr<IDeckLinkMutableVideoFrame> outputVideoFrame;
+	const BMDPixelFormat pixelFormat = DeckLinkPixelFormatForVideoIO(frame.pixelFormat);
+	if (output->CreateVideoFrameWithBuffer(
+		frame.width,
+		frame.height,
+		frame.rowBytes,
+		pixelFormat,
+		bmdFrameFlagFlipVertical,
+		videoBuffer,
+		&outputVideoFrame) != S_OK)
+	{
+		return false;
+	}
+
+	IDeckLinkVideoFrame* scheduledFrame = outputVideoFrame;
+	{
+		std::lock_guard<std::mutex> lock(mScheduledSystemFrameMutex);
+		mScheduledSystemFrameBuffers[scheduledFrame] = frame.bytes;
+	}
+
+	if (ScheduleFrame(outputVideoFrame))
+		return true;
+
+	{
+		std::lock_guard<std::mutex> lock(mScheduledSystemFrameMutex);
+		mScheduledSystemFrameBuffers.erase(scheduledFrame);
+	}
+	return false;
+}
+
 bool DeckLinkSession::ScheduleBlackFrame(IDeckLinkMutableVideoFrame* outputVideoFrame)
 {
 	if (outputVideoFrame == nullptr)
@@ -505,6 +617,9 @@ VideoPlayoutRecoveryDecision DeckLinkSession::AccountForCompletionResult(VideoIO
 
 bool DeckLinkSession::ScheduleOutputFrame(const VideoIOOutputFrame& frame)
 {
+	if (frame.nativeFrame == nullptr)
+		return ScheduleSystemMemoryFrame(frame);
+
 	IDeckLinkMutableVideoFrame* outputVideoFrame = static_cast<IDeckLinkMutableVideoFrame*>(frame.nativeFrame);
 	const bool scheduled = ScheduleFrame(outputVideoFrame);
 	if (outputVideoFrame != nullptr)
@@ -621,13 +736,29 @@ void DeckLinkSession::HandleVideoInputFrame(IDeckLinkVideoInputFrame* inputFrame
 
 void DeckLinkSession::HandlePlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult)
 {
+	void* completedSystemBuffer = nullptr;
 	if (completedFrame != nullptr)
 	{
-		CComPtr<IDeckLinkMutableVideoFrame> reusableFrame;
-		if (completedFrame->QueryInterface(IID_IDeckLinkMutableVideoFrame, reinterpret_cast<void**>(&reusableFrame)) == S_OK &&
-			reusableFrame != nullptr)
+		bool externalSystemFrame = false;
 		{
-			outputVideoFrameQueue.push_back(reusableFrame);
+			std::lock_guard<std::mutex> lock(mScheduledSystemFrameMutex);
+			auto externalFrame = mScheduledSystemFrameBuffers.find(completedFrame);
+			if (externalFrame != mScheduledSystemFrameBuffers.end())
+			{
+				completedSystemBuffer = externalFrame->second;
+				mScheduledSystemFrameBuffers.erase(externalFrame);
+				externalSystemFrame = true;
+			}
+		}
+
+		if (!externalSystemFrame)
+		{
+			CComPtr<IDeckLinkMutableVideoFrame> reusableFrame;
+			if (completedFrame->QueryInterface(IID_IDeckLinkMutableVideoFrame, reinterpret_cast<void**>(&reusableFrame)) == S_OK &&
+				reusableFrame != nullptr)
+			{
+				outputVideoFrameQueue.push_back(reusableFrame);
+			}
 		}
 	}
 
@@ -636,6 +767,7 @@ void DeckLinkSession::HandlePlayoutFrameCompleted(IDeckLinkVideoFrame* completed
 
 	VideoIOCompletion completion;
 	completion.result = TranslateCompletionResult(completionResult);
+	completion.outputFrameBuffer = completedSystemBuffer;
 	mOutputFrameCallback(completion);
 }
 

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h

@@ -11,7 +11,9 @@
 
 #include <atlbase.h>
 #include <deque>
+#include <mutex>
 #include <string>
+#include <unordered_map>
 
 class OpenGLComposite;
 
@@ -70,6 +72,7 @@ private:
 	bool AcquireNextOutputVideoFrame(CComPtr<IDeckLinkMutableVideoFrame>& outputVideoFrame);
 	bool PopulateOutputFrame(IDeckLinkMutableVideoFrame* outputVideoFrame, VideoIOOutputFrame& frame);
 	bool ScheduleFrame(IDeckLinkMutableVideoFrame* outputVideoFrame);
+	bool ScheduleSystemMemoryFrame(const VideoIOOutputFrame& frame);
 	bool ScheduleBlackFrame(IDeckLinkMutableVideoFrame* outputVideoFrame);
 	static VideoIOCompletionResult TranslateCompletionResult(BMDOutputFrameCompletionResult completionResult);
 
@@ -79,6 +82,8 @@ private:
 	CComPtr<IDeckLinkOutput>				output;
 	CComPtr<IDeckLinkKeyer>					keyer;
 	std::deque<CComPtr<IDeckLinkMutableVideoFrame>> outputVideoFrameQueue;
+	std::mutex								mScheduledSystemFrameMutex;
+	std::unordered_map<IDeckLinkVideoFrame*, void*> mScheduledSystemFrameBuffers;
 	VideoIOState							mState;
 	VideoPlayoutPolicy						mPlayoutPolicy;
 	VideoPlayoutScheduler					mScheduler;