Improvement

apps/LoopThroughWithOpenGLCompositing/videoio/RenderOutputQueue.cpp

@@ -11,6 +11,7 @@ void RenderOutputQueue::Configure(const VideoPlayoutPolicy& policy)
 	mPolicy = NormalizeVideoPlayoutPolicy(policy);
 	while (mReadyFrames.size() > CapacityLocked())
 	{
+		ReleaseFrame(mReadyFrames.front());
 		mReadyFrames.pop_front();
 		++mDroppedCount;
 	}
@@ -21,6 +22,7 @@ bool RenderOutputQueue::Push(RenderOutputFrame frame)
 	std::lock_guard<std::mutex> lock(mMutex);
 	if (mReadyFrames.size() >= CapacityLocked())
 	{
+		ReleaseFrame(mReadyFrames.front());
 		mReadyFrames.pop_front();
 		++mDroppedCount;
 	}
@@ -48,6 +50,8 @@ bool RenderOutputQueue::TryPop(RenderOutputFrame& frame)
 void RenderOutputQueue::Clear()
 {
 	std::lock_guard<std::mutex> lock(mMutex);
+	for (RenderOutputFrame& frame : mReadyFrames)
+		ReleaseFrame(frame);
 	mReadyFrames.clear();
 }
 
@@ -68,3 +72,10 @@ std::size_t RenderOutputQueue::CapacityLocked() const
 {
 	return static_cast<std::size_t>(mPolicy.maxReadyFrames);
 }
+
+void RenderOutputQueue::ReleaseFrame(RenderOutputFrame& frame)
+{
+	if (frame.releaseFrame)
+		frame.releaseFrame(frame.frame);
+	frame.releaseFrame = {};
+}

apps/LoopThroughWithOpenGLCompositing/videoio/RenderOutputQueue.h

@@ -5,6 +5,7 @@
 
 #include <cstdint>
 #include <deque>
+#include <functional>
 #include <mutex>
 
 struct RenderOutputFrame
@@ -12,6 +13,7 @@ struct RenderOutputFrame
 	VideoIOOutputFrame frame;
 	uint64_t frameIndex = 0;
 	bool stale = false;
+	std::function<void(VideoIOOutputFrame& frame)> releaseFrame;
 };
 
 struct RenderOutputQueueMetrics
@@ -37,6 +39,7 @@ public:
 
 private:
 	std::size_t CapacityLocked() const;
+	static void ReleaseFrame(RenderOutputFrame& frame);
 
 	mutable std::mutex mMutex;
 	VideoPlayoutPolicy mPolicy;

apps/LoopThroughWithOpenGLCompositing/videoio/SystemOutputFramePool.cpp

@@ -0,0 +1,253 @@
+#include "SystemOutputFramePool.h"
+
+#include <algorithm>
+
+namespace
+{
+SystemOutputFramePoolConfig NormalizeConfig(SystemOutputFramePoolConfig config)
+{
+	if (config.rowBytes == 0)
+		config.rowBytes = VideoIORowBytes(config.pixelFormat, config.width);
+	return config;
+}
+}
+
+SystemOutputFramePool::SystemOutputFramePool(const SystemOutputFramePoolConfig& config)
+{
+	Configure(config);
+}
+
+void SystemOutputFramePool::Configure(const SystemOutputFramePoolConfig& config)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mConfig = NormalizeConfig(config);
+	mReadySlots.clear();
+	mSlots.clear();
+	mSlots.resize(mConfig.capacity);
+
+	const std::size_t byteCount = FrameByteCount();
+	for (StoredSlot& slot : mSlots)
+	{
+		slot.bytes.resize(byteCount);
+		slot.state = OutputFrameSlotState::Free;
+		++slot.generation;
+	}
+
+	mAcquireMissCount = 0;
+	mReadyUnderrunCount = 0;
+}
+
+SystemOutputFramePoolConfig SystemOutputFramePool::Config() const
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	return mConfig;
+}
+
+bool SystemOutputFramePool::AcquireFreeSlot(OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	for (std::size_t index = 0; index < mSlots.size(); ++index)
+	{
+		if (mSlots[index].state != OutputFrameSlotState::Free)
+			continue;
+
+		mSlots[index].state = OutputFrameSlotState::Acquired;
+		++mSlots[index].generation;
+		FillOutputSlotLocked(index, slot);
+		return true;
+	}
+
+	slot = OutputFrameSlot();
+	++mAcquireMissCount;
+	return false;
+}
+
+bool SystemOutputFramePool::PublishReadySlot(const OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (!TransitionSlotLocked(slot, OutputFrameSlotState::Acquired, OutputFrameSlotState::Ready))
+		return false;
+
+	mReadySlots.push_back(slot.index);
+	return true;
+}
+
+bool SystemOutputFramePool::ConsumeReadySlot(OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	while (!mReadySlots.empty())
+	{
+		const std::size_t index = mReadySlots.front();
+		mReadySlots.pop_front();
+		if (index >= mSlots.size() || mSlots[index].state != OutputFrameSlotState::Ready)
+			continue;
+
+		mSlots[index].state = OutputFrameSlotState::Consumed;
+		FillOutputSlotLocked(index, slot);
+		return true;
+	}
+
+	slot = OutputFrameSlot();
+	++mReadyUnderrunCount;
+	return false;
+}
+
+bool SystemOutputFramePool::MarkScheduled(const OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (!IsValidSlotLocked(slot))
+		return false;
+	if (mSlots[slot.index].state != OutputFrameSlotState::Ready &&
+		mSlots[slot.index].state != OutputFrameSlotState::Consumed)
+	{
+		return false;
+	}
+
+	RemoveReadyIndexLocked(slot.index);
+	mSlots[slot.index].state = OutputFrameSlotState::Scheduled;
+	return true;
+}
+
+bool SystemOutputFramePool::MarkScheduledByBuffer(void* bytes)
+{
+	if (bytes == nullptr)
+		return false;
+
+	std::lock_guard<std::mutex> lock(mMutex);
+	for (std::size_t index = 0; index < mSlots.size(); ++index)
+	{
+		if (mSlots[index].bytes.empty() || mSlots[index].bytes.data() != bytes)
+			continue;
+		if (mSlots[index].state != OutputFrameSlotState::Ready &&
+			mSlots[index].state != OutputFrameSlotState::Consumed)
+		{
+			return false;
+		}
+
+		RemoveReadyIndexLocked(index);
+		mSlots[index].state = OutputFrameSlotState::Scheduled;
+		return true;
+	}
+	return false;
+}
+
+bool SystemOutputFramePool::ReleaseSlot(const OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	if (!IsValidSlotLocked(slot) || mSlots[slot.index].state == OutputFrameSlotState::Free)
+		return false;
+
+	return ReleaseSlotByIndexLocked(slot.index);
+}
+
+bool SystemOutputFramePool::ReleaseScheduledSlot(const OutputFrameSlot& slot)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	return TransitionSlotLocked(slot, OutputFrameSlotState::Scheduled, OutputFrameSlotState::Free);
+}
+
+bool SystemOutputFramePool::ReleaseSlotByBuffer(void* bytes)
+{
+	if (bytes == nullptr)
+		return false;
+
+	std::lock_guard<std::mutex> lock(mMutex);
+	for (std::size_t index = 0; index < mSlots.size(); ++index)
+	{
+		if (!mSlots[index].bytes.empty() && mSlots[index].bytes.data() == bytes)
+			return ReleaseSlotByIndexLocked(index);
+	}
+	return false;
+}
+
+void SystemOutputFramePool::Clear()
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mReadySlots.clear();
+	for (StoredSlot& slot : mSlots)
+	{
+		slot.state = OutputFrameSlotState::Free;
+		++slot.generation;
+	}
+}
+
+SystemOutputFramePoolMetrics SystemOutputFramePool::GetMetrics() const
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	SystemOutputFramePoolMetrics metrics;
+	metrics.capacity = mSlots.size();
+	metrics.readyCount = mReadySlots.size();
+	metrics.acquireMissCount = mAcquireMissCount;
+	metrics.readyUnderrunCount = mReadyUnderrunCount;
+
+	for (const StoredSlot& slot : mSlots)
+	{
+		switch (slot.state)
+		{
+		case OutputFrameSlotState::Free:
+			++metrics.freeCount;
+			break;
+		case OutputFrameSlotState::Acquired:
+			++metrics.acquiredCount;
+			break;
+		case OutputFrameSlotState::Ready:
+			break;
+		case OutputFrameSlotState::Consumed:
+			++metrics.consumedCount;
+			break;
+		case OutputFrameSlotState::Scheduled:
+			++metrics.scheduledCount;
+			break;
+		}
+	}
+
+	return metrics;
+}
+
+bool SystemOutputFramePool::IsValidSlotLocked(const OutputFrameSlot& slot) const
+{
+	return slot.index < mSlots.size() && mSlots[slot.index].generation == slot.generation;
+}
+
+bool SystemOutputFramePool::TransitionSlotLocked(const OutputFrameSlot& slot, OutputFrameSlotState expectedState, OutputFrameSlotState nextState)
+{
+	if (!IsValidSlotLocked(slot) || mSlots[slot.index].state != expectedState)
+		return false;
+
+	mSlots[slot.index].state = nextState;
+	return true;
+}
+
+void SystemOutputFramePool::FillOutputSlotLocked(std::size_t index, OutputFrameSlot& slot)
+{
+	StoredSlot& storedSlot = mSlots[index];
+	slot.index = index;
+	slot.generation = storedSlot.generation;
+	slot.frame.bytes = storedSlot.bytes.empty() ? nullptr : storedSlot.bytes.data();
+	slot.frame.rowBytes = static_cast<long>(mConfig.rowBytes);
+	slot.frame.width = mConfig.width;
+	slot.frame.height = mConfig.height;
+	slot.frame.pixelFormat = mConfig.pixelFormat;
+	slot.frame.nativeFrame = nullptr;
+	slot.frame.nativeBuffer = slot.frame.bytes;
+}
+
+void SystemOutputFramePool::RemoveReadyIndexLocked(std::size_t index)
+{
+	mReadySlots.erase(std::remove(mReadySlots.begin(), mReadySlots.end(), index), mReadySlots.end());
+}
+
+bool SystemOutputFramePool::ReleaseSlotByIndexLocked(std::size_t index)
+{
+	if (index >= mSlots.size() || mSlots[index].state == OutputFrameSlotState::Free)
+		return false;
+
+	RemoveReadyIndexLocked(index);
+	mSlots[index].state = OutputFrameSlotState::Free;
+	return true;
+}
+
+std::size_t SystemOutputFramePool::FrameByteCount() const
+{
+	return static_cast<std::size_t>(mConfig.rowBytes) * static_cast<std::size_t>(mConfig.height);
+}

apps/LoopThroughWithOpenGLCompositing/videoio/SystemOutputFramePool.h

@@ -0,0 +1,90 @@
+#pragma once
+
+#include "VideoIOTypes.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <deque>
+#include <mutex>
+#include <vector>
+
+enum class OutputFrameSlotState
+{
+	Free,
+	Acquired,
+	Ready,
+	Consumed,
+	Scheduled
+};
+
+struct SystemOutputFramePoolConfig
+{
+	unsigned width = 0;
+	unsigned height = 0;
+	VideoIOPixelFormat pixelFormat = VideoIOPixelFormat::Bgra8;
+	unsigned rowBytes = 0;
+	std::size_t capacity = 0;
+};
+
+struct OutputFrameSlot
+{
+	VideoIOOutputFrame frame;
+	std::size_t index = 0;
+	uint64_t generation = 0;
+};
+
+struct SystemOutputFramePoolMetrics
+{
+	std::size_t capacity = 0;
+	std::size_t freeCount = 0;
+	std::size_t acquiredCount = 0;
+	std::size_t readyCount = 0;
+	std::size_t consumedCount = 0;
+	std::size_t scheduledCount = 0;
+	uint64_t acquireMissCount = 0;
+	uint64_t readyUnderrunCount = 0;
+};
+
+class SystemOutputFramePool
+{
+public:
+	SystemOutputFramePool() = default;
+	explicit SystemOutputFramePool(const SystemOutputFramePoolConfig& config);
+
+	void Configure(const SystemOutputFramePoolConfig& config);
+	SystemOutputFramePoolConfig Config() const;
+
+	bool AcquireFreeSlot(OutputFrameSlot& slot);
+	bool PublishReadySlot(const OutputFrameSlot& slot);
+	bool ConsumeReadySlot(OutputFrameSlot& slot);
+	bool MarkScheduled(const OutputFrameSlot& slot);
+	bool MarkScheduledByBuffer(void* bytes);
+	bool ReleaseSlot(const OutputFrameSlot& slot);
+	bool ReleaseScheduledSlot(const OutputFrameSlot& slot);
+	bool ReleaseSlotByBuffer(void* bytes);
+	void Clear();
+
+	SystemOutputFramePoolMetrics GetMetrics() const;
+
+private:
+	struct StoredSlot
+	{
+		std::vector<unsigned char> bytes;
+		OutputFrameSlotState state = OutputFrameSlotState::Free;
+		uint64_t generation = 1;
+	};
+
+	bool IsValidSlotLocked(const OutputFrameSlot& slot) const;
+	bool TransitionSlotLocked(const OutputFrameSlot& slot, OutputFrameSlotState expectedState, OutputFrameSlotState nextState);
+	void FillOutputSlotLocked(std::size_t index, OutputFrameSlot& slot);
+	void RemoveReadyIndexLocked(std::size_t index);
+	bool ReleaseSlotByIndexLocked(std::size_t index);
+	std::size_t FrameByteCount() const;
+
+	mutable std::mutex mMutex;
+	SystemOutputFramePoolConfig mConfig;
+	std::vector<StoredSlot> mSlots;
+	std::deque<std::size_t> mReadySlots;
+	uint64_t mAcquireMissCount = 0;
+	uint64_t mReadyUnderrunCount = 0;
+};

apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp

@@ -34,6 +34,7 @@ void VideoBackend::ReleaseResources()
 	mReadyOutputQueue.Clear();
 	if (mVideoIODevice)
 		mVideoIODevice->ReleaseResources();
+	mSystemOutputFramePool.Clear();
 	if (!VideoBackendLifecycle::CanTransition(mLifecycle.State(), VideoBackendLifecycleState::Stopped))
 		ApplyLifecycleFailure("Video backend resources released before lifecycle completed.");
 	ApplyLifecycleTransition(VideoBackendLifecycleState::Stopped, "Video backend resources released.");
@@ -95,6 +96,14 @@ bool VideoBackend::ConfigureOutput(const VideoFormat& outputVideoMode, bool exte
 		ApplyLifecycleFailure(error.empty() ? "Video backend output configuration failed." : error);
 		return false;
 	}
+	SystemOutputFramePoolConfig poolConfig;
+	poolConfig.width = mVideoIODevice->OutputFrameWidth();
+	poolConfig.height = mVideoIODevice->OutputFrameHeight();
+	poolConfig.pixelFormat = mVideoIODevice->OutputPixelFormat();
+	poolConfig.rowBytes = mVideoIODevice->OutputFrameRowBytes();
+	poolConfig.capacity = mPlayoutPolicy.outputFramePoolSize;
+	mSystemOutputFramePool.Configure(poolConfig);
+	RecordSystemMemoryPlayoutStats();
 	return ApplyLifecycleTransition(VideoBackendLifecycleState::Configured, "Video backend configured.");
 }
 
@@ -460,6 +469,8 @@ std::chrono::milliseconds VideoBackend::OutputProducerWakeInterval() const
 
 void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& completion)
 {
+	if (completion.outputFrameBuffer != nullptr)
+		mSystemOutputFramePool.ReleaseSlotByBuffer(completion.outputFrameBuffer);
 	RecordFramePacing(completion.result);
 	PublishOutputFrameCompleted(completion);
 	const RenderOutputQueueMetrics initialQueueMetrics = mReadyOutputQueue.GetMetrics();
@@ -483,6 +494,7 @@ void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& complet
 	}
 	NotifyOutputProducer();
 	RecordBackendPlayoutHealth(completion.result, recoveryDecision);
+	RecordSystemMemoryPlayoutStats();
 }
 
 void VideoBackend::RecordBackendPlayoutHealth(VideoIOCompletionResult result, const VideoPlayoutRecoveryDecision& recoveryDecision)
@@ -582,10 +594,12 @@ OutputProductionPressure VideoBackend::BuildOutputProductionPressure(const Rende
 bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const VideoIOCompletion& completion)
 {
 	const auto renderStart = std::chrono::steady_clock::now();
+	OutputFrameSlot outputSlot;
 	VideoIOOutputFrame outputFrame;
 	const auto acquireStart = std::chrono::steady_clock::now();
-	if (!BeginOutputFrame(outputFrame))
+	if (!mSystemOutputFramePool.AcquireFreeSlot(outputSlot))
 		return false;
+	outputFrame = outputSlot.frame;
 	const auto acquireEnd = std::chrono::steady_clock::now();
 
 	bool rendered = true;
@@ -595,7 +609,7 @@ bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const Video
 	const auto renderRequestEnd = std::chrono::steady_clock::now();
 
 	const auto endAccessStart = std::chrono::steady_clock::now();
-	EndOutputFrame(outputFrame);
+	const bool publishedReady = mSystemOutputFramePool.PublishReadySlot(outputSlot);
 	const auto endAccessEnd = std::chrono::steady_clock::now();
 	const double acquireMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(acquireEnd - acquireStart).count();
 	const double renderRequestMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(renderRequestEnd - renderRequestStart).count();
@@ -603,15 +617,17 @@ bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const Video
 
 	if (!rendered)
 	{
+		mSystemOutputFramePool.ReleaseSlot(outputSlot);
 		ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output frame render request failed; skipping schedule for this frame.");
 		const double renderMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(
 			std::chrono::steady_clock::now() - renderStart).count();
 		RecordOutputRenderDuration(renderMilliseconds, acquireMilliseconds, renderRequestMilliseconds, endAccessMilliseconds);
-		if (outputFrame.nativeFrame != nullptr)
-		{
-			static_cast<IUnknown*>(outputFrame.nativeFrame)->Release();
-			outputFrame.nativeFrame = nullptr;
-		}
+		return false;
+	}
+
+	if (!publishedReady)
+	{
+		mSystemOutputFramePool.ReleaseSlot(outputSlot);
 		return false;
 	}
 
@@ -622,12 +638,13 @@ bool VideoBackend::RenderReadyOutputFrame(const VideoIOState& state, const Video
 	RenderOutputFrame readyFrame;
 	readyFrame.frame = outputFrame;
 	readyFrame.frameIndex = ++mNextReadyOutputFrameIndex;
+	readyFrame.releaseFrame = [this](VideoIOOutputFrame& frame) {
+		mSystemOutputFramePool.ReleaseSlotByBuffer(frame.bytes);
+	};
 	const bool pushed = mReadyOutputQueue.Push(readyFrame);
-	if (!pushed && outputFrame.nativeFrame != nullptr)
-	{
-		static_cast<IUnknown*>(outputFrame.nativeFrame)->Release();
-		outputFrame.nativeFrame = nullptr;
-	}
+	if (!pushed)
+		mSystemOutputFramePool.ReleaseSlot(outputSlot);
+	RecordSystemMemoryPlayoutStats();
 	return pushed;
 }
 
@@ -638,10 +655,21 @@ bool VideoBackend::ScheduleReadyOutputFrame()
 		return false;
 	RecordReadyQueueDepthSample(mReadyOutputQueue.GetMetrics());
 
-	if (!ScheduleOutputFrame(readyFrame.frame))
+	if (!mSystemOutputFramePool.MarkScheduledByBuffer(readyFrame.frame.bytes))
+	{
+		if (readyFrame.releaseFrame)
+			readyFrame.releaseFrame(readyFrame.frame);
 		return false;
+	}
+
+	if (!ScheduleOutputFrame(readyFrame.frame))
+	{
+		mSystemOutputFramePool.ReleaseSlotByBuffer(readyFrame.frame.bytes);
+		return false;
+	}
 
 	PublishOutputFrameScheduled(readyFrame.frame);
+	RecordSystemMemoryPlayoutStats();
 	return true;
 }
 
@@ -721,6 +749,21 @@ void VideoBackend::RecordReadyQueueDepthSample(const RenderOutputQueueMetrics& m
 		++mReadyQueueZeroDepthCount;
 }
 
+void VideoBackend::RecordSystemMemoryPlayoutStats()
+{
+	const SystemOutputFramePoolMetrics poolMetrics = mSystemOutputFramePool.GetMetrics();
+	const RenderOutputQueueMetrics queueMetrics = mReadyOutputQueue.GetMetrics();
+	mHealthTelemetry.TryRecordSystemMemoryPlayoutStats(
+		poolMetrics.freeCount,
+		poolMetrics.readyCount,
+		poolMetrics.scheduledCount,
+		poolMetrics.readyUnderrunCount,
+		0,
+		queueMetrics.droppedCount,
+		0.0,
+		0.0);
+}
+
 void VideoBackend::RecordOutputRenderDuration(double renderMilliseconds, double acquireMilliseconds, double renderRequestMilliseconds, double endAccessMilliseconds)
 {
 	std::lock_guard<std::mutex> lock(mOutputMetricsMutex);

apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h

@@ -2,6 +2,7 @@
 
 #include "OutputProductionController.h"
 #include "RenderOutputQueue.h"
+#include "SystemOutputFramePool.h"
 #include "VideoBackendLifecycle.h"
 #include "VideoIOTypes.h"
 #include "VideoPlayoutPolicy.h"
@@ -84,6 +85,7 @@ private:
 	bool ScheduleBlackUnderrunFrame();
 	void RecordFramePacing(VideoIOCompletionResult completionResult);
 	void RecordReadyQueueDepthSample(const RenderOutputQueueMetrics& metrics);
+	void RecordSystemMemoryPlayoutStats();
 	void RecordOutputRenderDuration(double renderMilliseconds, double acquireMilliseconds, double renderRequestMilliseconds, double endAccessMilliseconds);
 	bool ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message);
 	bool ApplyLifecycleFailure(const std::string& message);
@@ -102,6 +104,7 @@ private:
 	VideoPlayoutPolicy mPlayoutPolicy;
 	OutputProductionController mOutputProductionController;
 	RenderOutputQueue mReadyOutputQueue;
+	SystemOutputFramePool mSystemOutputFramePool;
 	std::unique_ptr<VideoIODevice> mVideoIODevice;
 	std::unique_ptr<OpenGLVideoIOBridge> mBridge;
 	std::mutex mOutputCompletionMutex;

apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h

@@ -76,6 +76,7 @@ struct VideoIOOutputFrame
 struct VideoIOCompletion
 {
 	VideoIOCompletionResult result = VideoIOCompletionResult::Completed;
+	void* outputFrameBuffer = nullptr;
 };
 
 struct VideoIOScheduleTime

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp

@@ -1,6 +1,7 @@
 #include "DeckLinkSession.h"
 
 #include <atlbase.h>
+#include <atomic>
 #include <cstdio>
 #include <cstring>
 #include <new>
@@ -10,6 +11,75 @@
 
 namespace
 {
+class SystemMemoryDeckLinkVideoBuffer : public IDeckLinkVideoBuffer
+{
+public:
+	SystemMemoryDeckLinkVideoBuffer(void* bytes, unsigned long long sizeBytes) :
+		mBytes(bytes),
+		mSizeBytes(sizeBytes),
+		mRefCount(1)
+	{
+	}
+
+	HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID* ppv) override
+	{
+		if (ppv == nullptr)
+			return E_POINTER;
+		if (iid == IID_IUnknown || iid == IID_IDeckLinkVideoBuffer)
+		{
+			*ppv = static_cast<IDeckLinkVideoBuffer*>(this);
+			AddRef();
+			return S_OK;
+		}
+		*ppv = nullptr;
+		return E_NOINTERFACE;
+	}
+
+	ULONG STDMETHODCALLTYPE AddRef() override
+	{
+		return ++mRefCount;
+	}
+
+	ULONG STDMETHODCALLTYPE Release() override
+	{
+		const ULONG refCount = --mRefCount;
+		if (refCount == 0)
+			delete this;
+		return refCount;
+	}
+
+	HRESULT STDMETHODCALLTYPE GetBytes(void** buffer) override
+	{
+		if (buffer == nullptr)
+			return E_POINTER;
+		*buffer = mBytes;
+		return mBytes != nullptr ? S_OK : E_FAIL;
+	}
+
+	HRESULT STDMETHODCALLTYPE GetSize(unsigned long long* bufferSize) override
+	{
+		if (bufferSize == nullptr)
+			return E_POINTER;
+		*bufferSize = mSizeBytes;
+		return S_OK;
+	}
+
+	HRESULT STDMETHODCALLTYPE StartAccess(BMDBufferAccessFlags) override
+	{
+		return S_OK;
+	}
+
+	HRESULT STDMETHODCALLTYPE EndAccess(BMDBufferAccessFlags) override
+	{
+		return S_OK;
+	}
+
+private:
+	void* mBytes = nullptr;
+	unsigned long long mSizeBytes = 0;
+	std::atomic<ULONG> mRefCount;
+};
+
 std::string BstrToUtf8(BSTR value)
 {
 	if (value == nullptr)
@@ -460,6 +530,48 @@ bool DeckLinkSession::ScheduleFrame(IDeckLinkMutableVideoFrame* outputVideoFrame
 		output->ScheduleVideoFrame(outputVideoFrame, scheduleTime.streamTime, scheduleTime.duration, scheduleTime.timeScale) == S_OK;
 }
 
+bool DeckLinkSession::ScheduleSystemMemoryFrame(const VideoIOOutputFrame& frame)
+{
+	if (output == nullptr || frame.bytes == nullptr || frame.rowBytes <= 0 || frame.height == 0)
+		return false;
+
+	CComPtr<IDeckLinkVideoBuffer> videoBuffer;
+	videoBuffer.Attach(new (std::nothrow) SystemMemoryDeckLinkVideoBuffer(
+		frame.bytes,
+		static_cast<unsigned long long>(frame.rowBytes) * static_cast<unsigned long long>(frame.height)));
+	if (videoBuffer == nullptr)
+		return false;
+
+	CComPtr<IDeckLinkMutableVideoFrame> outputVideoFrame;
+	const BMDPixelFormat pixelFormat = DeckLinkPixelFormatForVideoIO(frame.pixelFormat);
+	if (output->CreateVideoFrameWithBuffer(
+		frame.width,
+		frame.height,
+		frame.rowBytes,
+		pixelFormat,
+		bmdFrameFlagFlipVertical,
+		videoBuffer,
+		&outputVideoFrame) != S_OK)
+	{
+		return false;
+	}
+
+	IDeckLinkVideoFrame* scheduledFrame = outputVideoFrame;
+	{
+		std::lock_guard<std::mutex> lock(mScheduledSystemFrameMutex);
+		mScheduledSystemFrameBuffers[scheduledFrame] = frame.bytes;
+	}
+
+	if (ScheduleFrame(outputVideoFrame))
+		return true;
+
+	{
+		std::lock_guard<std::mutex> lock(mScheduledSystemFrameMutex);
+		mScheduledSystemFrameBuffers.erase(scheduledFrame);
+	}
+	return false;
+}
+
 bool DeckLinkSession::ScheduleBlackFrame(IDeckLinkMutableVideoFrame* outputVideoFrame)
 {
 	if (outputVideoFrame == nullptr)
@@ -505,6 +617,9 @@ VideoPlayoutRecoveryDecision DeckLinkSession::AccountForCompletionResult(VideoIO
 
 bool DeckLinkSession::ScheduleOutputFrame(const VideoIOOutputFrame& frame)
 {
+	if (frame.nativeFrame == nullptr)
+		return ScheduleSystemMemoryFrame(frame);
+
 	IDeckLinkMutableVideoFrame* outputVideoFrame = static_cast<IDeckLinkMutableVideoFrame*>(frame.nativeFrame);
 	const bool scheduled = ScheduleFrame(outputVideoFrame);
 	if (outputVideoFrame != nullptr)
@@ -621,13 +736,29 @@ void DeckLinkSession::HandleVideoInputFrame(IDeckLinkVideoInputFrame* inputFrame
 
 void DeckLinkSession::HandlePlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult)
 {
+	void* completedSystemBuffer = nullptr;
 	if (completedFrame != nullptr)
 	{
-		CComPtr<IDeckLinkMutableVideoFrame> reusableFrame;
-		if (completedFrame->QueryInterface(IID_IDeckLinkMutableVideoFrame, reinterpret_cast<void**>(&reusableFrame)) == S_OK &&
-			reusableFrame != nullptr)
+		bool externalSystemFrame = false;
 		{
-			outputVideoFrameQueue.push_back(reusableFrame);
+			std::lock_guard<std::mutex> lock(mScheduledSystemFrameMutex);
+			auto externalFrame = mScheduledSystemFrameBuffers.find(completedFrame);
+			if (externalFrame != mScheduledSystemFrameBuffers.end())
+			{
+				completedSystemBuffer = externalFrame->second;
+				mScheduledSystemFrameBuffers.erase(externalFrame);
+				externalSystemFrame = true;
+			}
+		}
+
+		if (!externalSystemFrame)
+		{
+			CComPtr<IDeckLinkMutableVideoFrame> reusableFrame;
+			if (completedFrame->QueryInterface(IID_IDeckLinkMutableVideoFrame, reinterpret_cast<void**>(&reusableFrame)) == S_OK &&
+				reusableFrame != nullptr)
+			{
+				outputVideoFrameQueue.push_back(reusableFrame);
+			}
 		}
 	}
 
@@ -636,6 +767,7 @@ void DeckLinkSession::HandlePlayoutFrameCompleted(IDeckLinkVideoFrame* completed
 
 	VideoIOCompletion completion;
 	completion.result = TranslateCompletionResult(completionResult);
+	completion.outputFrameBuffer = completedSystemBuffer;
 	mOutputFrameCallback(completion);
 }
 

apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h

@@ -11,7 +11,9 @@
 
 #include <atlbase.h>
 #include <deque>
+#include <mutex>
 #include <string>
+#include <unordered_map>
 
 class OpenGLComposite;
 
@@ -70,6 +72,7 @@ private:
 	bool AcquireNextOutputVideoFrame(CComPtr<IDeckLinkMutableVideoFrame>& outputVideoFrame);
 	bool PopulateOutputFrame(IDeckLinkMutableVideoFrame* outputVideoFrame, VideoIOOutputFrame& frame);
 	bool ScheduleFrame(IDeckLinkMutableVideoFrame* outputVideoFrame);
+	bool ScheduleSystemMemoryFrame(const VideoIOOutputFrame& frame);
 	bool ScheduleBlackFrame(IDeckLinkMutableVideoFrame* outputVideoFrame);
 	static VideoIOCompletionResult TranslateCompletionResult(BMDOutputFrameCompletionResult completionResult);
 
@@ -79,6 +82,8 @@ private:
 	CComPtr<IDeckLinkOutput>				output;
 	CComPtr<IDeckLinkKeyer>					keyer;
 	std::deque<CComPtr<IDeckLinkMutableVideoFrame>> outputVideoFrameQueue;
+	std::mutex								mScheduledSystemFrameMutex;
+	std::unordered_map<IDeckLinkVideoFrame*, void*> mScheduledSystemFrameBuffers;
 	VideoIOState							mState;
 	VideoPlayoutPolicy						mPlayoutPolicy;
 	VideoPlayoutScheduler					mScheduler;