aiden/video-shader-toys
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Wiki Activity

Stage 1 rewrite

Browse Source
This commit is contained in:
Aiden
2026-05-12 00:52:33 +10:00
parent bf23cd880a
commit ac729dc2b9
20 changed files with 1047 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
apps/LoopThroughWithOpenGLCompositing/videoio/RenderCadenceController.cpp Normal file
View File

@@ -0,0 +1,102 @@
#include "RenderCadenceController.h"
#include <algorithm>
#include <cmath>
void RenderCadenceController::Configure(Duration targetFrameDuration, TimePoint firstRenderTime, const RenderCadencePolicy& policy)
{
mTargetFrameDuration = IsPositive(targetFrameDuration) ? targetFrameDuration : std::chrono::milliseconds(1);
mPolicy = policy;
if (mPolicy.skipThresholdFrames < 1.0)
mPolicy.skipThresholdFrames = 1.0;
Reset(firstRenderTime);
}
void RenderCadenceController::Reset(TimePoint firstRenderTime)
{
mNextRenderTime = firstRenderTime;
mNextFrameIndex = 0;
mMetrics = RenderCadenceMetrics();
}
RenderCadenceDecision RenderCadenceController::Tick(TimePoint now)
{
RenderCadenceDecision decision;
decision.frameIndex = mNextFrameIndex;
decision.renderTargetTime = mNextRenderTime;
decision.nextRenderTime = mNextRenderTime;
if (now < mNextRenderTime)
{
decision.action = RenderCadenceAction::Wait;
decision.waitDuration = mNextRenderTime - now;
decision.reason = "waiting-for-next-render-tick";
return decision;
}
const Duration lateness = now - mNextRenderTime;
const uint64_t skippedTicks = SkippedTicksForLateness(lateness);
if (skippedTicks > 0)
{
decision.skippedTicks = skippedTicks;
decision.frameIndex = mNextFrameIndex + skippedTicks;
decision.renderTargetTime = mNextRenderTime + (mTargetFrameDuration * skippedTicks);
decision.reason = "late-skip-render-ticks";
mMetrics.skippedTickCount += skippedTicks;
}
else
{
decision.reason = IsPositive(lateness) ? "late-render-now" : "on-time-render";
}
decision.action = RenderCadenceAction::Render;
decision.lateness = now > decision.renderTargetTime
? now - decision.renderTargetTime
: Duration::zero();
mNextFrameIndex = decision.frameIndex + 1;
mNextRenderTime = decision.renderTargetTime + mTargetFrameDuration;
decision.nextRenderTime = mNextRenderTime;
++mMetrics.renderedFrameCount;
mMetrics.nextFrameIndex = mNextFrameIndex;
mMetrics.lastLateness = decision.lateness;
if (IsPositive(decision.lateness))
{
++mMetrics.lateFrameCount;
mMetrics.maxLateness = (std::max)(mMetrics.maxLateness, decision.lateness);
}
return decision;
}
uint64_t RenderCadenceController::SkippedTicksForLateness(Duration lateness) const
{
if (!mPolicy.skipLateTicks || !IsPositive(lateness) || !IsPositive(mTargetFrameDuration))
return 0;
const double lateFrames = static_cast<double>(lateness.count()) / static_cast<double>(mTargetFrameDuration.count());
if (lateFrames < mPolicy.skipThresholdFrames)
return 0;
const uint64_t elapsedTicks = static_cast<uint64_t>(std::floor(lateFrames));
if (elapsedTicks == 0)
return 0;
return (std::min)(elapsedTicks, mPolicy.maxSkippedTicksPerDecision);
}
bool RenderCadenceController::IsPositive(Duration duration)
{
return duration > Duration::zero();
}
const char* RenderCadenceActionName(RenderCadenceAction action)
{
switch (action)
{
case RenderCadenceAction::Render:
return "Render";
case RenderCadenceAction::Wait:
default:
return "Wait";
}
}

68
apps/LoopThroughWithOpenGLCompositing/videoio/RenderCadenceController.h Normal file
View File

@@ -0,0 +1,68 @@
#pragma once
#include <chrono>
#include <cstdint>
enum class RenderCadenceAction
{
Wait,
Render
};
struct RenderCadencePolicy
{
bool skipLateTicks = true;
uint64_t maxSkippedTicksPerDecision = 4;
double skipThresholdFrames = 2.0;
};
struct RenderCadenceDecision
{
RenderCadenceAction action = RenderCadenceAction::Wait;
uint64_t frameIndex = 0;
uint64_t skippedTicks = 0;
std::chrono::steady_clock::time_point renderTargetTime;
std::chrono::steady_clock::time_point nextRenderTime;
std::chrono::steady_clock::duration waitDuration = std::chrono::steady_clock::duration::zero();
std::chrono::steady_clock::duration lateness = std::chrono::steady_clock::duration::zero();
const char* reason = "waiting-for-next-render-tick";
};
struct RenderCadenceMetrics
{
uint64_t nextFrameIndex = 0;
uint64_t renderedFrameCount = 0;
uint64_t skippedTickCount = 0;
uint64_t lateFrameCount = 0;
std::chrono::steady_clock::duration lastLateness = std::chrono::steady_clock::duration::zero();
std::chrono::steady_clock::duration maxLateness = std::chrono::steady_clock::duration::zero();
};
class RenderCadenceController
{
public:
using Clock = std::chrono::steady_clock;
using TimePoint = Clock::time_point;
using Duration = Clock::duration;
void Configure(Duration targetFrameDuration, TimePoint firstRenderTime, const RenderCadencePolicy& policy = RenderCadencePolicy());
void Reset(TimePoint firstRenderTime);
RenderCadenceDecision Tick(TimePoint now);
Duration TargetFrameDuration() const { return mTargetFrameDuration; }
TimePoint NextRenderTime() const { return mNextRenderTime; }
uint64_t NextFrameIndex() const { return mNextFrameIndex; }
const RenderCadenceMetrics& Metrics() const { return mMetrics; }
private:
uint64_t SkippedTicksForLateness(Duration lateness) const;
static bool IsPositive(Duration duration);
Duration mTargetFrameDuration = std::chrono::milliseconds(16);
TimePoint mNextRenderTime;
uint64_t mNextFrameIndex = 0;
RenderCadencePolicy mPolicy;
RenderCadenceMetrics mMetrics;
};
const char* RenderCadenceActionName(RenderCadenceAction action);

41
apps/LoopThroughWithOpenGLCompositing/videoio/SystemOutputFramePool.cpp
View File

@@ -51,7 +51,7 @@ bool SystemOutputFramePool::AcquireFreeSlot(OutputFrameSlot& slot)
if (mSlots[index].state != OutputFrameSlotState::Free)
continue;
mSlots[index].state = OutputFrameSlotState::Acquired;
mSlots[index].state = OutputFrameSlotState::Rendering;
++mSlots[index].generation;
FillOutputSlotLocked(index, slot);
return true;
@@ -62,16 +62,26 @@ bool SystemOutputFramePool::AcquireFreeSlot(OutputFrameSlot& slot)
return false;
}
bool SystemOutputFramePool::AcquireRenderingSlot(OutputFrameSlot& slot)
{
return AcquireFreeSlot(slot);
}
bool SystemOutputFramePool::PublishReadySlot(const OutputFrameSlot& slot)
{
std::lock_guard<std::mutex> lock(mMutex);
if (!TransitionSlotLocked(slot, OutputFrameSlotState::Acquired, OutputFrameSlotState::Ready))
if (!TransitionSlotLocked(slot, OutputFrameSlotState::Rendering, OutputFrameSlotState::Completed))
return false;
mReadySlots.push_back(slot.index);
return true;
}
bool SystemOutputFramePool::PublishCompletedSlot(const OutputFrameSlot& slot)
{
return PublishReadySlot(slot);
}
bool SystemOutputFramePool::ConsumeReadySlot(OutputFrameSlot& slot)
{
std::lock_guard<std::mutex> lock(mMutex);
@@ -79,10 +89,9 @@ bool SystemOutputFramePool::ConsumeReadySlot(OutputFrameSlot& slot)
{
const std::size_t index = mReadySlots.front();
mReadySlots.pop_front();
if (index >= mSlots.size() || mSlots[index].state != OutputFrameSlotState::Ready)
if (index >= mSlots.size() || mSlots[index].state != OutputFrameSlotState::Completed)
continue;
mSlots[index].state = OutputFrameSlotState::Consumed;
FillOutputSlotLocked(index, slot);
return true;
}
@@ -92,16 +101,18 @@ bool SystemOutputFramePool::ConsumeReadySlot(OutputFrameSlot& slot)
return false;
}
bool SystemOutputFramePool::ConsumeCompletedSlot(OutputFrameSlot& slot)
{
return ConsumeReadySlot(slot);
}
bool SystemOutputFramePool::MarkScheduled(const OutputFrameSlot& slot)
{
std::lock_guard<std::mutex> lock(mMutex);
if (!IsValidSlotLocked(slot))
return false;
if (mSlots[slot.index].state != OutputFrameSlotState::Ready &&
mSlots[slot.index].state != OutputFrameSlotState::Consumed)
{
if (mSlots[slot.index].state != OutputFrameSlotState::Completed)
return false;
}
RemoveReadyIndexLocked(slot.index);
mSlots[slot.index].state = OutputFrameSlotState::Scheduled;
@@ -118,11 +129,8 @@ bool SystemOutputFramePool::MarkScheduledByBuffer(void* bytes)
{
if (mSlots[index].bytes.empty() || mSlots[index].bytes.data() != bytes)
continue;
if (mSlots[index].state != OutputFrameSlotState::Ready &&
mSlots[index].state != OutputFrameSlotState::Consumed)
{
if (mSlots[index].state != OutputFrameSlotState::Completed)
return false;
}
RemoveReadyIndexLocked(index);
mSlots[index].state = OutputFrameSlotState::Scheduled;
@@ -187,13 +195,12 @@ SystemOutputFramePoolMetrics SystemOutputFramePool::GetMetrics() const
case OutputFrameSlotState::Free:
++metrics.freeCount;
break;
case OutputFrameSlotState::Acquired:
case OutputFrameSlotState::Rendering:
++metrics.renderingCount;
++metrics.acquiredCount;
break;
case OutputFrameSlotState::Ready:
break;
case OutputFrameSlotState::Consumed:
++metrics.consumedCount;
case OutputFrameSlotState::Completed:
++metrics.completedCount;
break;
case OutputFrameSlotState::Scheduled:
++metrics.scheduledCount;

12
apps/LoopThroughWithOpenGLCompositing/videoio/SystemOutputFramePool.h
View File

@@ -11,9 +11,8 @@
enum class OutputFrameSlotState
{
Free,
Acquired,
Ready,
Consumed,
Rendering,
Completed,
Scheduled
};
@@ -37,10 +36,12 @@ struct SystemOutputFramePoolMetrics
{
std::size_t capacity = 0;
std::size_t freeCount = 0;
std::size_t renderingCount = 0;
std::size_t completedCount = 0;
std::size_t scheduledCount = 0;
std::size_t acquiredCount = 0;
std::size_t readyCount = 0;
std::size_t consumedCount = 0;
std::size_t scheduledCount = 0;
uint64_t acquireMissCount = 0;
uint64_t readyUnderrunCount = 0;
};
@@ -55,8 +56,11 @@ public:
SystemOutputFramePoolConfig Config() const;
bool AcquireFreeSlot(OutputFrameSlot& slot);
bool AcquireRenderingSlot(OutputFrameSlot& slot);
bool PublishReadySlot(const OutputFrameSlot& slot);
bool PublishCompletedSlot(const OutputFrameSlot& slot);
bool ConsumeReadySlot(OutputFrameSlot& slot);
bool ConsumeCompletedSlot(OutputFrameSlot& slot);
bool MarkScheduled(const OutputFrameSlot& slot);
bool MarkScheduledByBuffer(void* bytes);
bool ReleaseSlot(const OutputFrameSlot& slot);

20
apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.cpp
View File

@@ -513,7 +513,6 @@ void VideoBackend::ProcessOutputFrameCompletion(const VideoIOCompletion& complet
}
NotifyOutputProducer();
NotifyOutputProducer();
RecordBackendPlayoutHealth(completion.result, recoveryDecision);
RecordSystemMemoryPlayoutStats();
}
@@ -702,10 +701,12 @@ bool VideoBackend::ScheduleReadyOutputFrame()
if (!ScheduleOutputFrame(readyFrame.frame))
{
RecordDeckLinkBufferTelemetry();
mSystemOutputFramePool.ReleaseSlotByBuffer(readyFrame.frame.bytes);
return false;
}
RecordDeckLinkBufferTelemetry();
PublishOutputFrameScheduled(readyFrame.frame);
RecordSystemMemoryPlayoutStats();
return true;
@@ -726,10 +727,12 @@ bool VideoBackend::ScheduleBlackUnderrunFrame()
if (!ScheduleOutputFrame(outputFrame))
{
RecordDeckLinkBufferTelemetry();
ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: black fallback frame scheduling failed.");
return false;
}
RecordDeckLinkBufferTelemetry();
ApplyLifecycleTransition(VideoBackendLifecycleState::Degraded, "Output underrun: scheduled black fallback frame.");
PublishOutputFrameScheduled(outputFrame);
return true;
@@ -787,10 +790,25 @@ void VideoBackend::RecordReadyQueueDepthSample(const RenderOutputQueueMetrics& m
++mReadyQueueZeroDepthCount;
}
void VideoBackend::RecordDeckLinkBufferTelemetry()
{
if (!mVideoIODevice)
return;
const VideoIOState& state = mVideoIODevice->State();
mHealthTelemetry.TryRecordDeckLinkBufferTelemetry(
state.actualDeckLinkBufferedFramesAvailable,
state.actualDeckLinkBufferedFrames,
static_cast<std::size_t>(mPlayoutPolicy.targetPrerollFrames),
state.deckLinkScheduleCallMilliseconds,
state.deckLinkScheduleFailureCount);
}
void VideoBackend::RecordSystemMemoryPlayoutStats()
{
const SystemOutputFramePoolMetrics poolMetrics = mSystemOutputFramePool.GetMetrics();
const RenderOutputQueueMetrics queueMetrics = mReadyOutputQueue.GetMetrics();
RecordDeckLinkBufferTelemetry();
mHealthTelemetry.TryRecordSystemMemoryPlayoutStats(
poolMetrics.freeCount,
poolMetrics.readyCount,

1
apps/LoopThroughWithOpenGLCompositing/videoio/VideoBackend.h
View File

@@ -86,6 +86,7 @@ private:
bool ScheduleBlackUnderrunFrame();
void RecordFramePacing(VideoIOCompletionResult completionResult);
void RecordReadyQueueDepthSample(const RenderOutputQueueMetrics& metrics);
void RecordDeckLinkBufferTelemetry();
void RecordSystemMemoryPlayoutStats();
void RecordOutputRenderDuration(double renderMilliseconds, double acquireMilliseconds, double renderRequestMilliseconds, double endAccessMilliseconds);
bool ApplyLifecycleTransition(VideoBackendLifecycleState state, const std::string& message);

4
apps/LoopThroughWithOpenGLCompositing/videoio/VideoIOTypes.h
View File

@@ -50,6 +50,10 @@ struct VideoIOState
bool keyerInterfaceAvailable = false;
bool externalKeyingActive = false;
double frameBudgetMilliseconds = 0.0;
bool actualDeckLinkBufferedFramesAvailable = false;
uint64_t actualDeckLinkBufferedFrames = 0;
double deckLinkScheduleCallMilliseconds = 0.0;
uint64_t deckLinkScheduleFailureCount = 0;
};
struct VideoIOFrame

39
apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.cpp
View File

@@ -2,6 +2,7 @@
#include <atlbase.h>
#include <atomic>
#include <chrono>
#include <cstdio>
#include <cstring>
#include <new>
@@ -526,8 +527,20 @@ bool DeckLinkSession::PopulateOutputFrame(IDeckLinkMutableVideoFrame* outputVide
bool DeckLinkSession::ScheduleFrame(IDeckLinkMutableVideoFrame* outputVideoFrame)
{
const VideoIOScheduleTime scheduleTime = mScheduler.NextScheduleTime();
return outputVideoFrame != nullptr &&
output->ScheduleVideoFrame(outputVideoFrame, scheduleTime.streamTime, scheduleTime.duration, scheduleTime.timeScale) == S_OK;
if (outputVideoFrame == nullptr || output == nullptr)
{
++mState.deckLinkScheduleFailureCount;
return false;
}
const auto scheduleStart = std::chrono::steady_clock::now();
const HRESULT result = output->ScheduleVideoFrame(outputVideoFrame, scheduleTime.streamTime, scheduleTime.duration, scheduleTime.timeScale);
const auto scheduleEnd = std::chrono::steady_clock::now();
mState.deckLinkScheduleCallMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(scheduleEnd - scheduleStart).count();
if (result != S_OK)
++mState.deckLinkScheduleFailureCount;
RefreshBufferedVideoFrameCount();
return result == S_OK;
}
bool DeckLinkSession::ScheduleSystemMemoryFrame(const VideoIOOutputFrame& frame)
@@ -592,6 +605,26 @@ bool DeckLinkSession::ScheduleBlackFrame(IDeckLinkMutableVideoFrame* outputVideo
return ScheduleFrame(outputVideoFrame);
}
void DeckLinkSession::RefreshBufferedVideoFrameCount()
{
if (output == nullptr)
{
mState.actualDeckLinkBufferedFramesAvailable = false;
return;
}
unsigned int bufferedFrameCount = 0;
if (output->GetBufferedVideoFrameCount(&bufferedFrameCount) == S_OK)
{
mState.actualDeckLinkBufferedFrames = bufferedFrameCount;
mState.actualDeckLinkBufferedFramesAvailable = true;
}
else
{
mState.actualDeckLinkBufferedFramesAvailable = false;
}
}
bool DeckLinkSession::BeginOutputFrame(VideoIOOutputFrame& frame)
{
CComPtr<IDeckLinkMutableVideoFrame> outputVideoFrame;
@@ -736,6 +769,8 @@ void DeckLinkSession::HandleVideoInputFrame(IDeckLinkVideoInputFrame* inputFrame
void DeckLinkSession::HandlePlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult)
{
RefreshBufferedVideoFrameCount();
void* completedSystemBuffer = nullptr;
if (completedFrame != nullptr)
{

1
apps/LoopThroughWithOpenGLCompositing/videoio/decklink/DeckLinkSession.h
View File

@@ -74,6 +74,7 @@ private:
bool ScheduleFrame(IDeckLinkMutableVideoFrame* outputVideoFrame);
bool ScheduleSystemMemoryFrame(const VideoIOOutputFrame& frame);
bool ScheduleBlackFrame(IDeckLinkMutableVideoFrame* outputVideoFrame);
void RefreshBufferedVideoFrameCount();
static VideoIOCompletionResult TranslateCompletionResult(BMDOutputFrameCompletionResult completionResult);
CComPtr<CaptureDelegate> captureDelegate;