Clean up
This commit is contained in:
@@ -1,183 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace RenderCadenceCompositor
|
||||
{
|
||||
struct CadenceTelemetrySnapshot
|
||||
{
|
||||
double sampleSeconds = 0.0;
|
||||
double renderFps = 0.0;
|
||||
double scheduleFps = 0.0;
|
||||
std::size_t freeFrames = 0;
|
||||
std::size_t completedFrames = 0;
|
||||
std::size_t scheduledFrames = 0;
|
||||
uint64_t renderedTotal = 0;
|
||||
uint64_t scheduledTotal = 0;
|
||||
uint64_t completedPollMisses = 0;
|
||||
uint64_t scheduleFailures = 0;
|
||||
uint64_t completedDrops = 0;
|
||||
uint64_t acquireMisses = 0;
|
||||
uint64_t completions = 0;
|
||||
uint64_t displayedLate = 0;
|
||||
uint64_t dropped = 0;
|
||||
uint64_t clockOverruns = 0;
|
||||
uint64_t clockSkippedFrames = 0;
|
||||
uint64_t shaderBuildsCommitted = 0;
|
||||
uint64_t shaderBuildFailures = 0;
|
||||
double renderFrameMilliseconds = 0.0;
|
||||
double renderFrameBudgetUsedPercent = 0.0;
|
||||
double renderFrameMaxMilliseconds = 0.0;
|
||||
double readbackQueueMilliseconds = 0.0;
|
||||
double completedReadbackCopyMilliseconds = 0.0;
|
||||
uint64_t inputFramesReceived = 0;
|
||||
uint64_t inputFramesDropped = 0;
|
||||
uint64_t inputConsumeMisses = 0;
|
||||
uint64_t inputUploadMisses = 0;
|
||||
std::size_t inputReadyFrames = 0;
|
||||
std::size_t inputReadingFrames = 0;
|
||||
double inputLatestAgeMilliseconds = 0.0;
|
||||
double inputUploadMilliseconds = 0.0;
|
||||
bool inputFormatSupported = true;
|
||||
bool inputSignalPresent = false;
|
||||
double inputCaptureFps = 0.0;
|
||||
double inputConvertMilliseconds = 0.0;
|
||||
double inputSubmitMilliseconds = 0.0;
|
||||
uint64_t inputNoSignalFrames = 0;
|
||||
uint64_t inputUnsupportedFrames = 0;
|
||||
uint64_t inputSubmitMisses = 0;
|
||||
std::string inputCaptureFormat = "none";
|
||||
bool deckLinkBufferedAvailable = false;
|
||||
uint64_t deckLinkBuffered = 0;
|
||||
double deckLinkScheduleCallMilliseconds = 0.0;
|
||||
bool deckLinkScheduleLeadAvailable = false;
|
||||
int64_t deckLinkPlaybackStreamTime = 0;
|
||||
uint64_t deckLinkPlaybackFrameIndex = 0;
|
||||
uint64_t deckLinkNextScheduleFrameIndex = 0;
|
||||
int64_t deckLinkScheduleLeadFrames = 0;
|
||||
uint64_t deckLinkScheduleRealignments = 0;
|
||||
};
|
||||
|
||||
class CadenceTelemetry
|
||||
{
|
||||
public:
|
||||
template <typename SystemFrameExchange, typename Output, typename OutputThread>
|
||||
CadenceTelemetrySnapshot Sample(
|
||||
const SystemFrameExchange& exchange,
|
||||
const Output& output,
|
||||
const OutputThread& outputThread)
|
||||
{
|
||||
const auto now = Clock::now();
|
||||
const double seconds = mHasLastSample
|
||||
? std::chrono::duration_cast<std::chrono::duration<double>>(now - mLastSampleTime).count()
|
||||
: 0.0;
|
||||
|
||||
const auto exchangeMetrics = exchange.Metrics();
|
||||
const auto outputMetrics = output.Metrics();
|
||||
const auto threadMetrics = outputThread.Metrics();
|
||||
|
||||
CadenceTelemetrySnapshot snapshot;
|
||||
snapshot.sampleSeconds = seconds;
|
||||
snapshot.renderedTotal = exchangeMetrics.completedFrames;
|
||||
snapshot.scheduledTotal = exchangeMetrics.scheduledFrames;
|
||||
snapshot.freeFrames = exchangeMetrics.freeCount;
|
||||
snapshot.completedFrames = exchangeMetrics.completedCount;
|
||||
snapshot.scheduledFrames = exchangeMetrics.scheduledCount;
|
||||
snapshot.completedPollMisses = threadMetrics.completedPollMisses;
|
||||
snapshot.scheduleFailures = outputMetrics.scheduleFailures > threadMetrics.scheduleFailures
|
||||
? outputMetrics.scheduleFailures
|
||||
: threadMetrics.scheduleFailures;
|
||||
snapshot.completedDrops = exchangeMetrics.completedDrops;
|
||||
snapshot.acquireMisses = exchangeMetrics.acquireMisses;
|
||||
snapshot.completions = outputMetrics.completions;
|
||||
snapshot.displayedLate = outputMetrics.displayedLate;
|
||||
snapshot.dropped = outputMetrics.dropped;
|
||||
snapshot.deckLinkBufferedAvailable = outputMetrics.actualBufferedFramesAvailable;
|
||||
snapshot.deckLinkBuffered = outputMetrics.actualBufferedFrames;
|
||||
snapshot.deckLinkScheduleCallMilliseconds = outputMetrics.scheduleCallMilliseconds;
|
||||
snapshot.deckLinkScheduleLeadAvailable = outputMetrics.scheduleLeadAvailable;
|
||||
snapshot.deckLinkPlaybackStreamTime = outputMetrics.playbackStreamTime;
|
||||
snapshot.deckLinkPlaybackFrameIndex = outputMetrics.playbackFrameIndex;
|
||||
snapshot.deckLinkNextScheduleFrameIndex = outputMetrics.nextScheduleFrameIndex;
|
||||
snapshot.deckLinkScheduleLeadFrames = outputMetrics.scheduleLeadFrames;
|
||||
snapshot.deckLinkScheduleRealignments = outputMetrics.scheduleRealignmentCount;
|
||||
|
||||
if (mHasLastSample && seconds > 0.0)
|
||||
{
|
||||
snapshot.renderFps = static_cast<double>(snapshot.renderedTotal - mLastRenderedFrames) / seconds;
|
||||
snapshot.scheduleFps = static_cast<double>(snapshot.scheduledTotal - mLastScheduledFrames) / seconds;
|
||||
}
|
||||
|
||||
mLastSampleTime = now;
|
||||
mLastRenderedFrames = snapshot.renderedTotal;
|
||||
mLastScheduledFrames = snapshot.scheduledTotal;
|
||||
mHasLastSample = true;
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
template <typename SystemFrameExchange, typename Output, typename OutputThread, typename RenderThread>
|
||||
CadenceTelemetrySnapshot Sample(
|
||||
const SystemFrameExchange& exchange,
|
||||
const Output& output,
|
||||
const OutputThread& outputThread,
|
||||
const RenderThread& renderThread)
|
||||
{
|
||||
CadenceTelemetrySnapshot snapshot = Sample(exchange, output, outputThread);
|
||||
const auto renderMetrics = renderThread.GetMetrics();
|
||||
snapshot.clockOverruns = renderMetrics.clockOverruns;
|
||||
snapshot.clockSkippedFrames = renderMetrics.skippedFrames;
|
||||
snapshot.shaderBuildsCommitted = renderMetrics.shaderBuildsCommitted;
|
||||
snapshot.shaderBuildFailures = renderMetrics.shaderBuildFailures;
|
||||
snapshot.renderFrameMilliseconds = renderMetrics.renderFrameMilliseconds;
|
||||
snapshot.renderFrameBudgetUsedPercent = renderMetrics.renderFrameBudgetUsedPercent;
|
||||
snapshot.renderFrameMaxMilliseconds = renderMetrics.renderFrameMaxMilliseconds;
|
||||
snapshot.readbackQueueMilliseconds = renderMetrics.readbackQueueMilliseconds;
|
||||
snapshot.completedReadbackCopyMilliseconds = renderMetrics.completedReadbackCopyMilliseconds;
|
||||
snapshot.inputFramesReceived = renderMetrics.inputFramesReceived;
|
||||
snapshot.inputFramesDropped = renderMetrics.inputFramesDropped;
|
||||
snapshot.inputConsumeMisses = renderMetrics.inputConsumeMisses;
|
||||
snapshot.inputUploadMisses = renderMetrics.inputUploadMisses;
|
||||
snapshot.inputReadyFrames = renderMetrics.inputReadyFrames;
|
||||
snapshot.inputReadingFrames = renderMetrics.inputReadingFrames;
|
||||
snapshot.inputLatestAgeMilliseconds = renderMetrics.inputLatestAgeMilliseconds;
|
||||
snapshot.inputUploadMilliseconds = renderMetrics.inputUploadMilliseconds;
|
||||
snapshot.inputFormatSupported = renderMetrics.inputFormatSupported;
|
||||
snapshot.inputSignalPresent = renderMetrics.inputSignalPresent;
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
template <typename SystemFrameExchange, typename Output, typename OutputThread, typename RenderThread, typename InputEdge>
|
||||
CadenceTelemetrySnapshot Sample(
|
||||
const SystemFrameExchange& exchange,
|
||||
const Output& output,
|
||||
const OutputThread& outputThread,
|
||||
const RenderThread& renderThread,
|
||||
const InputEdge& inputEdge)
|
||||
{
|
||||
CadenceTelemetrySnapshot snapshot = Sample(exchange, output, outputThread, renderThread);
|
||||
const auto inputMetrics = inputEdge.Metrics();
|
||||
snapshot.inputConvertMilliseconds = inputMetrics.convertMilliseconds;
|
||||
snapshot.inputSubmitMilliseconds = inputMetrics.submitMilliseconds;
|
||||
snapshot.inputNoSignalFrames = inputMetrics.noInputSourceFrames;
|
||||
snapshot.inputUnsupportedFrames = inputMetrics.unsupportedFrames;
|
||||
snapshot.inputSubmitMisses = inputMetrics.submitMisses;
|
||||
snapshot.inputCaptureFormat = inputMetrics.captureFormat ? inputMetrics.captureFormat : "none";
|
||||
if (snapshot.sampleSeconds > 0.0)
|
||||
snapshot.inputCaptureFps = static_cast<double>(inputMetrics.capturedFrames - mLastInputCapturedFrames) / snapshot.sampleSeconds;
|
||||
mLastInputCapturedFrames = inputMetrics.capturedFrames;
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
private:
|
||||
using Clock = std::chrono::steady_clock;
|
||||
|
||||
Clock::time_point mLastSampleTime = Clock::now();
|
||||
uint64_t mLastRenderedFrames = 0;
|
||||
uint64_t mLastScheduledFrames = 0;
|
||||
uint64_t mLastInputCapturedFrames = 0;
|
||||
bool mHasLastSample = false;
|
||||
};
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "CadenceTelemetry.h"
|
||||
#include "../json/JsonWriter.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace RenderCadenceCompositor
|
||||
{
|
||||
inline void WriteCadenceTelemetryJson(JsonWriter& writer, const CadenceTelemetrySnapshot& snapshot)
|
||||
{
|
||||
writer.BeginObject();
|
||||
writer.KeyDouble("sampleSeconds", snapshot.sampleSeconds);
|
||||
writer.KeyDouble("renderFps", snapshot.renderFps);
|
||||
writer.KeyDouble("scheduleFps", snapshot.scheduleFps);
|
||||
writer.KeyUInt("free", static_cast<uint64_t>(snapshot.freeFrames));
|
||||
writer.KeyUInt("completed", static_cast<uint64_t>(snapshot.completedFrames));
|
||||
writer.KeyUInt("scheduled", static_cast<uint64_t>(snapshot.scheduledFrames));
|
||||
writer.KeyUInt("renderedTotal", snapshot.renderedTotal);
|
||||
writer.KeyUInt("scheduledTotal", snapshot.scheduledTotal);
|
||||
writer.KeyUInt("completedPollMisses", snapshot.completedPollMisses);
|
||||
writer.KeyUInt("scheduleFailures", snapshot.scheduleFailures);
|
||||
writer.KeyUInt("completedDrops", snapshot.completedDrops);
|
||||
writer.KeyUInt("acquireMisses", snapshot.acquireMisses);
|
||||
writer.KeyUInt("completions", snapshot.completions);
|
||||
writer.KeyUInt("late", snapshot.displayedLate);
|
||||
writer.KeyUInt("dropped", snapshot.dropped);
|
||||
writer.KeyUInt("clockOverruns", snapshot.clockOverruns);
|
||||
writer.KeyUInt("clockSkippedFrames", snapshot.clockSkippedFrames);
|
||||
writer.KeyUInt("clockOveruns", snapshot.clockOverruns);
|
||||
writer.KeyUInt("clockSkipped", snapshot.clockSkippedFrames);
|
||||
writer.KeyUInt("shaderCommitted", snapshot.shaderBuildsCommitted);
|
||||
writer.KeyUInt("shaderFailures", snapshot.shaderBuildFailures);
|
||||
writer.KeyDouble("renderFrameMs", snapshot.renderFrameMilliseconds);
|
||||
writer.KeyDouble("renderFrameBudgetUsedPercent", snapshot.renderFrameBudgetUsedPercent);
|
||||
writer.KeyDouble("renderFrameMaxMs", snapshot.renderFrameMaxMilliseconds);
|
||||
writer.KeyDouble("readbackQueueMs", snapshot.readbackQueueMilliseconds);
|
||||
writer.KeyDouble("completedReadbackCopyMs", snapshot.completedReadbackCopyMilliseconds);
|
||||
writer.KeyUInt("inputFramesReceived", snapshot.inputFramesReceived);
|
||||
writer.KeyUInt("inputFramesDropped", snapshot.inputFramesDropped);
|
||||
writer.KeyUInt("inputConsumeMisses", snapshot.inputConsumeMisses);
|
||||
writer.KeyUInt("inputUploadMisses", snapshot.inputUploadMisses);
|
||||
writer.KeyUInt("inputReadyFrames", static_cast<uint64_t>(snapshot.inputReadyFrames));
|
||||
writer.KeyUInt("inputReadingFrames", static_cast<uint64_t>(snapshot.inputReadingFrames));
|
||||
writer.KeyDouble("inputLatestAgeMs", snapshot.inputLatestAgeMilliseconds);
|
||||
writer.KeyDouble("inputUploadMs", snapshot.inputUploadMilliseconds);
|
||||
writer.KeyBool("inputFormatSupported", snapshot.inputFormatSupported);
|
||||
writer.KeyBool("inputSignalPresent", snapshot.inputSignalPresent);
|
||||
writer.KeyDouble("inputCaptureFps", snapshot.inputCaptureFps);
|
||||
writer.KeyDouble("inputConvertMs", snapshot.inputConvertMilliseconds);
|
||||
writer.KeyDouble("inputSubmitMs", snapshot.inputSubmitMilliseconds);
|
||||
writer.KeyUInt("inputNoSignalFrames", snapshot.inputNoSignalFrames);
|
||||
writer.KeyUInt("inputUnsupportedFrames", snapshot.inputUnsupportedFrames);
|
||||
writer.KeyUInt("inputSubmitMisses", snapshot.inputSubmitMisses);
|
||||
writer.KeyString("inputCaptureFormat", snapshot.inputCaptureFormat);
|
||||
writer.KeyBool("deckLinkBufferedAvailable", snapshot.deckLinkBufferedAvailable);
|
||||
writer.Key("deckLinkBuffered");
|
||||
if (snapshot.deckLinkBufferedAvailable)
|
||||
writer.UInt(snapshot.deckLinkBuffered);
|
||||
else
|
||||
writer.Null();
|
||||
writer.KeyDouble("scheduleCallMs", snapshot.deckLinkScheduleCallMilliseconds);
|
||||
writer.KeyBool("deckLinkScheduleLeadAvailable", snapshot.deckLinkScheduleLeadAvailable);
|
||||
writer.Key("deckLinkScheduleLeadFrames");
|
||||
if (snapshot.deckLinkScheduleLeadAvailable)
|
||||
writer.Int(snapshot.deckLinkScheduleLeadFrames);
|
||||
else
|
||||
writer.Null();
|
||||
writer.KeyUInt("deckLinkPlaybackFrameIndex", snapshot.deckLinkPlaybackFrameIndex);
|
||||
writer.KeyUInt("deckLinkNextScheduleFrameIndex", snapshot.deckLinkNextScheduleFrameIndex);
|
||||
writer.KeyInt("deckLinkPlaybackStreamTime", snapshot.deckLinkPlaybackStreamTime);
|
||||
writer.KeyUInt("deckLinkScheduleRealignments", snapshot.deckLinkScheduleRealignments);
|
||||
writer.EndObject();
|
||||
}
|
||||
|
||||
inline std::string CadenceTelemetryToJson(const CadenceTelemetrySnapshot& snapshot)
|
||||
{
|
||||
JsonWriter writer;
|
||||
WriteCadenceTelemetryJson(writer, snapshot);
|
||||
return writer.StringValue();
|
||||
}
|
||||
}
|
||||
@@ -1,122 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "CadenceTelemetry.h"
|
||||
#include "../logging/Logger.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
|
||||
namespace RenderCadenceCompositor
|
||||
{
|
||||
struct TelemetryHealthMonitorConfig
|
||||
{
|
||||
std::chrono::milliseconds interval = std::chrono::seconds(1);
|
||||
std::size_t scheduledStarvationThreshold = 0;
|
||||
};
|
||||
|
||||
class TelemetryHealthMonitor
|
||||
{
|
||||
public:
|
||||
explicit TelemetryHealthMonitor(TelemetryHealthMonitorConfig config = TelemetryHealthMonitorConfig()) :
|
||||
mConfig(config)
|
||||
{
|
||||
}
|
||||
|
||||
TelemetryHealthMonitor(const TelemetryHealthMonitor&) = delete;
|
||||
TelemetryHealthMonitor& operator=(const TelemetryHealthMonitor&) = delete;
|
||||
|
||||
~TelemetryHealthMonitor()
|
||||
{
|
||||
Stop();
|
||||
}
|
||||
|
||||
template <typename SystemFrameExchange, typename Output, typename OutputThread, typename RenderThread>
|
||||
void Start(const SystemFrameExchange& exchange, const Output& output, const OutputThread& outputThread, const RenderThread& renderThread)
|
||||
{
|
||||
if (mRunning)
|
||||
return;
|
||||
mStopping = false;
|
||||
mThread = std::thread([this, &exchange, &output, &outputThread, &renderThread]() {
|
||||
CadenceTelemetry telemetry;
|
||||
CadenceTelemetrySnapshot previous;
|
||||
bool hasPrevious = false;
|
||||
while (!mStopping)
|
||||
{
|
||||
std::this_thread::sleep_for(mConfig.interval);
|
||||
const CadenceTelemetrySnapshot snapshot = telemetry.Sample(exchange, output, outputThread, renderThread);
|
||||
ReportHealth(snapshot, hasPrevious ? &previous : nullptr);
|
||||
previous = snapshot;
|
||||
hasPrevious = true;
|
||||
}
|
||||
});
|
||||
mRunning = true;
|
||||
}
|
||||
|
||||
void Stop()
|
||||
{
|
||||
mStopping = true;
|
||||
if (mThread.joinable())
|
||||
mThread.join();
|
||||
mRunning = false;
|
||||
}
|
||||
|
||||
private:
|
||||
void ReportHealth(const CadenceTelemetrySnapshot& snapshot, const CadenceTelemetrySnapshot* previous) const
|
||||
{
|
||||
if (!previous)
|
||||
return;
|
||||
|
||||
const uint64_t lateDelta = snapshot.displayedLate - previous->displayedLate;
|
||||
const uint64_t droppedDelta = snapshot.dropped - previous->dropped;
|
||||
const uint64_t scheduleFailureDelta = snapshot.scheduleFailures - previous->scheduleFailures;
|
||||
|
||||
if (droppedDelta > 0 || lateDelta > 0)
|
||||
{
|
||||
std::ostringstream message;
|
||||
message << "DeckLink reported frame timing issue: lateDelta=" << lateDelta
|
||||
<< " droppedDelta=" << droppedDelta
|
||||
<< " totalLate=" << snapshot.displayedLate
|
||||
<< " totalDropped=" << snapshot.dropped
|
||||
<< " scheduleLead=";
|
||||
if (snapshot.deckLinkScheduleLeadAvailable)
|
||||
message << snapshot.deckLinkScheduleLeadFrames;
|
||||
else
|
||||
message << "n/a";
|
||||
message << " realignments=" << snapshot.deckLinkScheduleRealignments;
|
||||
LogWarning("telemetry", message.str());
|
||||
}
|
||||
|
||||
if (scheduleFailureDelta > 0)
|
||||
{
|
||||
std::ostringstream message;
|
||||
message << "DeckLink schedule failures increased: delta=" << scheduleFailureDelta
|
||||
<< " total=" << snapshot.scheduleFailures;
|
||||
LogWarning("telemetry", message.str());
|
||||
}
|
||||
|
||||
const bool appScheduledStarved = snapshot.scheduledFrames <= mConfig.scheduledStarvationThreshold
|
||||
&& snapshot.scheduledTotal > 0;
|
||||
const bool deckLinkStarved = snapshot.deckLinkBufferedAvailable && snapshot.deckLinkBuffered == 0;
|
||||
if (appScheduledStarved || deckLinkStarved)
|
||||
{
|
||||
std::ostringstream message;
|
||||
message << "Output buffer starvation detected: scheduled=" << snapshot.scheduledFrames
|
||||
<< " decklinkBuffered=";
|
||||
if (snapshot.deckLinkBufferedAvailable)
|
||||
message << snapshot.deckLinkBuffered;
|
||||
else
|
||||
message << "n/a";
|
||||
message << " renderFps=" << snapshot.renderFps
|
||||
<< " scheduleFps=" << snapshot.scheduleFps;
|
||||
LogError("telemetry", message.str());
|
||||
}
|
||||
}
|
||||
|
||||
TelemetryHealthMonitorConfig mConfig;
|
||||
std::thread mThread;
|
||||
std::atomic<bool> mStopping{ false };
|
||||
std::atomic<bool> mRunning{ false };
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user