117 lines
3.3 KiB
C++
117 lines
3.3 KiB
C++
#pragma once
|
|
|
|
#include "CadenceTelemetry.h"
|
|
#include "../logging/Logger.h"
|
|
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <sstream>
|
|
#include <thread>
|
|
|
|
namespace RenderCadenceCompositor
|
|
{
|
|
struct TelemetryHealthMonitorConfig
|
|
{
|
|
std::chrono::milliseconds interval = std::chrono::seconds(1);
|
|
std::size_t scheduledStarvationThreshold = 0;
|
|
};
|
|
|
|
class TelemetryHealthMonitor
|
|
{
|
|
public:
|
|
explicit TelemetryHealthMonitor(TelemetryHealthMonitorConfig config = TelemetryHealthMonitorConfig()) :
|
|
mConfig(config)
|
|
{
|
|
}
|
|
|
|
TelemetryHealthMonitor(const TelemetryHealthMonitor&) = delete;
|
|
TelemetryHealthMonitor& operator=(const TelemetryHealthMonitor&) = delete;
|
|
|
|
~TelemetryHealthMonitor()
|
|
{
|
|
Stop();
|
|
}
|
|
|
|
template <typename SystemFrameExchange, typename Output, typename OutputThread, typename RenderThread>
|
|
void Start(const SystemFrameExchange& exchange, const Output& output, const OutputThread& outputThread, const RenderThread& renderThread)
|
|
{
|
|
if (mRunning)
|
|
return;
|
|
mStopping = false;
|
|
mThread = std::thread([this, &exchange, &output, &outputThread, &renderThread]() {
|
|
CadenceTelemetry telemetry;
|
|
CadenceTelemetrySnapshot previous;
|
|
bool hasPrevious = false;
|
|
while (!mStopping)
|
|
{
|
|
std::this_thread::sleep_for(mConfig.interval);
|
|
const CadenceTelemetrySnapshot snapshot = telemetry.Sample(exchange, output, outputThread, renderThread);
|
|
ReportHealth(snapshot, hasPrevious ? &previous : nullptr);
|
|
previous = snapshot;
|
|
hasPrevious = true;
|
|
}
|
|
});
|
|
mRunning = true;
|
|
}
|
|
|
|
void Stop()
|
|
{
|
|
mStopping = true;
|
|
if (mThread.joinable())
|
|
mThread.join();
|
|
mRunning = false;
|
|
}
|
|
|
|
private:
|
|
void ReportHealth(const CadenceTelemetrySnapshot& snapshot, const CadenceTelemetrySnapshot* previous) const
|
|
{
|
|
if (!previous)
|
|
return;
|
|
|
|
const uint64_t lateDelta = snapshot.displayedLate - previous->displayedLate;
|
|
const uint64_t droppedDelta = snapshot.dropped - previous->dropped;
|
|
const uint64_t scheduleFailureDelta = snapshot.scheduleFailures - previous->scheduleFailures;
|
|
|
|
if (droppedDelta > 0 || lateDelta > 0)
|
|
{
|
|
std::ostringstream message;
|
|
message << "DeckLink reported frame timing issue: lateDelta=" << lateDelta
|
|
<< " droppedDelta=" << droppedDelta
|
|
<< " totalLate=" << snapshot.displayedLate
|
|
<< " totalDropped=" << snapshot.dropped;
|
|
LogWarning("telemetry", message.str());
|
|
}
|
|
|
|
if (scheduleFailureDelta > 0)
|
|
{
|
|
std::ostringstream message;
|
|
message << "DeckLink schedule failures increased: delta=" << scheduleFailureDelta
|
|
<< " total=" << snapshot.scheduleFailures;
|
|
LogWarning("telemetry", message.str());
|
|
}
|
|
|
|
const bool appScheduledStarved = snapshot.scheduledFrames <= mConfig.scheduledStarvationThreshold
|
|
&& snapshot.scheduledTotal > 0;
|
|
const bool deckLinkStarved = snapshot.deckLinkBufferedAvailable && snapshot.deckLinkBuffered == 0;
|
|
if (appScheduledStarved || deckLinkStarved)
|
|
{
|
|
std::ostringstream message;
|
|
message << "Output buffer starvation detected: scheduled=" << snapshot.scheduledFrames
|
|
<< " decklinkBuffered=";
|
|
if (snapshot.deckLinkBufferedAvailable)
|
|
message << snapshot.deckLinkBuffered;
|
|
else
|
|
message << "n/a";
|
|
message << " renderFps=" << snapshot.renderFps
|
|
<< " scheduleFps=" << snapshot.scheduleFps;
|
|
LogError("telemetry", message.str());
|
|
}
|
|
}
|
|
|
|
TelemetryHealthMonitorConfig mConfig;
|
|
std::thread mThread;
|
|
std::atomic<bool> mStopping{ false };
|
|
std::atomic<bool> mRunning{ false };
|
|
};
|
|
}
|