Files
video-shader-toys/apps/LoopThroughWithOpenGLCompositing/gl/RenderEngine.cpp
Aiden a24cdc0630
All checks were successful
CI / React UI Build (push) Successful in 11s
CI / Native Windows Build And Tests (push) Successful in 2m38s
CI / Windows Release Package (push) Successful in 2m42s
Osc orchestration
2026-05-10 23:59:48 +10:00

562 lines
18 KiB
C++

#include "RenderEngine.h"
#include "RuntimeParameterUtils.h"
#include <gl/gl.h>
#include <algorithm>
#include <cctype>
#include <cmath>
#include <cstddef>
namespace
{
constexpr auto kOscOverlayCommitDelay = std::chrono::milliseconds(150);
constexpr double kOscSmoothingReferenceFps = 60.0;
constexpr double kOscSmoothingMaxStepSeconds = 0.25;
std::string SimplifyOscControlKey(const std::string& text)
{
std::string simplified;
for (unsigned char ch : text)
{
if (std::isalnum(ch))
simplified.push_back(static_cast<char>(std::tolower(ch)));
}
return simplified;
}
bool MatchesOscControlKey(const std::string& candidate, const std::string& key)
{
return candidate == key || SimplifyOscControlKey(candidate) == SimplifyOscControlKey(key);
}
double ClampOscAlpha(double value)
{
return (std::max)(0.0, (std::min)(1.0, value));
}
double ComputeTimeBasedOscAlpha(double smoothing, double deltaSeconds)
{
const double clampedSmoothing = ClampOscAlpha(smoothing);
if (clampedSmoothing <= 0.0)
return 0.0;
if (clampedSmoothing >= 1.0)
return 1.0;
const double clampedDeltaSeconds = (std::max)(0.0, (std::min)(kOscSmoothingMaxStepSeconds, deltaSeconds));
if (clampedDeltaSeconds <= 0.0)
return 0.0;
const double frameScale = clampedDeltaSeconds * kOscSmoothingReferenceFps;
return ClampOscAlpha(1.0 - std::pow(1.0 - clampedSmoothing, frameScale));
}
JsonValue BuildOscCommitValue(const ShaderParameterDefinition& definition, const ShaderParameterValue& value)
{
switch (definition.type)
{
case ShaderParameterType::Boolean:
return JsonValue(value.booleanValue);
case ShaderParameterType::Enum:
return JsonValue(value.enumValue);
case ShaderParameterType::Text:
return JsonValue(value.textValue);
case ShaderParameterType::Trigger:
case ShaderParameterType::Float:
return JsonValue(value.numberValues.empty() ? 0.0 : value.numberValues.front());
case ShaderParameterType::Vec2:
case ShaderParameterType::Color:
{
JsonValue array = JsonValue::MakeArray();
for (double number : value.numberValues)
array.pushBack(JsonValue(number));
return array;
}
}
return JsonValue();
}
}
RenderEngine::RenderEngine(
RuntimeSnapshotProvider& runtimeSnapshotProvider,
HealthTelemetry& healthTelemetry,
CRITICAL_SECTION& mutex,
HDC hdc,
HGLRC hglrc,
RenderEffectCallback renderEffect,
ScreenshotCallback screenshotReady,
PreviewPaintCallback previewPaint) :
mRenderer(),
mRenderPass(mRenderer),
mRenderPipeline(mRenderer, runtimeSnapshotProvider, healthTelemetry, std::move(renderEffect), std::move(screenshotReady), std::move(previewPaint)),
mShaderPrograms(mRenderer, runtimeSnapshotProvider),
mRuntimeSnapshotProvider(runtimeSnapshotProvider),
mMutex(mutex),
mHdc(hdc),
mHglrc(hglrc)
{
}
RenderEngine::~RenderEngine()
{
mRenderer.DestroyResources();
}
bool RenderEngine::CompileDecodeShader(int errorMessageSize, char* errorMessage)
{
return mShaderPrograms.CompileDecodeShader(errorMessageSize, errorMessage);
}
bool RenderEngine::CompileOutputPackShader(int errorMessageSize, char* errorMessage)
{
return mShaderPrograms.CompileOutputPackShader(errorMessageSize, errorMessage);
}
bool RenderEngine::InitializeResources(
unsigned inputFrameWidth,
unsigned inputFrameHeight,
unsigned captureTextureWidth,
unsigned outputFrameWidth,
unsigned outputFrameHeight,
unsigned outputPackTextureWidth,
std::string& error)
{
return mRenderer.InitializeResources(
inputFrameWidth,
inputFrameHeight,
captureTextureWidth,
outputFrameWidth,
outputFrameHeight,
outputPackTextureWidth,
error);
}
bool RenderEngine::CompileLayerPrograms(unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage)
{
return mShaderPrograms.CompileLayerPrograms(inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
}
bool RenderEngine::CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage)
{
return mShaderPrograms.CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage);
}
bool RenderEngine::ApplyPreparedShaderBuild(
const PreparedShaderBuild& preparedBuild,
unsigned inputFrameWidth,
unsigned inputFrameHeight,
bool preserveFeedbackState,
int errorMessageSize,
char* errorMessage)
{
if (!CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage))
return false;
mCachedLayerRenderStates = mShaderPrograms.CommittedLayerStates();
mCachedRenderStateVersion = preparedBuild.renderSnapshot.versions.renderStateVersion;
mCachedParameterStateVersion = preparedBuild.renderSnapshot.versions.parameterStateVersion;
mCachedRenderStateWidth = preparedBuild.renderSnapshot.outputWidth;
mCachedRenderStateHeight = preparedBuild.renderSnapshot.outputHeight;
ResetTemporalHistoryState();
if (!preserveFeedbackState)
ResetShaderFeedbackState();
return true;
}
const std::vector<RuntimeRenderState>& RenderEngine::CommittedLayerStates() const
{
return mShaderPrograms.CommittedLayerStates();
}
void RenderEngine::ResetTemporalHistoryState()
{
mShaderPrograms.ResetTemporalHistoryState();
}
void RenderEngine::ResetShaderFeedbackState()
{
mShaderPrograms.ResetShaderFeedbackState();
}
void RenderEngine::ClearOscOverlayState()
{
mOscOverlayStates.clear();
}
void RenderEngine::UpdateOscOverlayState(
const std::vector<OscOverlayUpdate>& updates,
const std::vector<OscOverlayCommitCompletion>& completedCommits)
{
for (const OscOverlayCommitCompletion& completedCommit : completedCommits)
{
auto overlayIt = mOscOverlayStates.find(completedCommit.routeKey);
if (overlayIt == mOscOverlayStates.end())
continue;
OscOverlayState& overlay = overlayIt->second;
if (overlay.commitQueued &&
overlay.pendingCommitGeneration == completedCommit.generation &&
overlay.generation == completedCommit.generation)
{
mOscOverlayStates.erase(overlayIt);
}
}
const auto now = std::chrono::steady_clock::now();
for (const OscOverlayUpdate& update : updates)
{
auto overlayIt = mOscOverlayStates.find(update.routeKey);
if (overlayIt == mOscOverlayStates.end())
{
OscOverlayState overlay;
overlay.layerKey = update.layerKey;
overlay.parameterKey = update.parameterKey;
overlay.targetValue = update.targetValue;
overlay.lastUpdatedTime = now;
overlay.lastAppliedTime = now;
overlay.generation = 1;
mOscOverlayStates[update.routeKey] = std::move(overlay);
}
else
{
overlayIt->second.targetValue = update.targetValue;
overlayIt->second.lastUpdatedTime = now;
overlayIt->second.generation += 1;
overlayIt->second.commitQueued = false;
}
}
}
void RenderEngine::ResizeView(int width, int height)
{
mRenderer.ResizeView(width, height);
}
bool RenderEngine::TryPresentPreview(bool force, unsigned previewFps, unsigned outputFrameWidth, unsigned outputFrameHeight)
{
if (!force)
{
if (previewFps == 0)
return false;
const auto now = std::chrono::steady_clock::now();
const auto minimumInterval = std::chrono::microseconds(1000000 / (previewFps == 0 ? 1u : previewFps));
if (mLastPreviewPresentTime != std::chrono::steady_clock::time_point() &&
now - mLastPreviewPresentTime < minimumInterval)
{
return false;
}
}
if (!TryEnterCriticalSection(&mMutex))
return false;
mRenderer.PresentToWindow(mHdc, outputFrameWidth, outputFrameHeight);
mLastPreviewPresentTime = std::chrono::steady_clock::now();
LeaveCriticalSection(&mMutex);
return true;
}
bool RenderEngine::TryUploadInputFrame(const VideoIOFrame& inputFrame, const VideoIOState& videoState)
{
if (inputFrame.hasNoInputSource || inputFrame.bytes == nullptr)
return true;
const long textureSize = inputFrame.rowBytes * static_cast<long>(inputFrame.height);
if (!TryEnterCriticalSection(&mMutex))
return false;
wglMakeCurrent(mHdc, mHglrc);
glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mRenderer.TextureUploadBuffer());
glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, inputFrame.bytes, GL_DYNAMIC_DRAW);
glBindTexture(GL_TEXTURE_2D, mRenderer.CaptureTexture());
if (inputFrame.pixelFormat == VideoIOPixelFormat::V210)
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, videoState.captureTextureWidth, videoState.inputFrameSize.height, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
else
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, videoState.captureTextureWidth, videoState.inputFrameSize.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
wglMakeCurrent(NULL, NULL);
LeaveCriticalSection(&mMutex);
return true;
}
bool RenderEngine::RenderOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame)
{
EnterCriticalSection(&mMutex);
wglMakeCurrent(mHdc, mHglrc);
const bool rendered = mRenderPipeline.RenderFrame(context, outputFrame);
wglMakeCurrent(NULL, NULL);
LeaveCriticalSection(&mMutex);
return rendered;
}
bool RenderEngine::ResolveRenderLayerStates(
bool useCommittedLayerStates,
unsigned renderWidth,
unsigned renderHeight,
double oscSmoothing,
std::vector<OscOverlayCommitRequest>* commitRequests,
std::vector<RuntimeRenderState>& layerStates)
{
layerStates.clear();
if (useCommittedLayerStates)
{
layerStates = mShaderPrograms.CommittedLayerStates();
ApplyOscOverlays(layerStates, false, oscSmoothing, commitRequests);
mRuntimeSnapshotProvider.RefreshDynamicRenderStateFields(layerStates);
return true;
}
const RuntimeSnapshotVersions versions = mRuntimeSnapshotProvider.GetVersions();
const bool renderStateCacheValid =
!mCachedLayerRenderStates.empty() &&
mCachedRenderStateVersion == versions.renderStateVersion &&
mCachedRenderStateWidth == renderWidth &&
mCachedRenderStateHeight == renderHeight;
if (renderStateCacheValid)
{
RuntimeRenderStateSnapshot renderSnapshot;
renderSnapshot.outputWidth = renderWidth;
renderSnapshot.outputHeight = renderHeight;
renderSnapshot.versions.renderStateVersion = mCachedRenderStateVersion;
renderSnapshot.versions.parameterStateVersion = mCachedParameterStateVersion;
renderSnapshot.states = mCachedLayerRenderStates;
ApplyOscOverlays(renderSnapshot.states, true, oscSmoothing, commitRequests);
if (mCachedParameterStateVersion != versions.parameterStateVersion &&
mRuntimeSnapshotProvider.TryRefreshSnapshotParameters(renderSnapshot))
{
mCachedParameterStateVersion = renderSnapshot.versions.parameterStateVersion;
ApplyOscOverlays(renderSnapshot.states, true, oscSmoothing, commitRequests);
}
mCachedLayerRenderStates = renderSnapshot.states;
layerStates = renderSnapshot.states;
mRuntimeSnapshotProvider.RefreshDynamicRenderStateFields(layerStates);
return true;
}
RuntimeRenderStateSnapshot renderSnapshot;
if (mRuntimeSnapshotProvider.TryGetRenderStateSnapshot(renderWidth, renderHeight, renderSnapshot))
{
mCachedLayerRenderStates = renderSnapshot.states;
mCachedRenderStateVersion = renderSnapshot.versions.renderStateVersion;
mCachedParameterStateVersion = renderSnapshot.versions.parameterStateVersion;
mCachedRenderStateWidth = renderSnapshot.outputWidth;
mCachedRenderStateHeight = renderSnapshot.outputHeight;
ApplyOscOverlays(mCachedLayerRenderStates, true, oscSmoothing, commitRequests);
layerStates = mCachedLayerRenderStates;
return true;
}
ApplyOscOverlays(mCachedLayerRenderStates, true, oscSmoothing, commitRequests);
layerStates = mCachedLayerRenderStates;
mRuntimeSnapshotProvider.RefreshDynamicRenderStateFields(layerStates);
return !layerStates.empty();
}
void RenderEngine::ApplyOscOverlays(
std::vector<RuntimeRenderState>& states,
bool allowCommit,
double smoothing,
std::vector<OscOverlayCommitRequest>* commitRequests)
{
if (states.empty() || mOscOverlayStates.empty())
return;
const auto now = std::chrono::steady_clock::now();
const double clampedSmoothing = ClampOscAlpha(smoothing);
std::vector<std::string> overlayKeysToRemove;
for (auto& item : mOscOverlayStates)
{
const std::string& routeKey = item.first;
OscOverlayState& overlay = item.second;
auto stateIt = std::find_if(states.begin(), states.end(),
[&overlay](const RuntimeRenderState& state)
{
return MatchesOscControlKey(state.layerId, overlay.layerKey) ||
MatchesOscControlKey(state.shaderId, overlay.layerKey) ||
MatchesOscControlKey(state.shaderName, overlay.layerKey);
});
if (stateIt == states.end())
continue;
auto definitionIt = std::find_if(stateIt->parameterDefinitions.begin(), stateIt->parameterDefinitions.end(),
[&overlay](const ShaderParameterDefinition& definition)
{
return MatchesOscControlKey(definition.id, overlay.parameterKey) ||
MatchesOscControlKey(definition.label, overlay.parameterKey);
});
if (definitionIt == stateIt->parameterDefinitions.end())
continue;
ShaderParameterValue targetValue;
std::string normalizeError;
if (!NormalizeAndValidateParameterValue(*definitionIt, overlay.targetValue, targetValue, normalizeError))
continue;
if (definitionIt->type == ShaderParameterType::Trigger)
{
ShaderParameterValue& value = stateIt->parameterValues[definitionIt->id];
const double previousCount = value.numberValues.empty() ? 0.0 : value.numberValues[0];
const double triggerTime = stateIt->timeSeconds;
value.numberValues = { previousCount + 1.0, triggerTime };
overlayKeysToRemove.push_back(routeKey);
continue;
}
const bool smoothable =
clampedSmoothing > 0.0 &&
(definitionIt->type == ShaderParameterType::Float ||
definitionIt->type == ShaderParameterType::Vec2 ||
definitionIt->type == ShaderParameterType::Color);
if (!smoothable)
{
overlay.currentValue = targetValue;
overlay.hasCurrentValue = true;
stateIt->parameterValues[definitionIt->id] = overlay.currentValue;
if (allowCommit &&
!overlay.commitQueued &&
now - overlay.lastUpdatedTime >= kOscOverlayCommitDelay &&
commitRequests)
{
commitRequests->push_back({ routeKey, overlay.layerKey, overlay.parameterKey, overlay.targetValue, overlay.generation });
overlay.pendingCommitGeneration = overlay.generation;
overlay.commitQueued = true;
}
continue;
}
if (!overlay.hasCurrentValue)
{
overlay.currentValue = DefaultValueForDefinition(*definitionIt);
auto currentIt = stateIt->parameterValues.find(definitionIt->id);
if (currentIt != stateIt->parameterValues.end())
overlay.currentValue = currentIt->second;
overlay.hasCurrentValue = true;
}
if (overlay.currentValue.numberValues.size() != targetValue.numberValues.size())
overlay.currentValue.numberValues = targetValue.numberValues;
double smoothingAlpha = clampedSmoothing;
if (overlay.lastAppliedTime != std::chrono::steady_clock::time_point())
{
const double deltaSeconds =
std::chrono::duration_cast<std::chrono::duration<double>>(now - overlay.lastAppliedTime).count();
smoothingAlpha = ComputeTimeBasedOscAlpha(clampedSmoothing, deltaSeconds);
}
overlay.lastAppliedTime = now;
ShaderParameterValue nextValue = targetValue;
bool converged = true;
for (std::size_t index = 0; index < targetValue.numberValues.size(); ++index)
{
const double currentNumber = overlay.currentValue.numberValues[index];
const double targetNumber = targetValue.numberValues[index];
const double delta = targetNumber - currentNumber;
double nextNumber = currentNumber + delta * smoothingAlpha;
if (std::fabs(delta) <= 0.0005)
nextNumber = targetNumber;
else
converged = false;
nextValue.numberValues[index] = nextNumber;
}
if (converged)
nextValue.numberValues = targetValue.numberValues;
overlay.currentValue = nextValue;
overlay.hasCurrentValue = true;
stateIt->parameterValues[definitionIt->id] = overlay.currentValue;
if (allowCommit &&
converged &&
!overlay.commitQueued &&
now - overlay.lastUpdatedTime >= kOscOverlayCommitDelay &&
commitRequests)
{
commitRequests->push_back({ routeKey, overlay.layerKey, overlay.parameterKey, BuildOscCommitValue(*definitionIt, overlay.currentValue), overlay.generation });
overlay.pendingCommitGeneration = overlay.generation;
overlay.commitQueued = true;
}
}
for (const std::string& overlayKey : overlayKeysToRemove)
mOscOverlayStates.erase(overlayKey);
}
void RenderEngine::RenderLayerStack(
bool hasInputSource,
const std::vector<RuntimeRenderState>& layerStates,
unsigned inputFrameWidth,
unsigned inputFrameHeight,
unsigned captureTextureWidth,
VideoIOPixelFormat inputPixelFormat,
unsigned historyCap)
{
mRenderPass.Render(
hasInputSource,
layerStates,
inputFrameWidth,
inputFrameHeight,
captureTextureWidth,
inputPixelFormat,
historyCap,
[this](const RuntimeRenderState& state, OpenGLRenderer::LayerProgram::TextBinding& textBinding, std::string& error) {
return mShaderPrograms.UpdateTextBindingTexture(state, textBinding, error);
},
[this](const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength, bool feedbackAvailable) {
return mShaderPrograms.UpdateGlobalParamsBuffer(state, availableSourceHistoryLength, availableTemporalHistoryLength, feedbackAvailable);
});
}
bool RenderEngine::ReadOutputFrameRgba(unsigned width, unsigned height, std::vector<unsigned char>& bottomUpPixels)
{
if (width == 0 || height == 0)
return false;
EnterCriticalSection(&mMutex);
wglMakeCurrent(mHdc, mHglrc);
bottomUpPixels.resize(static_cast<std::size_t>(width) * height * 4);
glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.OutputFramebuffer());
glReadBuffer(GL_COLOR_ATTACHMENT0);
glPixelStorei(GL_PACK_ALIGNMENT, 1);
glPixelStorei(GL_PACK_ROW_LENGTH, 0);
glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, bottomUpPixels.data());
glPixelStorei(GL_PACK_ALIGNMENT, 4);
wglMakeCurrent(NULL, NULL);
LeaveCriticalSection(&mMutex);
return true;
}
bool RenderEngine::CaptureOutputFrameRgbaTopDown(unsigned width, unsigned height, std::vector<unsigned char>& topDownPixels)
{
std::vector<unsigned char> bottomUpPixels;
if (!ReadOutputFrameRgba(width, height, bottomUpPixels))
return false;
topDownPixels.resize(bottomUpPixels.size());
const std::size_t rowBytes = static_cast<std::size_t>(width) * 4;
for (unsigned y = 0; y < height; ++y)
{
const unsigned sourceY = height - 1 - y;
std::copy(
bottomUpPixels.begin() + static_cast<std::ptrdiff_t>(sourceY * rowBytes),
bottomUpPixels.begin() + static_cast<std::ptrdiff_t>((sourceY + 1) * rowBytes),
topDownPixels.begin() + static_cast<std::ptrdiff_t>(y * rowBytes));
}
return true;
}