#include "RenderEngine.h" #include "RuntimeParameterUtils.h" #include "ShaderBuildQueue.h" #include #include #include #include #include namespace { constexpr auto kOscOverlayCommitDelay = std::chrono::milliseconds(150); constexpr double kOscSmoothingReferenceFps = 60.0; constexpr double kOscSmoothingMaxStepSeconds = 0.25; std::string SimplifyOscControlKey(const std::string& text) { std::string simplified; for (unsigned char ch : text) { if (std::isalnum(ch)) simplified.push_back(static_cast(std::tolower(ch))); } return simplified; } bool MatchesOscControlKey(const std::string& candidate, const std::string& key) { return candidate == key || SimplifyOscControlKey(candidate) == SimplifyOscControlKey(key); } double ClampOscAlpha(double value) { return (std::max)(0.0, (std::min)(1.0, value)); } double ComputeTimeBasedOscAlpha(double smoothing, double deltaSeconds) { const double clampedSmoothing = ClampOscAlpha(smoothing); if (clampedSmoothing <= 0.0) return 0.0; if (clampedSmoothing >= 1.0) return 1.0; const double clampedDeltaSeconds = (std::max)(0.0, (std::min)(kOscSmoothingMaxStepSeconds, deltaSeconds)); if (clampedDeltaSeconds <= 0.0) return 0.0; const double frameScale = clampedDeltaSeconds * kOscSmoothingReferenceFps; return ClampOscAlpha(1.0 - std::pow(1.0 - clampedSmoothing, frameScale)); } JsonValue BuildOscCommitValue(const ShaderParameterDefinition& definition, const ShaderParameterValue& value) { switch (definition.type) { case ShaderParameterType::Boolean: return JsonValue(value.booleanValue); case ShaderParameterType::Enum: return JsonValue(value.enumValue); case ShaderParameterType::Text: return JsonValue(value.textValue); case ShaderParameterType::Trigger: case ShaderParameterType::Float: return JsonValue(value.numberValues.empty() ? 0.0 : value.numberValues.front()); case ShaderParameterType::Vec2: case ShaderParameterType::Color: { JsonValue array = JsonValue::MakeArray(); for (double number : value.numberValues) array.pushBack(JsonValue(number)); return array; } } return JsonValue(); } } RenderEngine::RenderEngine( RuntimeSnapshotProvider& runtimeSnapshotProvider, HealthTelemetry& healthTelemetry, CRITICAL_SECTION& mutex, HDC hdc, HGLRC hglrc, RenderEffectCallback renderEffect, ScreenshotCallback screenshotReady, PreviewPaintCallback previewPaint) : mRenderer(), mRenderPass(mRenderer), mRenderPipeline(mRenderer, runtimeSnapshotProvider, healthTelemetry, std::move(renderEffect), std::move(screenshotReady), std::move(previewPaint)), mShaderPrograms(mRenderer, runtimeSnapshotProvider), mRuntimeSnapshotProvider(runtimeSnapshotProvider), mMutex(mutex), mHdc(hdc), mHglrc(hglrc) { } RenderEngine::~RenderEngine() { mRenderer.DestroyResources(); } bool RenderEngine::CompileDecodeShader(int errorMessageSize, char* errorMessage) { return mShaderPrograms.CompileDecodeShader(errorMessageSize, errorMessage); } bool RenderEngine::CompileOutputPackShader(int errorMessageSize, char* errorMessage) { return mShaderPrograms.CompileOutputPackShader(errorMessageSize, errorMessage); } bool RenderEngine::InitializeResources( unsigned inputFrameWidth, unsigned inputFrameHeight, unsigned captureTextureWidth, unsigned outputFrameWidth, unsigned outputFrameHeight, unsigned outputPackTextureWidth, std::string& error) { return mRenderer.InitializeResources( inputFrameWidth, inputFrameHeight, captureTextureWidth, outputFrameWidth, outputFrameHeight, outputPackTextureWidth, error); } bool RenderEngine::CompileLayerPrograms(unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage) { return mShaderPrograms.CompileLayerPrograms(inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage); } bool RenderEngine::CommitPreparedLayerPrograms(const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, int errorMessageSize, char* errorMessage) { return mShaderPrograms.CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage); } bool RenderEngine::ApplyPreparedShaderBuild( const PreparedShaderBuild& preparedBuild, unsigned inputFrameWidth, unsigned inputFrameHeight, bool preserveFeedbackState, int errorMessageSize, char* errorMessage) { if (!CommitPreparedLayerPrograms(preparedBuild, inputFrameWidth, inputFrameHeight, errorMessageSize, errorMessage)) return false; mCachedLayerRenderStates = mShaderPrograms.CommittedLayerStates(); mCachedRenderStateVersion = preparedBuild.renderSnapshot.versions.renderStateVersion; mCachedParameterStateVersion = preparedBuild.renderSnapshot.versions.parameterStateVersion; mCachedRenderStateWidth = preparedBuild.renderSnapshot.outputWidth; mCachedRenderStateHeight = preparedBuild.renderSnapshot.outputHeight; ResetTemporalHistoryState(); if (!preserveFeedbackState) ResetShaderFeedbackState(); return true; } RenderEngine::PreparedShaderBuildApplyResult RenderEngine::TryApplyReadyShaderBuild( ShaderBuildQueue& shaderBuildQueue, unsigned inputFrameWidth, unsigned inputFrameHeight, bool preserveFeedbackState) { PreparedShaderBuildApplyResult result; PreparedShaderBuild readyBuild; if (!shaderBuildQueue.TryConsumeReadyBuild(readyBuild)) return result; result.hadReadyBuild = true; char compilerErrorMessage[1024] = {}; if (!ApplyPreparedShaderBuild( readyBuild, inputFrameWidth, inputFrameHeight, preserveFeedbackState, sizeof(compilerErrorMessage), compilerErrorMessage)) { result.errorMessage = compilerErrorMessage; return result; } result.applied = true; return result; } const std::vector& RenderEngine::CommittedLayerStates() const { return mShaderPrograms.CommittedLayerStates(); } void RenderEngine::ResetTemporalHistoryState() { mShaderPrograms.ResetTemporalHistoryState(); } void RenderEngine::ResetShaderFeedbackState() { mShaderPrograms.ResetShaderFeedbackState(); } void RenderEngine::ApplyRuntimeCoordinatorRenderReset(RuntimeCoordinatorRenderResetScope resetScope) { switch (resetScope) { case RuntimeCoordinatorRenderResetScope::TemporalHistoryOnly: ResetTemporalHistoryState(); break; case RuntimeCoordinatorRenderResetScope::TemporalHistoryAndFeedback: ResetTemporalHistoryState(); ResetShaderFeedbackState(); break; case RuntimeCoordinatorRenderResetScope::None: default: break; } } void RenderEngine::ClearOscOverlayState() { mOscOverlayStates.clear(); } void RenderEngine::UpdateOscOverlayState( const std::vector& updates, const std::vector& completedCommits) { for (const OscOverlayCommitCompletion& completedCommit : completedCommits) { auto overlayIt = mOscOverlayStates.find(completedCommit.routeKey); if (overlayIt == mOscOverlayStates.end()) continue; OscOverlayState& overlay = overlayIt->second; if (overlay.commitQueued && overlay.pendingCommitGeneration == completedCommit.generation && overlay.generation == completedCommit.generation) { mOscOverlayStates.erase(overlayIt); } } const auto now = std::chrono::steady_clock::now(); for (const OscOverlayUpdate& update : updates) { auto overlayIt = mOscOverlayStates.find(update.routeKey); if (overlayIt == mOscOverlayStates.end()) { OscOverlayState overlay; overlay.layerKey = update.layerKey; overlay.parameterKey = update.parameterKey; overlay.targetValue = update.targetValue; overlay.lastUpdatedTime = now; overlay.lastAppliedTime = now; overlay.generation = 1; mOscOverlayStates[update.routeKey] = std::move(overlay); } else { overlayIt->second.targetValue = update.targetValue; overlayIt->second.lastUpdatedTime = now; overlayIt->second.generation += 1; overlayIt->second.commitQueued = false; } } } void RenderEngine::ResizeView(int width, int height) { mRenderer.ResizeView(width, height); } bool RenderEngine::TryPresentPreview(bool force, unsigned previewFps, unsigned outputFrameWidth, unsigned outputFrameHeight) { if (!force) { if (previewFps == 0) return false; const auto now = std::chrono::steady_clock::now(); const auto minimumInterval = std::chrono::microseconds(1000000 / (previewFps == 0 ? 1u : previewFps)); if (mLastPreviewPresentTime != std::chrono::steady_clock::time_point() && now - mLastPreviewPresentTime < minimumInterval) { return false; } } if (!TryEnterCriticalSection(&mMutex)) return false; mRenderer.PresentToWindow(mHdc, outputFrameWidth, outputFrameHeight); mLastPreviewPresentTime = std::chrono::steady_clock::now(); LeaveCriticalSection(&mMutex); return true; } bool RenderEngine::TryUploadInputFrame(const VideoIOFrame& inputFrame, const VideoIOState& videoState) { if (inputFrame.hasNoInputSource || inputFrame.bytes == nullptr) return true; const long textureSize = inputFrame.rowBytes * static_cast(inputFrame.height); if (!TryEnterCriticalSection(&mMutex)) return false; wglMakeCurrent(mHdc, mHglrc); glPixelStorei(GL_UNPACK_ALIGNMENT, 4); glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mRenderer.TextureUploadBuffer()); glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, inputFrame.bytes, GL_DYNAMIC_DRAW); glBindTexture(GL_TEXTURE_2D, mRenderer.CaptureTexture()); if (inputFrame.pixelFormat == VideoIOPixelFormat::V210) glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, videoState.captureTextureWidth, videoState.inputFrameSize.height, GL_RGBA, GL_UNSIGNED_BYTE, NULL); else glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, videoState.captureTextureWidth, videoState.inputFrameSize.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glBindTexture(GL_TEXTURE_2D, 0); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); wglMakeCurrent(NULL, NULL); LeaveCriticalSection(&mMutex); return true; } bool RenderEngine::RenderOutputFrame(const RenderPipelineFrameContext& context, VideoIOOutputFrame& outputFrame) { EnterCriticalSection(&mMutex); wglMakeCurrent(mHdc, mHglrc); const bool rendered = mRenderPipeline.RenderFrame(context, outputFrame); wglMakeCurrent(NULL, NULL); LeaveCriticalSection(&mMutex); return rendered; } bool RenderEngine::ResolveRenderLayerStates( bool useCommittedLayerStates, unsigned renderWidth, unsigned renderHeight, double oscSmoothing, std::vector* commitRequests, std::vector& layerStates) { layerStates.clear(); if (useCommittedLayerStates) { layerStates = mShaderPrograms.CommittedLayerStates(); ApplyOscOverlays(layerStates, false, oscSmoothing, commitRequests); mRuntimeSnapshotProvider.RefreshDynamicRenderStateFields(layerStates); return true; } const RuntimeSnapshotVersions versions = mRuntimeSnapshotProvider.GetVersions(); const bool renderStateCacheValid = !mCachedLayerRenderStates.empty() && mCachedRenderStateVersion == versions.renderStateVersion && mCachedRenderStateWidth == renderWidth && mCachedRenderStateHeight == renderHeight; if (renderStateCacheValid) { RuntimeRenderStateSnapshot renderSnapshot; renderSnapshot.outputWidth = renderWidth; renderSnapshot.outputHeight = renderHeight; renderSnapshot.versions.renderStateVersion = mCachedRenderStateVersion; renderSnapshot.versions.parameterStateVersion = mCachedParameterStateVersion; renderSnapshot.states = mCachedLayerRenderStates; ApplyOscOverlays(renderSnapshot.states, true, oscSmoothing, commitRequests); if (mCachedParameterStateVersion != versions.parameterStateVersion && mRuntimeSnapshotProvider.TryRefreshPublishedSnapshotParameters(renderSnapshot)) { mCachedParameterStateVersion = renderSnapshot.versions.parameterStateVersion; ApplyOscOverlays(renderSnapshot.states, true, oscSmoothing, commitRequests); } mCachedLayerRenderStates = renderSnapshot.states; layerStates = renderSnapshot.states; mRuntimeSnapshotProvider.RefreshDynamicRenderStateFields(layerStates); return true; } RuntimeRenderStateSnapshot renderSnapshot; if (mRuntimeSnapshotProvider.TryPublishRenderStateSnapshot(renderWidth, renderHeight, renderSnapshot)) { mCachedLayerRenderStates = renderSnapshot.states; mCachedRenderStateVersion = renderSnapshot.versions.renderStateVersion; mCachedParameterStateVersion = renderSnapshot.versions.parameterStateVersion; mCachedRenderStateWidth = renderSnapshot.outputWidth; mCachedRenderStateHeight = renderSnapshot.outputHeight; ApplyOscOverlays(mCachedLayerRenderStates, true, oscSmoothing, commitRequests); layerStates = mCachedLayerRenderStates; return true; } ApplyOscOverlays(mCachedLayerRenderStates, true, oscSmoothing, commitRequests); layerStates = mCachedLayerRenderStates; mRuntimeSnapshotProvider.RefreshDynamicRenderStateFields(layerStates); return !layerStates.empty(); } void RenderEngine::ApplyOscOverlays( std::vector& states, bool allowCommit, double smoothing, std::vector* commitRequests) { if (states.empty() || mOscOverlayStates.empty()) return; const auto now = std::chrono::steady_clock::now(); const double clampedSmoothing = ClampOscAlpha(smoothing); std::vector overlayKeysToRemove; for (auto& item : mOscOverlayStates) { const std::string& routeKey = item.first; OscOverlayState& overlay = item.second; auto stateIt = std::find_if(states.begin(), states.end(), [&overlay](const RuntimeRenderState& state) { return MatchesOscControlKey(state.layerId, overlay.layerKey) || MatchesOscControlKey(state.shaderId, overlay.layerKey) || MatchesOscControlKey(state.shaderName, overlay.layerKey); }); if (stateIt == states.end()) continue; auto definitionIt = std::find_if(stateIt->parameterDefinitions.begin(), stateIt->parameterDefinitions.end(), [&overlay](const ShaderParameterDefinition& definition) { return MatchesOscControlKey(definition.id, overlay.parameterKey) || MatchesOscControlKey(definition.label, overlay.parameterKey); }); if (definitionIt == stateIt->parameterDefinitions.end()) continue; ShaderParameterValue targetValue; std::string normalizeError; if (!NormalizeAndValidateParameterValue(*definitionIt, overlay.targetValue, targetValue, normalizeError)) continue; if (definitionIt->type == ShaderParameterType::Trigger) { ShaderParameterValue& value = stateIt->parameterValues[definitionIt->id]; const double previousCount = value.numberValues.empty() ? 0.0 : value.numberValues[0]; const double triggerTime = stateIt->timeSeconds; value.numberValues = { previousCount + 1.0, triggerTime }; overlayKeysToRemove.push_back(routeKey); continue; } const bool smoothable = clampedSmoothing > 0.0 && (definitionIt->type == ShaderParameterType::Float || definitionIt->type == ShaderParameterType::Vec2 || definitionIt->type == ShaderParameterType::Color); if (!smoothable) { overlay.currentValue = targetValue; overlay.hasCurrentValue = true; stateIt->parameterValues[definitionIt->id] = overlay.currentValue; if (allowCommit && !overlay.commitQueued && now - overlay.lastUpdatedTime >= kOscOverlayCommitDelay && commitRequests) { commitRequests->push_back({ routeKey, overlay.layerKey, overlay.parameterKey, overlay.targetValue, overlay.generation }); overlay.pendingCommitGeneration = overlay.generation; overlay.commitQueued = true; } continue; } if (!overlay.hasCurrentValue) { overlay.currentValue = DefaultValueForDefinition(*definitionIt); auto currentIt = stateIt->parameterValues.find(definitionIt->id); if (currentIt != stateIt->parameterValues.end()) overlay.currentValue = currentIt->second; overlay.hasCurrentValue = true; } if (overlay.currentValue.numberValues.size() != targetValue.numberValues.size()) overlay.currentValue.numberValues = targetValue.numberValues; double smoothingAlpha = clampedSmoothing; if (overlay.lastAppliedTime != std::chrono::steady_clock::time_point()) { const double deltaSeconds = std::chrono::duration_cast>(now - overlay.lastAppliedTime).count(); smoothingAlpha = ComputeTimeBasedOscAlpha(clampedSmoothing, deltaSeconds); } overlay.lastAppliedTime = now; ShaderParameterValue nextValue = targetValue; bool converged = true; for (std::size_t index = 0; index < targetValue.numberValues.size(); ++index) { const double currentNumber = overlay.currentValue.numberValues[index]; const double targetNumber = targetValue.numberValues[index]; const double delta = targetNumber - currentNumber; double nextNumber = currentNumber + delta * smoothingAlpha; if (std::fabs(delta) <= 0.0005) nextNumber = targetNumber; else converged = false; nextValue.numberValues[index] = nextNumber; } if (converged) nextValue.numberValues = targetValue.numberValues; overlay.currentValue = nextValue; overlay.hasCurrentValue = true; stateIt->parameterValues[definitionIt->id] = overlay.currentValue; if (allowCommit && converged && !overlay.commitQueued && now - overlay.lastUpdatedTime >= kOscOverlayCommitDelay && commitRequests) { commitRequests->push_back({ routeKey, overlay.layerKey, overlay.parameterKey, BuildOscCommitValue(*definitionIt, overlay.currentValue), overlay.generation }); overlay.pendingCommitGeneration = overlay.generation; overlay.commitQueued = true; } } for (const std::string& overlayKey : overlayKeysToRemove) mOscOverlayStates.erase(overlayKey); } void RenderEngine::RenderLayerStack( bool hasInputSource, const std::vector& layerStates, unsigned inputFrameWidth, unsigned inputFrameHeight, unsigned captureTextureWidth, VideoIOPixelFormat inputPixelFormat, unsigned historyCap) { mRenderPass.Render( hasInputSource, layerStates, inputFrameWidth, inputFrameHeight, captureTextureWidth, inputPixelFormat, historyCap, [this](const RuntimeRenderState& state, OpenGLRenderer::LayerProgram::TextBinding& textBinding, std::string& error) { return mShaderPrograms.UpdateTextBindingTexture(state, textBinding, error); }, [this](const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength, bool feedbackAvailable) { return mShaderPrograms.UpdateGlobalParamsBuffer(state, availableSourceHistoryLength, availableTemporalHistoryLength, feedbackAvailable); }); } bool RenderEngine::ReadOutputFrameRgba(unsigned width, unsigned height, std::vector& bottomUpPixels) { if (width == 0 || height == 0) return false; EnterCriticalSection(&mMutex); wglMakeCurrent(mHdc, mHglrc); bottomUpPixels.resize(static_cast(width) * height * 4); glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer.OutputFramebuffer()); glReadBuffer(GL_COLOR_ATTACHMENT0); glPixelStorei(GL_PACK_ALIGNMENT, 1); glPixelStorei(GL_PACK_ROW_LENGTH, 0); glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, bottomUpPixels.data()); glPixelStorei(GL_PACK_ALIGNMENT, 4); wglMakeCurrent(NULL, NULL); LeaveCriticalSection(&mMutex); return true; } bool RenderEngine::CaptureOutputFrameRgbaTopDown(unsigned width, unsigned height, std::vector& topDownPixels) { std::vector bottomUpPixels; if (!ReadOutputFrameRgba(width, height, bottomUpPixels)) return false; topDownPixels.resize(bottomUpPixels.size()); const std::size_t rowBytes = static_cast(width) * 4; for (unsigned y = 0; y < height; ++y) { const unsigned sourceY = height - 1 - y; std::copy( bottomUpPixels.begin() + static_cast(sourceY * rowBytes), bottomUpPixels.begin() + static_cast((sourceY + 1) * rowBytes), topDownPixels.begin() + static_cast(y * rowBytes)); } return true; }