diff --git a/OSC/Test.json b/OSC/Test.json index 0c376f5..5c16726 100644 --- a/OSC/Test.json +++ b/OSC/Test.json @@ -36,7 +36,7 @@ "preArgs": "", "typeTags": "", "decimals": 2, - "target": "127.0.0.1:9000", + "target": "192.168.1.46:9000", "ignoreDefaults": false, "bypass": false, "onCreate": "", @@ -53,8 +53,8 @@ "visible": true, "interaction": true, "comments": "XY control for Fisheye Reproject pan and tilt.", - "width": 420, - "height": 420, + "width": 460, + "height": 250, "expand": false, "colorText": "auto", "colorWidget": "auto", @@ -70,14 +70,14 @@ "css": "", "pips": true, "snap": false, - "spring": false, + "spring": true, "rangeX": { - "min": -60, - "max": 60 + "min": -1, + "max": 1 }, "rangeY": { - "min": 45, - "max": -45 + "min": 1, + "max": -1 }, "logScaleX": false, "logScaleY": false, @@ -94,13 +94,13 @@ "address": "/VideoShaderToys/fisheye-reproject/xy", "preArgs": "", "typeTags": "", - "decimals": "2f", - "target": "127.0.0.1:9000", + "decimals": "3f", + "target": "192.168.1.46:9000", "ignoreDefaults": false, "bypass": true, - "onCreate": "", - "onValue": "var pan = Array.isArray(value) ? Number(value[0]) : 0;\nvar tilt = Array.isArray(value) ? Number(value[1]) : 0;\nsend('127.0.0.1:9000', '/VideoShaderToys/fisheye-reproject/panDegrees', {type: 'f', value: pan});\nsend('127.0.0.1:9000', '/VideoShaderToys/fisheye-reproject/tiltDegrees', {type: 'f', value: tilt});", - "onTouch": "", + "onCreate": "var state = globalThis.__fisheyePanTiltStick = globalThis.__fisheyePanTiltStick || {};\nstate.target = '192.168.1.46:9000';\nstate.panAddress = '/VideoShaderToys/fisheye-reproject/panDegrees';\nstate.tiltAddress = '/VideoShaderToys/fisheye-reproject/tiltDegrees';\nstate.minPan = -60;\nstate.maxPan = 60;\nstate.minTilt = -45;\nstate.maxTilt = 45;\nstate.pan = 0;\nstate.tilt = 0;\nstate.stickX = 0;\nstate.stickY = 0;\nstate.tickMs = 16;\nstate.stepPan = 0.75;\nstate.stepTilt = 0.75;\nstate.deadzone = 0.14;\nstate.applyCurve = function(input) {\n var amount = Math.abs(input);\n if (amount <= state.deadzone) {\n return 0;\n }\n var normalized = (amount - state.deadzone) / (1 - state.deadzone);\n var softened = normalized * normalized * (3 - (2 * normalized));\n return (input < 0 ? -1 : 1) * softened;\n};\nif (state.timer) {\n clearInterval(state.timer);\n state.timer = null;\n}", + "onValue": "var state = globalThis.__fisheyePanTiltStick = globalThis.__fisheyePanTiltStick || {};\nvar stickX = Array.isArray(value) ? Number(value[0]) : 0;\nvar stickY = Array.isArray(value) ? Number(value[1]) : 0;\nstate.stickX = isFinite(stickX) ? state.applyCurve(stickX) : 0;\nstate.stickY = isFinite(stickY) ? state.applyCurve(stickY) : 0;", + "onTouch": "var state = globalThis.__fisheyePanTiltStick = globalThis.__fisheyePanTiltStick || {};\nif (value) {\n if (!state.timer) {\n state.timer = setInterval(function() {\n if (!state.stickX && !state.stickY) {\n return;\n }\n state.pan = Math.max(state.minPan, Math.min(state.maxPan, state.pan + (state.stickX * state.stepPan)));\n state.tilt = Math.max(state.minTilt, Math.min(state.maxTilt, state.tilt + (state.stickY * state.stepTilt)));\n send(state.target, state.panAddress, {type: 'f', value: state.pan});\n send(state.target, state.tiltAddress, {type: 'f', value: state.tilt});\n }, state.tickMs);\n }\n} else {\n state.stickX = 0;\n state.stickY = 0;\n if (state.timer) {\n clearInterval(state.timer);\n state.timer = null;\n }\n}", "pointSize": 20, "ephemeral": false, "label": "", @@ -121,7 +121,7 @@ "interaction": true, "comments": "", "width": 90, - "height": 420, + "height": 250, "expand": false, "colorText": "auto", "colorWidget": "auto", @@ -144,90 +144,29 @@ "gradient": [], "snap": false, "touchZone": "all", - "spring": false, + "spring": true, "doubleTap": false, "range": { - "min": 100, - "max": 10 + "min": -1, + "max": 1 }, "logScale": false, "sensitivity": 1, "steps": "", "origin": "auto", - "value": "", - "default": 90, + "value": 0, + "default": 0, "linkId": "", "address": "/VideoShaderToys/fisheye-reproject/virtualFovDegrees", "preArgs": "", "typeTags": "", - "decimals": 2, - "target": "127.0.0.1:9000", - "ignoreDefaults": false, - "bypass": false, - "onCreate": "", - "onValue": "", - "onTouch": "" - }, - { - "type": "xy", - "top": 700, - "left": 190, - "lock": false, - "id": "Pan Pad", - "visible": true, - "interaction": true, - "comments": "", - "width": "auto", - "height": "auto", - "expand": false, - "colorText": "auto", - "colorWidget": "auto", - "colorStroke": "auto", - "colorFill": "auto", - "alphaStroke": "auto", - "alphaFillOff": "auto", - "alphaFillOn": "auto", - "lineWidth": "auto", - "borderRadius": "auto", - "padding": "auto", - "html": "", - "css": "", - "pointSize": 20, - "ephemeral": false, - "pips": true, - "label": "", - "snap": false, - "spring": false, - "rangeX": { - "min": -1, - "max": 1 - }, - "rangeY": { - "min": -1, - "max": 1 - }, - "logScaleX": false, - "logScaleY": false, - "stepsX": false, - "stepsY": false, - "clipX": "", - "clipY": "", - "axisLock": "", - "doubleTap": false, - "sensitivity": 1, - "value": "", - "default": "", - "linkId": "", - "address": "/VideoShaderToys/video-transform/pan", - "preArgs": "", - "typeTags": "", - "decimals": 2, - "target": "", + "decimals": "3f", + "target": "192.168.1.46:9000", "ignoreDefaults": false, "bypass": true, - "onCreate": "", - "onValue": "var x = Array.isArray(value) ? Number(value[0]) : 0;\nvar y = Array.isArray(value) ? Number(value[1]) : 0;\nsend('127.0.0.1:9000', '/VideoShaderToys/video-transform/pan', {type: 'f', value: x}, {type: 'f', value: y});", - "onTouch": "" + "onCreate": "var state = globalThis.__fisheyeFovStick = globalThis.__fisheyeFovStick || {};\nstate.target = '192.168.1.46:9000';\nstate.address = '/VideoShaderToys/fisheye-reproject/virtualFovDegrees';\nstate.minFov = 10;\nstate.maxFov = 100;\nstate.fov = 90;\nstate.stick = 0;\nstate.tickMs = 16;\nstate.stepFov = 0.6;\nstate.deadzone = 0.14;\nstate.applyCurve = function(input) {\n var amount = Math.abs(input);\n if (amount <= state.deadzone) {\n return 0;\n }\n var normalized = (amount - state.deadzone) / (1 - state.deadzone);\n var softened = normalized * normalized * (3 - (2 * normalized));\n return (input < 0 ? -1 : 1) * softened;\n};\nif (state.timer) {\n clearInterval(state.timer);\n state.timer = null;\n}", + "onValue": "var state = globalThis.__fisheyeFovStick = globalThis.__fisheyeFovStick || {};\nvar stick = Number(value);\nstate.stick = isFinite(stick) ? state.applyCurve(stick) : 0;", + "onTouch": "var state = globalThis.__fisheyeFovStick = globalThis.__fisheyeFovStick || {};\nif (value) {\n if (!state.timer) {\n state.timer = setInterval(function() {\n if (!state.stick) {\n return;\n }\n state.fov = Math.max(state.minFov, Math.min(state.maxFov, state.fov - (state.stick * state.stepFov)));\n send(state.target, state.address, {type: 'f', value: state.fov});\n }, state.tickMs);\n }\n} else {\n state.stick = 0;\n if (state.timer) {\n clearInterval(state.timer);\n state.timer = null;\n }\n}" } ], "tabs": [] diff --git a/apps/LoopThroughWithOpenGLCompositing/control/RuntimeServices.cpp b/apps/LoopThroughWithOpenGLCompositing/control/RuntimeServices.cpp index 9a360fa..b94a531 100644 --- a/apps/LoopThroughWithOpenGLCompositing/control/RuntimeServices.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/control/RuntimeServices.cpp @@ -102,6 +102,7 @@ bool RuntimeServices::ApplyPendingOscUpdates(std::vector& appl } AppliedOscUpdate appliedUpdate; + appliedUpdate.routeKey = entry.first; appliedUpdate.layerKey = entry.second.layerKey; appliedUpdate.parameterKey = entry.second.parameterKey; appliedUpdate.targetValue = targetValue; @@ -112,6 +113,35 @@ bool RuntimeServices::ApplyPendingOscUpdates(std::vector& appl return true; } +bool RuntimeServices::QueueOscCommit(const std::string& routeKey, const std::string& layerKey, const std::string& parameterKey, const JsonValue& value, uint64_t generation, std::string& error) +{ + (void)error; + + PendingOscCommit commit; + commit.routeKey = routeKey; + commit.layerKey = layerKey; + commit.parameterKey = parameterKey; + commit.value = value; + commit.generation = generation; + + { + std::lock_guard lock(mPendingOscCommitMutex); + mPendingOscCommits[routeKey] = std::move(commit); + } + return true; +} + +void RuntimeServices::ConsumeCompletedOscCommits(std::vector& completedCommits) +{ + completedCommits.clear(); + + std::lock_guard lock(mCompletedOscCommitMutex); + if (mCompletedOscCommits.empty()) + return; + + completedCommits.swap(mCompletedOscCommits); +} + RuntimePollEvents RuntimeServices::ConsumePollEvents() { RuntimePollEvents events; @@ -149,6 +179,33 @@ void RuntimeServices::PollLoop(RuntimeHost& runtimeHost) { while (mPollRunning) { + std::map pendingCommits; + { + std::lock_guard lock(mPendingOscCommitMutex); + pendingCommits.swap(mPendingOscCommits); + } + for (const auto& entry : pendingCommits) + { + std::string commitError; + if (runtimeHost.UpdateLayerParameterByControlKey( + entry.second.layerKey, + entry.second.parameterKey, + entry.second.value, + false, + commitError)) + { + CompletedOscCommit completedCommit; + completedCommit.routeKey = entry.second.routeKey; + completedCommit.generation = entry.second.generation; + std::lock_guard lock(mCompletedOscCommitMutex); + mCompletedOscCommits.push_back(std::move(completedCommit)); + } + else if (!commitError.empty()) + { + OutputDebugStringA(("OSC commit failed: " + commitError + "\n").c_str()); + } + } + bool registryChanged = false; bool reloadRequested = false; std::string runtimeError; diff --git a/apps/LoopThroughWithOpenGLCompositing/control/RuntimeServices.h b/apps/LoopThroughWithOpenGLCompositing/control/RuntimeServices.h index 2dc6153..6d6038a 100644 --- a/apps/LoopThroughWithOpenGLCompositing/control/RuntimeServices.h +++ b/apps/LoopThroughWithOpenGLCompositing/control/RuntimeServices.h @@ -28,11 +28,18 @@ class RuntimeServices public: struct AppliedOscUpdate { + std::string routeKey; std::string layerKey; std::string parameterKey; JsonValue targetValue; }; + struct CompletedOscCommit + { + std::string routeKey; + uint64_t generation = 0; + }; + RuntimeServices(); ~RuntimeServices(); @@ -43,6 +50,8 @@ public: void RequestBroadcastState(); bool QueueOscUpdate(const std::string& layerKey, const std::string& parameterKey, const std::string& valueJson, std::string& error); bool ApplyPendingOscUpdates(std::vector& appliedUpdates, std::string& error); + bool QueueOscCommit(const std::string& routeKey, const std::string& layerKey, const std::string& parameterKey, const JsonValue& value, uint64_t generation, std::string& error); + void ConsumeCompletedOscCommits(std::vector& completedCommits); RuntimePollEvents ConsumePollEvents(); private: @@ -53,6 +62,15 @@ private: std::string valueJson; }; + struct PendingOscCommit + { + std::string routeKey; + std::string layerKey; + std::string parameterKey; + JsonValue value; + uint64_t generation = 0; + }; + void StartPolling(RuntimeHost& runtimeHost); void StopPolling(); void PollLoop(RuntimeHost& runtimeHost); @@ -68,4 +86,8 @@ private: std::string mPollError; std::mutex mPendingOscMutex; std::map mPendingOscUpdates; + std::mutex mPendingOscCommitMutex; + std::map mPendingOscCommits; + std::mutex mCompletedOscCommitMutex; + std::vector mCompletedOscCommits; }; diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp index e272ed9..a453410 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,8 @@ namespace { constexpr auto kOscOverlayCommitDelay = std::chrono::milliseconds(150); +constexpr double kOscSmoothingReferenceFps = 60.0; +constexpr double kOscSmoothingMaxStepSeconds = 0.25; std::string SimplifyOscControlKey(const std::string& text) { @@ -49,6 +52,22 @@ double ClampOscAlpha(double value) return (std::max)(0.0, (std::min)(1.0, value)); } +double ComputeTimeBasedOscAlpha(double smoothing, double deltaSeconds) +{ + const double clampedSmoothing = ClampOscAlpha(smoothing); + if (clampedSmoothing <= 0.0) + return 0.0; + if (clampedSmoothing >= 1.0) + return 1.0; + + const double clampedDeltaSeconds = (std::max)(0.0, (std::min)(kOscSmoothingMaxStepSeconds, deltaSeconds)); + if (clampedDeltaSeconds <= 0.0) + return 0.0; + + const double frameScale = clampedDeltaSeconds * kOscSmoothingReferenceFps; + return ClampOscAlpha(1.0 - std::pow(1.0 - clampedSmoothing, frameScale)); +} + JsonValue BuildOscCommitValue(const ShaderParameterDefinition& definition, const ShaderParameterValue& value) { switch (definition.type) @@ -378,18 +397,35 @@ void OpenGLComposite::renderEffect() { ProcessRuntimePollResults(); std::vector appliedOscUpdates; + std::vector completedOscCommits; if (mRuntimeHost && mRuntimeServices) { std::string oscError; if (!mRuntimeServices->ApplyPendingOscUpdates(appliedOscUpdates, oscError) && !oscError.empty()) OutputDebugStringA(("OSC apply failed: " + oscError + "\n").c_str()); + mRuntimeServices->ConsumeCompletedOscCommits(completedOscCommits); + } + + for (const RuntimeServices::CompletedOscCommit& completedCommit : completedOscCommits) + { + auto overlayIt = mOscOverlayStates.find(completedCommit.routeKey); + if (overlayIt == mOscOverlayStates.end()) + continue; + + OscOverlayState& overlay = overlayIt->second; + if (overlay.commitQueued && + overlay.pendingCommitGeneration == completedCommit.generation && + overlay.generation == completedCommit.generation) + { + mOscOverlayStates.erase(overlayIt); + } } std::set pendingOscRouteKeys; const auto oscNow = std::chrono::steady_clock::now(); for (const RuntimeServices::AppliedOscUpdate& update : appliedOscUpdates) { - const std::string routeKey = update.layerKey + "\n" + update.parameterKey; + const std::string routeKey = update.routeKey; auto overlayIt = mOscOverlayStates.find(routeKey); if (overlayIt == mOscOverlayStates.end()) { @@ -398,12 +434,16 @@ void OpenGLComposite::renderEffect() overlay.parameterKey = update.parameterKey; overlay.targetValue = update.targetValue; overlay.lastUpdatedTime = oscNow; + overlay.lastAppliedTime = oscNow; + overlay.generation = 1; mOscOverlayStates[routeKey] = std::move(overlay); } else { overlayIt->second.targetValue = update.targetValue; overlayIt->second.lastUpdatedTime = oscNow; + overlayIt->second.generation += 1; + overlayIt->second.commitQueued = false; } pendingOscRouteKeys.insert(routeKey); } @@ -465,11 +505,17 @@ void OpenGLComposite::renderEffect() overlay.currentValue = targetValue; overlay.hasCurrentValue = true; stateIt->parameterValues[definitionIt->id] = overlay.currentValue; - if (allowCommit && oscNow - overlay.lastUpdatedTime >= kOscOverlayCommitDelay) + if (allowCommit && + !overlay.commitQueued && + oscNow - overlay.lastUpdatedTime >= kOscOverlayCommitDelay && + mRuntimeServices) { std::string commitError; - if (mRuntimeHost->UpdateLayerParameterByControlKey(overlay.layerKey, overlay.parameterKey, overlay.targetValue, false, commitError)) - overlayKeysToRemove.push_back(item.first); + if (mRuntimeServices->QueueOscCommit(item.first, overlay.layerKey, overlay.parameterKey, overlay.targetValue, overlay.generation, commitError)) + { + overlay.pendingCommitGeneration = overlay.generation; + overlay.commitQueued = true; + } } continue; } @@ -486,6 +532,15 @@ void OpenGLComposite::renderEffect() if (overlay.currentValue.numberValues.size() != targetValue.numberValues.size()) overlay.currentValue.numberValues = targetValue.numberValues; + double smoothingAlpha = smoothing; + if (overlay.lastAppliedTime != std::chrono::steady_clock::time_point()) + { + const double deltaSeconds = + std::chrono::duration_cast>(oscNow - overlay.lastAppliedTime).count(); + smoothingAlpha = ComputeTimeBasedOscAlpha(smoothing, deltaSeconds); + } + overlay.lastAppliedTime = oscNow; + ShaderParameterValue nextValue = targetValue; bool converged = true; for (std::size_t index = 0; index < targetValue.numberValues.size(); ++index) @@ -493,7 +548,7 @@ void OpenGLComposite::renderEffect() const double currentNumber = overlay.currentValue.numberValues[index]; const double targetNumber = targetValue.numberValues[index]; const double delta = targetNumber - currentNumber; - double nextNumber = currentNumber + delta * smoothing; + double nextNumber = currentNumber + delta * smoothingAlpha; if (std::fabs(delta) <= 0.0005) nextNumber = targetNumber; else @@ -507,20 +562,24 @@ void OpenGLComposite::renderEffect() overlay.currentValue = nextValue; overlay.hasCurrentValue = true; stateIt->parameterValues[definitionIt->id] = overlay.currentValue; - if (allowCommit && converged && oscNow - overlay.lastUpdatedTime >= kOscOverlayCommitDelay) + if (allowCommit && + converged && + !overlay.commitQueued && + oscNow - overlay.lastUpdatedTime >= kOscOverlayCommitDelay && + mRuntimeServices) { std::string commitError; JsonValue committedValue = BuildOscCommitValue(*definitionIt, overlay.currentValue); - if (mRuntimeHost->UpdateLayerParameterByControlKey(overlay.layerKey, overlay.parameterKey, committedValue, false, commitError)) - overlayKeysToRemove.push_back(item.first); + if (mRuntimeServices->QueueOscCommit(item.first, overlay.layerKey, overlay.parameterKey, committedValue, overlay.generation, commitError)) + { + overlay.pendingCommitGeneration = overlay.generation; + overlay.commitQueued = true; + } } } - if (allowCommit) - { - for (const std::string& overlayKey : overlayKeysToRemove) - mOscOverlayStates.erase(overlayKey); - } + for (const std::string& overlayKey : overlayKeysToRemove) + mOscOverlayStates.erase(overlayKey); }; const bool hasInputSource = mVideoIO->HasInputSource(); diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h index 8c75f13..5ea3f2b 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h +++ b/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h @@ -81,6 +81,10 @@ private: ShaderParameterValue currentValue; bool hasCurrentValue = false; std::chrono::steady_clock::time_point lastUpdatedTime; + std::chrono::steady_clock::time_point lastAppliedTime; + uint64_t generation = 0; + uint64_t pendingCommitGeneration = 0; + bool commitQueued = false; }; HWND hGLWnd; diff --git a/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.cpp b/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.cpp index 658ecd8..0dff738 100644 --- a/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.cpp +++ b/apps/LoopThroughWithOpenGLCompositing/gl/pipeline/OpenGLVideoIOBridge.cpp @@ -65,7 +65,10 @@ void OpenGLVideoIOBridge::VideoFrameArrived(const VideoIOFrame& inputFrame) const long textureSize = inputFrame.rowBytes * static_cast(inputFrame.height); - EnterCriticalSection(&mMutex); + // Never let input upload stall the playout/render callback. If the GL bridge + // is busy producing an output frame, skip this upload and use the next input. + if (!TryEnterCriticalSection(&mMutex)) + return; wglMakeCurrent(mHdc, mHglrc); // make OpenGL context current in this thread @@ -93,32 +96,29 @@ void OpenGLVideoIOBridge::PlayoutFrameCompleted(const VideoIOCompletion& complet { RecordFramePacing(completion.result); - EnterCriticalSection(&mMutex); - VideoIOOutputFrame outputFrame; if (!mVideoIO.BeginOutputFrame(outputFrame)) - { - LeaveCriticalSection(&mMutex); return; - } const VideoIOState& state = mVideoIO.State(); RenderPipelineFrameContext frameContext; frameContext.videoState = state; frameContext.completion = completion; + EnterCriticalSection(&mMutex); + // make GL context current in this thread wglMakeCurrent(mHdc, mHglrc); mRenderPipeline.RenderFrame(frameContext, outputFrame); + wglMakeCurrent(NULL, NULL); + + LeaveCriticalSection(&mMutex); mVideoIO.EndOutputFrame(outputFrame); mVideoIO.AccountForCompletionResult(completion.result); - // Schedule the next frame for playout + // Schedule the next frame for playout after the GL bridge is released so + // input uploads are not blocked by non-GL output bookkeeping. mVideoIO.ScheduleOutputFrame(outputFrame); - - wglMakeCurrent(NULL, NULL); - - LeaveCriticalSection(&mMutex); }