From c2de2c37384517804ac5b70b883ce3d1fcf5b2cc Mon Sep 17 00:00:00 2001 From: Aiden Date: Thu, 21 May 2026 15:58:23 +1000 Subject: [PATCH] Hot reload --- src/app/RuntimeLayerController.cpp | 2 + src/app/RuntimeLayerController.h | 4 +- src/app/RuntimeLayerControllerBuild.cpp | 5 +- src/app/RuntimeLayerControllerControls.cpp | 20 +++++ src/control/RuntimeControlCommand.cpp | 5 ++ src/control/RuntimeControlCommand.h | 1 + src/runtime/RuntimeLayerModel.cpp | 88 +++++++++++++++++++ src/runtime/RuntimeLayerModel.h | 3 + src/runtime/RuntimeShaderArtifact.h | 1 + src/runtime/RuntimeSlangShaderCompiler.cpp | 1 + src/runtime/SupportedShaderCatalog.cpp | 44 ++++++++++ src/runtime/SupportedShaderCatalog.h | 1 + ...adenceCompositorHttpControlServerTests.cpp | 12 +++ ...adenceCompositorRuntimeLayerModelTests.cpp | 54 ++++++++++-- 14 files changed, 231 insertions(+), 10 deletions(-) diff --git a/src/app/RuntimeLayerController.cpp b/src/app/RuntimeLayerController.cpp index 4e78987..6927ba8 100644 --- a/src/app/RuntimeLayerController.cpp +++ b/src/app/RuntimeLayerController.cpp @@ -21,6 +21,8 @@ void RuntimeLayerController::SetPublisher(RenderLayerPublisher publisher) void RuntimeLayerController::Initialize(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames, std::string& runtimeShaderId) { + mShaderLibrary = shaderLibrary; + mMaxTemporalHistoryFrames = maxTemporalHistoryFrames; LoadSupportedShaderCatalog(shaderLibrary, maxTemporalHistoryFrames); InitializeLayerModel(runtimeShaderId); } diff --git a/src/app/RuntimeLayerController.h b/src/app/RuntimeLayerController.h index e331685..b95101f 100644 --- a/src/app/RuntimeLayerController.h +++ b/src/app/RuntimeLayerController.h @@ -39,7 +39,7 @@ public: const SupportedShaderCatalog& ShaderCatalog() const { return mShaderCatalog; } private: - void LoadSupportedShaderCatalog(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames); + bool LoadSupportedShaderCatalog(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames); void InitializeLayerModel(std::string& runtimeShaderId); void StartLayerShaderBuild(const std::string& layerId, const std::string& shaderId); void RetireLayerShaderBuild(const std::string& layerId); @@ -53,6 +53,8 @@ private: static bool ExtractStringField(const std::string& body, const char* fieldName, std::string& value, std::string& error); RenderLayerPublisher mPublisher; + std::string mShaderLibrary; + unsigned mMaxTemporalHistoryFrames = 0; SupportedShaderCatalog mShaderCatalog; mutable std::mutex mRuntimeLayerMutex; RuntimeLayerModel mRuntimeLayerModel; diff --git a/src/app/RuntimeLayerControllerBuild.cpp b/src/app/RuntimeLayerControllerBuild.cpp index 65e860a..68acc29 100644 --- a/src/app/RuntimeLayerControllerBuild.cpp +++ b/src/app/RuntimeLayerControllerBuild.cpp @@ -7,14 +7,14 @@ namespace RenderCadenceCompositor { -void RuntimeLayerController::LoadSupportedShaderCatalog(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames) +bool RuntimeLayerController::LoadSupportedShaderCatalog(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames) { const std::filesystem::path shaderRoot = FindRepoPath(shaderLibrary); std::string error; if (!mShaderCatalog.Load(shaderRoot, maxTemporalHistoryFrames, error)) { LogWarning("runtime-shader", "Supported shader catalog is empty: " + error); - return; + return false; } std::size_t preparedFontAtlases = 0; @@ -25,6 +25,7 @@ void RuntimeLayerController::LoadSupportedShaderCatalog(const std::string& shade "runtime-shader", "Supported shader catalog loaded with " + std::to_string(mShaderCatalog.Shaders().size()) + " shader(s), prepared " + std::to_string(preparedFontAtlases) + " font atlas asset(s)."); + return true; } void RuntimeLayerController::InitializeLayerModel(std::string& runtimeShaderId) diff --git a/src/app/RuntimeLayerControllerControls.cpp b/src/app/RuntimeLayerControllerControls.cpp index de82d2f..a32359f 100644 --- a/src/app/RuntimeLayerControllerControls.cpp +++ b/src/app/RuntimeLayerControllerControls.cpp @@ -129,6 +129,26 @@ ControlActionResult RuntimeLayerController::HandleControlCommand(const RuntimeCo PublishRuntimeRenderLayers(); return { true, std::string() }; } + case RuntimeControlCommandType::ReloadShaders: + { + if (!LoadSupportedShaderCatalog(mShaderLibrary, mMaxTemporalHistoryFrames)) + return { false, "Shader reload failed; see logs for details." }; + + std::vector> buildsToStart; + { + std::lock_guard lock(mRuntimeLayerMutex); + if (!mRuntimeLayerModel.ReloadFromCatalog(mShaderCatalog, buildsToStart, error)) + return { false, error }; + } + + for (const auto& build : buildsToStart) + { + Log("runtime-shader", "Reload queued shader rebuild: " + build.first + " shader=" + build.second); + StartLayerShaderBuild(build.first, build.second); + } + PublishRuntimeRenderLayers(); + return { true, std::string() }; + } case RuntimeControlCommandType::Unsupported: break; } diff --git a/src/control/RuntimeControlCommand.cpp b/src/control/RuntimeControlCommand.cpp index 96ad9e5..c2f7895 100644 --- a/src/control/RuntimeControlCommand.cpp +++ b/src/control/RuntimeControlCommand.cpp @@ -119,6 +119,11 @@ bool ParseRuntimeControlCommand( command.type = RuntimeControlCommandType::ResetLayerParameters; return RequireStringField(root, "layerId", command.layerId, error); } + if (path == "/api/reload") + { + command.type = RuntimeControlCommandType::ReloadShaders; + return true; + } command.type = RuntimeControlCommandType::Unsupported; error = "Endpoint is not implemented in RenderCadenceCompositor yet."; diff --git a/src/control/RuntimeControlCommand.h b/src/control/RuntimeControlCommand.h index 0bb0095..3e3f299 100644 --- a/src/control/RuntimeControlCommand.h +++ b/src/control/RuntimeControlCommand.h @@ -15,6 +15,7 @@ enum class RuntimeControlCommandType SetLayerShader, UpdateLayerParameter, ResetLayerParameters, + ReloadShaders, Unsupported }; diff --git a/src/runtime/RuntimeLayerModel.cpp b/src/runtime/RuntimeLayerModel.cpp index db86c1d..ef5041b 100644 --- a/src/runtime/RuntimeLayerModel.cpp +++ b/src/runtime/RuntimeLayerModel.cpp @@ -8,6 +8,35 @@ namespace RenderCadenceCompositor { +namespace +{ +JsonValue ParameterValueToJson(const ShaderParameterDefinition& definition, const ShaderParameterValue& value) +{ + switch (definition.type) + { + case ShaderParameterType::Float: + return JsonValue(value.numberValues.empty() ? 0.0 : value.numberValues.front()); + case ShaderParameterType::Vec2: + case ShaderParameterType::Color: + { + JsonValue array = JsonValue::MakeArray(); + for (double number : value.numberValues) + array.pushBack(JsonValue(number)); + return array; + } + case ShaderParameterType::Boolean: + return JsonValue(value.booleanValue); + case ShaderParameterType::Enum: + return JsonValue(value.enumValue); + case ShaderParameterType::Text: + return JsonValue(value.textValue); + case ShaderParameterType::Trigger: + return JsonValue(value.numberValues.empty() ? 0.0 : value.numberValues.front()); + } + return JsonValue(); +} +} + bool RuntimeLayerModel::InitializeSingleLayer(const SupportedShaderCatalog& shaderCatalog, const std::string& shaderId, std::string& error) { Clear(); @@ -27,6 +56,7 @@ bool RuntimeLayerModel::InitializeSingleLayer(const SupportedShaderCatalog& shad Layer layer; layer.id = AllocateLayerId(); layer.shaderId = shaderPackage->id; + layer.packageFingerprint = ShaderPackageFingerprint(*shaderPackage); layer.shaderName = shaderPackage->displayName.empty() ? shaderPackage->id : shaderPackage->displayName; layer.buildState = RuntimeLayerBuildState::Pending; layer.message = "Runtime Slang build is waiting to start."; @@ -48,6 +78,7 @@ bool RuntimeLayerModel::AddLayer(const SupportedShaderCatalog& shaderCatalog, co Layer layer; layer.id = AllocateLayerId(); layer.shaderId = shaderPackage->id; + layer.packageFingerprint = ShaderPackageFingerprint(*shaderPackage); layer.shaderName = shaderPackage->displayName.empty() ? shaderPackage->id : shaderPackage->displayName; layer.buildState = RuntimeLayerBuildState::Pending; layer.message = "Runtime Slang build is waiting to start."; @@ -130,6 +161,7 @@ bool RuntimeLayerModel::SetLayerShader(const SupportedShaderCatalog& shaderCatal } layer->shaderId = shaderPackage->id; + layer->packageFingerprint = ShaderPackageFingerprint(*shaderPackage); layer->shaderName = shaderPackage->displayName.empty() ? shaderPackage->id : shaderPackage->displayName; layer->buildState = RuntimeLayerBuildState::Pending; layer->message = "Runtime Slang build is waiting to start."; @@ -195,6 +227,61 @@ bool RuntimeLayerModel::ResetParameters(const std::string& layerId, std::string& return true; } +bool RuntimeLayerModel::ReloadFromCatalog(const SupportedShaderCatalog& shaderCatalog, std::vector>& buildsToStart, std::string& error) +{ + buildsToStart.clear(); + for (Layer& layer : mLayers) + { + const ShaderPackage* shaderPackage = shaderCatalog.FindPackage(layer.shaderId); + if (!shaderPackage) + { + layer.buildState = RuntimeLayerBuildState::Failed; + layer.message = "Shader '" + layer.shaderId + "' is no longer available after reload."; + continue; + } + + const std::string nextFingerprint = ShaderPackageFingerprint(*shaderPackage); + if (layer.packageFingerprint == nextFingerprint) + continue; + + std::map previousDefinitions; + for (const ShaderParameterDefinition& definition : layer.parameterDefinitions) + previousDefinitions[definition.id] = definition; + + std::map nextValues; + for (const ShaderParameterDefinition& nextDefinition : shaderPackage->parameters) + { + const auto previousDefinitionIt = previousDefinitions.find(nextDefinition.id); + const auto previousValueIt = layer.parameterValues.find(nextDefinition.id); + if (previousDefinitionIt != previousDefinitions.end() + && previousValueIt != layer.parameterValues.end() + && previousDefinitionIt->second.type == nextDefinition.type) + { + ShaderParameterValue preservedValue; + JsonValue valueJson = ParameterValueToJson(previousDefinitionIt->second, previousValueIt->second); + std::string normalizeError; + if (NormalizeAndValidateParameterValue(nextDefinition, valueJson, preservedValue, normalizeError)) + { + nextValues[nextDefinition.id] = preservedValue; + continue; + } + } + nextValues[nextDefinition.id] = DefaultValueForDefinition(nextDefinition); + } + + layer.shaderName = shaderPackage->displayName.empty() ? shaderPackage->id : shaderPackage->displayName; + layer.packageFingerprint = nextFingerprint; + layer.parameterDefinitions = shaderPackage->parameters; + layer.parameterValues = std::move(nextValues); + if (layer.renderReady) + layer.artifact.parameterValues = layer.parameterValues; + buildsToStart.push_back({ layer.id, layer.shaderId }); + } + + error.clear(); + return true; +} + void RuntimeLayerModel::Clear() { mLayers.clear(); @@ -229,6 +316,7 @@ bool RuntimeLayerModel::MarkBuildReady(const RuntimeShaderArtifact& artifact, st } layer->shaderName = artifact.displayName.empty() ? artifact.shaderId : artifact.displayName; + layer->packageFingerprint = artifact.packageFingerprint; layer->buildState = RuntimeLayerBuildState::Ready; layer->message = artifact.message; layer->renderReady = true; diff --git a/src/runtime/RuntimeLayerModel.h b/src/runtime/RuntimeLayerModel.h index c03e7e2..463ea83 100644 --- a/src/runtime/RuntimeLayerModel.h +++ b/src/runtime/RuntimeLayerModel.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -61,6 +62,7 @@ public: bool SetLayerShader(const SupportedShaderCatalog& shaderCatalog, const std::string& layerId, const std::string& shaderId, std::string& error); bool UpdateParameter(const std::string& layerId, const std::string& parameterId, const JsonValue& value, std::string& error); bool ResetParameters(const std::string& layerId, std::string& error); + bool ReloadFromCatalog(const SupportedShaderCatalog& shaderCatalog, std::vector>& buildsToStart, std::string& error); bool MarkBuildStarted(const std::string& layerId, const std::string& message, std::string& error); bool MarkBuildReady(const RuntimeShaderArtifact& artifact, std::string& error); bool MarkBuildFailedForShader(const std::string& shaderId, const std::string& message); @@ -75,6 +77,7 @@ private: { std::string id; std::string shaderId; + std::string packageFingerprint; std::string shaderName; bool bypass = false; RuntimeLayerBuildState buildState = RuntimeLayerBuildState::Pending; diff --git a/src/runtime/RuntimeShaderArtifact.h b/src/runtime/RuntimeShaderArtifact.h index 192ed37..fbfd472 100644 --- a/src/runtime/RuntimeShaderArtifact.h +++ b/src/runtime/RuntimeShaderArtifact.h @@ -25,6 +25,7 @@ struct RuntimeShaderArtifact { std::string layerId; std::string shaderId; + std::string packageFingerprint; std::string displayName; std::string fragmentShaderSource; std::vector passes; diff --git a/src/runtime/RuntimeSlangShaderCompiler.cpp b/src/runtime/RuntimeSlangShaderCompiler.cpp index bfe932b..4df89cc 100644 --- a/src/runtime/RuntimeSlangShaderCompiler.cpp +++ b/src/runtime/RuntimeSlangShaderCompiler.cpp @@ -153,6 +153,7 @@ RuntimeSlangShaderBuild RuntimeSlangShaderCompiler::BuildShader(const std::strin const double milliseconds = std::chrono::duration_cast>(end - start).count(); build.succeeded = true; build.artifact.shaderId = shaderPackage.id; + build.artifact.packageFingerprint = RenderCadenceCompositor::ShaderPackageFingerprint(shaderPackage); build.artifact.displayName = shaderPackage.displayName; build.artifact.parameterDefinitions = shaderPackage.parameters; build.artifact.fontAtlases = std::move(fontAtlasOutputs); diff --git a/src/runtime/SupportedShaderCatalog.cpp b/src/runtime/SupportedShaderCatalog.cpp index 1bb974f..c5ffe52 100644 --- a/src/runtime/SupportedShaderCatalog.cpp +++ b/src/runtime/SupportedShaderCatalog.cpp @@ -3,6 +3,7 @@ #include "ShaderPackageRegistry.h" #include +#include #include namespace RenderCadenceCompositor @@ -74,6 +75,49 @@ ShaderSupportResult CheckStatelessSinglePassShaderSupport(const ShaderPackage& s return { true, std::string() }; } +std::string ShaderPackageFingerprint(const ShaderPackage& shaderPackage) +{ + const auto fileTimeText = [](std::filesystem::file_time_type value) { + return std::to_string(value.time_since_epoch().count()); + }; + + std::ostringstream source; + source << shaderPackage.id << "\n" + << shaderPackage.displayName << "\n" + << shaderPackage.description << "\n" + << shaderPackage.category << "\n" + << shaderPackage.entryPoint << "\n" + << fileTimeText(shaderPackage.manifestWriteTime) << "\n" + << fileTimeText(shaderPackage.shaderWriteTime) << "\n"; + for (const ShaderPassDefinition& pass : shaderPackage.passes) + { + source << "pass:" << pass.id << ":" << pass.entryPoint << ":" << pass.outputName << ":" + << fileTimeText(pass.sourceWriteTime) << "\n"; + for (const std::string& inputName : pass.inputNames) + source << "input:" << inputName << "\n"; + } + for (const ShaderFontAsset& font : shaderPackage.fontAssets) + source << "font:" << font.id << ":" << font.path.string() << ":" << fileTimeText(font.writeTime) << "\n"; + for (const ShaderParameterDefinition& parameter : shaderPackage.parameters) + { + source << "param:" << parameter.id << ":" << static_cast(parameter.type) << ":" + << parameter.label << ":" << parameter.description << ":" << parameter.fontId << ":" + << parameter.defaultTextValue << ":" << parameter.defaultBoolean << ":" + << parameter.defaultEnumValue << ":" << parameter.maxLength << "\n"; + for (double value : parameter.defaultNumbers) + source << "default:" << value << "\n"; + for (double value : parameter.minNumbers) + source << "min:" << value << "\n"; + for (double value : parameter.maxNumbers) + source << "max:" << value << "\n"; + for (const ShaderParameterOption& option : parameter.enumOptions) + source << "option:" << option.value << ":" << option.label << "\n"; + } + + const std::string text = source.str(); + return std::to_string(text.size()) + ":" + std::to_string(std::hash{}(text)); +} + bool SupportedShaderCatalog::Load(const std::filesystem::path& shaderRoot, unsigned maxTemporalHistoryFrames, std::string& error) { mShaders.clear(); diff --git a/src/runtime/SupportedShaderCatalog.h b/src/runtime/SupportedShaderCatalog.h index 95df5bd..e57d2a8 100644 --- a/src/runtime/SupportedShaderCatalog.h +++ b/src/runtime/SupportedShaderCatalog.h @@ -25,6 +25,7 @@ struct ShaderSupportResult }; ShaderSupportResult CheckStatelessSinglePassShaderSupport(const ShaderPackage& shaderPackage); +std::string ShaderPackageFingerprint(const ShaderPackage& shaderPackage); class SupportedShaderCatalog { diff --git a/tests/RenderCadenceCompositorHttpControlServerTests.cpp b/tests/RenderCadenceCompositorHttpControlServerTests.cpp index 7c6af72..e12faa7 100644 --- a/tests/RenderCadenceCompositorHttpControlServerTests.cpp +++ b/tests/RenderCadenceCompositorHttpControlServerTests.cpp @@ -1,4 +1,5 @@ #include "HttpControlServer.h" +#include "RuntimeControlCommand.h" #include #include @@ -170,6 +171,16 @@ void TestGenericPostCallbackHandlesControlRoutes() Expect(response.body.find("\"ok\":true") != std::string::npos, "generic control callback returns action success"); } +void TestReloadRouteParsesAsControlCommand() +{ + using namespace RenderCadenceCompositor; + + RuntimeControlCommand command; + std::string error; + Expect(ParseRuntimeControlCommand("/api/reload", "{}", command, error), "reload route parses as a control command"); + Expect(command.type == RuntimeControlCommandType::ReloadShaders, "reload route maps to reload command type"); +} + void TestUnknownEndpointReturns404() { using namespace RenderCadenceCompositor; @@ -193,6 +204,7 @@ int main() TestKnownPostEndpointReturnsActionError(); TestLayerPostEndpointsUseCallbacks(); TestGenericPostCallbackHandlesControlRoutes(); + TestReloadRouteParsesAsControlCommand(); TestUnknownEndpointReturns404(); if (gFailures != 0) diff --git a/tests/RenderCadenceCompositorRuntimeLayerModelTests.cpp b/tests/RenderCadenceCompositorRuntimeLayerModelTests.cpp index 55f3c75..0253cba 100644 --- a/tests/RenderCadenceCompositorRuntimeLayerModelTests.cpp +++ b/tests/RenderCadenceCompositorRuntimeLayerModelTests.cpp @@ -34,28 +34,42 @@ void WriteFile(const std::filesystem::path& path, const std::string& contents) output << contents; } -RenderCadenceCompositor::SupportedShaderCatalog MakeCatalog(std::filesystem::path& root) +std::string SolidShaderManifest(double gainDefault, bool includeMix) { - root = MakeTestRoot(); - WriteFile(root / "solid" / "shader.slang", "float4 shadeVideo(float2 uv) { return float4(uv, 0.0, 1.0); }\n"); - WriteFile(root / "solid" / "shader.json", R"({ + std::string parameters = + "{ \"id\": \"gain\", \"label\": \"Gain\", \"type\": \"float\", \"default\": " + std::to_string(gainDefault) + " },\n" + "\t\t\t{ \"id\": \"drop\", \"label\": \"Drop\", \"type\": \"trigger\" }"; + if (includeMix) + parameters += ",\n\t\t\t{ \"id\": \"mix\", \"label\": \"Mix\", \"type\": \"float\", \"default\": 0.25 }"; + + return R"({ "id": "solid", "name": "Solid", "description": "Solid test shader", "category": "Tests", "entryPoint": "shadeVideo", "parameters": [ - { "id": "gain", "label": "Gain", "type": "float", "default": 0.5 }, - { "id": "drop", "label": "Drop", "type": "trigger" } + )" + parameters + R"( ] - })"); + })"; +} +RenderCadenceCompositor::SupportedShaderCatalog LoadCatalog(const std::filesystem::path& root) +{ RenderCadenceCompositor::SupportedShaderCatalog catalog; std::string error; Expect(catalog.Load(root, 4, error), error.empty() ? "catalog loads test shader" : error); return catalog; } +RenderCadenceCompositor::SupportedShaderCatalog MakeCatalog(std::filesystem::path& root) +{ + root = MakeTestRoot(); + WriteFile(root / "solid" / "shader.slang", "float4 shadeVideo(float2 uv) { return float4(uv, 0.0, 1.0); }\n"); + WriteFile(root / "solid" / "shader.json", SolidShaderManifest(0.5, false)); + return LoadCatalog(root); +} + void TestSingleLayerLifecycle() { std::filesystem::path root; @@ -196,6 +210,31 @@ void TestLayerControlsUpdateDisplayAndRenderModels() std::filesystem::remove_all(root); } + +void TestReloadRefreshesChangedShaderMetadataAndPreservesValues() +{ + std::filesystem::path root; + RenderCadenceCompositor::SupportedShaderCatalog catalog = MakeCatalog(root); + + RenderCadenceCompositor::RuntimeLayerModel model; + std::string error; + std::string layerId; + Expect(model.AddLayer(catalog, "solid", layerId, error), "reload test layer can be added"); + Expect(model.UpdateParameter(layerId, "gain", JsonValue(0.75), error), "reload test parameter can be customized"); + + WriteFile(root / "solid" / "shader.json", SolidShaderManifest(0.1, true)); + RenderCadenceCompositor::SupportedShaderCatalog reloadedCatalog = LoadCatalog(root); + std::vector> buildsToStart; + Expect(model.ReloadFromCatalog(reloadedCatalog, buildsToStart, error), "reload refreshes model from changed catalog"); + Expect(buildsToStart.size() == 1 && buildsToStart[0].first == layerId && buildsToStart[0].second == "solid", "changed shader queues a layer rebuild"); + + RenderCadenceCompositor::RuntimeLayerModelSnapshot snapshot = model.Snapshot(); + Expect(snapshot.displayLayers[0].parameterDefinitions.size() == 3, "reload exposes new JSON parameter definitions to UI"); + Expect(snapshot.displayLayers[0].parameterValues.at("gain").numberValues.front() == 0.75, "reload preserves compatible parameter values"); + Expect(snapshot.displayLayers[0].parameterValues.at("mix").numberValues.front() == 0.25, "reload initializes newly added parameters"); + + std::filesystem::remove_all(root); +} } int main() @@ -205,6 +244,7 @@ int main() TestBuildFailureStaysDisplaySide(); TestAddAndRemoveLayers(); TestLayerControlsUpdateDisplayAndRenderModels(); + TestReloadRefreshesChangedShaderMetadataAndPreservesValues(); if (gFailures != 0) {