Hot reload

src/app/RuntimeLayerController.cpp

@@ -21,6 +21,8 @@ void RuntimeLayerController::SetPublisher(RenderLayerPublisher publisher)
 
 void RuntimeLayerController::Initialize(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames, std::string& runtimeShaderId)
 {
+	mShaderLibrary = shaderLibrary;
+	mMaxTemporalHistoryFrames = maxTemporalHistoryFrames;
 	LoadSupportedShaderCatalog(shaderLibrary, maxTemporalHistoryFrames);
 	InitializeLayerModel(runtimeShaderId);
 }

src/app/RuntimeLayerController.h

@@ -39,7 +39,7 @@ public:
 	const SupportedShaderCatalog& ShaderCatalog() const { return mShaderCatalog; }
 
 private:
-	void LoadSupportedShaderCatalog(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames);
+	bool LoadSupportedShaderCatalog(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames);
 	void InitializeLayerModel(std::string& runtimeShaderId);
 	void StartLayerShaderBuild(const std::string& layerId, const std::string& shaderId);
 	void RetireLayerShaderBuild(const std::string& layerId);
@@ -53,6 +53,8 @@ private:
 	static bool ExtractStringField(const std::string& body, const char* fieldName, std::string& value, std::string& error);
 
 	RenderLayerPublisher mPublisher;
+	std::string mShaderLibrary;
+	unsigned mMaxTemporalHistoryFrames = 0;
 	SupportedShaderCatalog mShaderCatalog;
 	mutable std::mutex mRuntimeLayerMutex;
 	RuntimeLayerModel mRuntimeLayerModel;

src/app/RuntimeLayerControllerBuild.cpp

@@ -7,14 +7,14 @@
 
 namespace RenderCadenceCompositor
 {
-void RuntimeLayerController::LoadSupportedShaderCatalog(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames)
+bool RuntimeLayerController::LoadSupportedShaderCatalog(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames)
 {
 	const std::filesystem::path shaderRoot = FindRepoPath(shaderLibrary);
 	std::string error;
 	if (!mShaderCatalog.Load(shaderRoot, maxTemporalHistoryFrames, error))
 	{
 		LogWarning("runtime-shader", "Supported shader catalog is empty: " + error);
-		return;
+		return false;
 	}
 
 	std::size_t preparedFontAtlases = 0;
@@ -25,6 +25,7 @@ void RuntimeLayerController::LoadSupportedShaderCatalog(const std::string& shade
 		"runtime-shader",
 		"Supported shader catalog loaded with " + std::to_string(mShaderCatalog.Shaders().size()) +
 			" shader(s), prepared " + std::to_string(preparedFontAtlases) + " font atlas asset(s).");
+	return true;
 }
 
 void RuntimeLayerController::InitializeLayerModel(std::string& runtimeShaderId)

src/app/RuntimeLayerControllerControls.cpp

@@ -129,6 +129,26 @@ ControlActionResult RuntimeLayerController::HandleControlCommand(const RuntimeCo
 		PublishRuntimeRenderLayers();
 		return { true, std::string() };
 	}
+	case RuntimeControlCommandType::ReloadShaders:
+	{
+		if (!LoadSupportedShaderCatalog(mShaderLibrary, mMaxTemporalHistoryFrames))
+			return { false, "Shader reload failed; see logs for details." };
+
+		std::vector<std::pair<std::string, std::string>> buildsToStart;
+		{
+			std::lock_guard<std::mutex> lock(mRuntimeLayerMutex);
+			if (!mRuntimeLayerModel.ReloadFromCatalog(mShaderCatalog, buildsToStart, error))
+				return { false, error };
+		}
+
+		for (const auto& build : buildsToStart)
+		{
+			Log("runtime-shader", "Reload queued shader rebuild: " + build.first + " shader=" + build.second);
+			StartLayerShaderBuild(build.first, build.second);
+		}
+		PublishRuntimeRenderLayers();
+		return { true, std::string() };
+	}
 	case RuntimeControlCommandType::Unsupported:
 		break;
 	}

src/control/RuntimeControlCommand.cpp

@@ -119,6 +119,11 @@ bool ParseRuntimeControlCommand(
 		command.type = RuntimeControlCommandType::ResetLayerParameters;
 		return RequireStringField(root, "layerId", command.layerId, error);
 	}
+	if (path == "/api/reload")
+	{
+		command.type = RuntimeControlCommandType::ReloadShaders;
+		return true;
+	}
 
 	command.type = RuntimeControlCommandType::Unsupported;
 	error = "Endpoint is not implemented in RenderCadenceCompositor yet.";

src/control/RuntimeControlCommand.h

@@ -15,6 +15,7 @@ enum class RuntimeControlCommandType
 	SetLayerShader,
 	UpdateLayerParameter,
 	ResetLayerParameters,
+	ReloadShaders,
 	Unsupported
 };
 

src/runtime/RuntimeLayerModel.cpp

@@ -8,6 +8,35 @@
 
 namespace RenderCadenceCompositor
 {
+namespace
+{
+JsonValue ParameterValueToJson(const ShaderParameterDefinition& definition, const ShaderParameterValue& value)
+{
+	switch (definition.type)
+	{
+	case ShaderParameterType::Float:
+		return JsonValue(value.numberValues.empty() ? 0.0 : value.numberValues.front());
+	case ShaderParameterType::Vec2:
+	case ShaderParameterType::Color:
+	{
+		JsonValue array = JsonValue::MakeArray();
+		for (double number : value.numberValues)
+			array.pushBack(JsonValue(number));
+		return array;
+	}
+	case ShaderParameterType::Boolean:
+		return JsonValue(value.booleanValue);
+	case ShaderParameterType::Enum:
+		return JsonValue(value.enumValue);
+	case ShaderParameterType::Text:
+		return JsonValue(value.textValue);
+	case ShaderParameterType::Trigger:
+		return JsonValue(value.numberValues.empty() ? 0.0 : value.numberValues.front());
+	}
+	return JsonValue();
+}
+}
+
 bool RuntimeLayerModel::InitializeSingleLayer(const SupportedShaderCatalog& shaderCatalog, const std::string& shaderId, std::string& error)
 {
 	Clear();
@@ -27,6 +56,7 @@ bool RuntimeLayerModel::InitializeSingleLayer(const SupportedShaderCatalog& shad
 	Layer layer;
 	layer.id = AllocateLayerId();
 	layer.shaderId = shaderPackage->id;
+	layer.packageFingerprint = ShaderPackageFingerprint(*shaderPackage);
 	layer.shaderName = shaderPackage->displayName.empty() ? shaderPackage->id : shaderPackage->displayName;
 	layer.buildState = RuntimeLayerBuildState::Pending;
 	layer.message = "Runtime Slang build is waiting to start.";
@@ -48,6 +78,7 @@ bool RuntimeLayerModel::AddLayer(const SupportedShaderCatalog& shaderCatalog, co
 	Layer layer;
 	layer.id = AllocateLayerId();
 	layer.shaderId = shaderPackage->id;
+	layer.packageFingerprint = ShaderPackageFingerprint(*shaderPackage);
 	layer.shaderName = shaderPackage->displayName.empty() ? shaderPackage->id : shaderPackage->displayName;
 	layer.buildState = RuntimeLayerBuildState::Pending;
 	layer.message = "Runtime Slang build is waiting to start.";
@@ -130,6 +161,7 @@ bool RuntimeLayerModel::SetLayerShader(const SupportedShaderCatalog& shaderCatal
 	}
 
 	layer->shaderId = shaderPackage->id;
+	layer->packageFingerprint = ShaderPackageFingerprint(*shaderPackage);
 	layer->shaderName = shaderPackage->displayName.empty() ? shaderPackage->id : shaderPackage->displayName;
 	layer->buildState = RuntimeLayerBuildState::Pending;
 	layer->message = "Runtime Slang build is waiting to start.";
@@ -195,6 +227,61 @@ bool RuntimeLayerModel::ResetParameters(const std::string& layerId, std::string&
 	return true;
 }
 
+bool RuntimeLayerModel::ReloadFromCatalog(const SupportedShaderCatalog& shaderCatalog, std::vector<std::pair<std::string, std::string>>& buildsToStart, std::string& error)
+{
+	buildsToStart.clear();
+	for (Layer& layer : mLayers)
+	{
+		const ShaderPackage* shaderPackage = shaderCatalog.FindPackage(layer.shaderId);
+		if (!shaderPackage)
+		{
+			layer.buildState = RuntimeLayerBuildState::Failed;
+			layer.message = "Shader '" + layer.shaderId + "' is no longer available after reload.";
+			continue;
+		}
+
+		const std::string nextFingerprint = ShaderPackageFingerprint(*shaderPackage);
+		if (layer.packageFingerprint == nextFingerprint)
+			continue;
+
+		std::map<std::string, ShaderParameterDefinition> previousDefinitions;
+		for (const ShaderParameterDefinition& definition : layer.parameterDefinitions)
+			previousDefinitions[definition.id] = definition;
+
+		std::map<std::string, ShaderParameterValue> nextValues;
+		for (const ShaderParameterDefinition& nextDefinition : shaderPackage->parameters)
+		{
+			const auto previousDefinitionIt = previousDefinitions.find(nextDefinition.id);
+			const auto previousValueIt = layer.parameterValues.find(nextDefinition.id);
+			if (previousDefinitionIt != previousDefinitions.end()
+				&& previousValueIt != layer.parameterValues.end()
+				&& previousDefinitionIt->second.type == nextDefinition.type)
+			{
+				ShaderParameterValue preservedValue;
+				JsonValue valueJson = ParameterValueToJson(previousDefinitionIt->second, previousValueIt->second);
+				std::string normalizeError;
+				if (NormalizeAndValidateParameterValue(nextDefinition, valueJson, preservedValue, normalizeError))
+				{
+					nextValues[nextDefinition.id] = preservedValue;
+					continue;
+				}
+			}
+			nextValues[nextDefinition.id] = DefaultValueForDefinition(nextDefinition);
+		}
+
+		layer.shaderName = shaderPackage->displayName.empty() ? shaderPackage->id : shaderPackage->displayName;
+		layer.packageFingerprint = nextFingerprint;
+		layer.parameterDefinitions = shaderPackage->parameters;
+		layer.parameterValues = std::move(nextValues);
+		if (layer.renderReady)
+			layer.artifact.parameterValues = layer.parameterValues;
+		buildsToStart.push_back({ layer.id, layer.shaderId });
+	}
+
+	error.clear();
+	return true;
+}
+
 void RuntimeLayerModel::Clear()
 {
 	mLayers.clear();
@@ -229,6 +316,7 @@ bool RuntimeLayerModel::MarkBuildReady(const RuntimeShaderArtifact& artifact, st
 	}
 
 	layer->shaderName = artifact.displayName.empty() ? artifact.shaderId : artifact.displayName;
+	layer->packageFingerprint = artifact.packageFingerprint;
 	layer->buildState = RuntimeLayerBuildState::Ready;
 	layer->message = artifact.message;
 	layer->renderReady = true;

src/runtime/RuntimeLayerModel.h

@@ -7,6 +7,7 @@
 #include <chrono>
 #include <cstdint>
 #include <map>
+#include <utility>
 #include <string>
 #include <vector>
 
@@ -61,6 +62,7 @@ public:
 	bool SetLayerShader(const SupportedShaderCatalog& shaderCatalog, const std::string& layerId, const std::string& shaderId, std::string& error);
 	bool UpdateParameter(const std::string& layerId, const std::string& parameterId, const JsonValue& value, std::string& error);
 	bool ResetParameters(const std::string& layerId, std::string& error);
+	bool ReloadFromCatalog(const SupportedShaderCatalog& shaderCatalog, std::vector<std::pair<std::string, std::string>>& buildsToStart, std::string& error);
 	bool MarkBuildStarted(const std::string& layerId, const std::string& message, std::string& error);
 	bool MarkBuildReady(const RuntimeShaderArtifact& artifact, std::string& error);
 	bool MarkBuildFailedForShader(const std::string& shaderId, const std::string& message);
@@ -75,6 +77,7 @@ private:
 	{
 		std::string id;
 		std::string shaderId;
+		std::string packageFingerprint;
 		std::string shaderName;
 		bool bypass = false;
 		RuntimeLayerBuildState buildState = RuntimeLayerBuildState::Pending;

src/runtime/RuntimeShaderArtifact.h

@@ -25,6 +25,7 @@ struct RuntimeShaderArtifact
 {
 	std::string layerId;
 	std::string shaderId;
+	std::string packageFingerprint;
 	std::string displayName;
 	std::string fragmentShaderSource;
 	std::vector<RuntimeShaderPassArtifact> passes;

src/runtime/RuntimeSlangShaderCompiler.cpp

@@ -153,6 +153,7 @@ RuntimeSlangShaderBuild RuntimeSlangShaderCompiler::BuildShader(const std::strin
 		const double milliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(end - start).count();
 		build.succeeded = true;
 		build.artifact.shaderId = shaderPackage.id;
+		build.artifact.packageFingerprint = RenderCadenceCompositor::ShaderPackageFingerprint(shaderPackage);
 		build.artifact.displayName = shaderPackage.displayName;
 		build.artifact.parameterDefinitions = shaderPackage.parameters;
 		build.artifact.fontAtlases = std::move(fontAtlasOutputs);

src/runtime/SupportedShaderCatalog.cpp

@@ -3,6 +3,7 @@
 #include "ShaderPackageRegistry.h"
 
 #include <map>
+#include <sstream>
 #include <utility>
 
 namespace RenderCadenceCompositor
@@ -74,6 +75,49 @@ ShaderSupportResult CheckStatelessSinglePassShaderSupport(const ShaderPackage& s
 	return { true, std::string() };
 }
 
+std::string ShaderPackageFingerprint(const ShaderPackage& shaderPackage)
+{
+	const auto fileTimeText = [](std::filesystem::file_time_type value) {
+		return std::to_string(value.time_since_epoch().count());
+	};
+
+	std::ostringstream source;
+	source << shaderPackage.id << "\n"
+		<< shaderPackage.displayName << "\n"
+		<< shaderPackage.description << "\n"
+		<< shaderPackage.category << "\n"
+		<< shaderPackage.entryPoint << "\n"
+		<< fileTimeText(shaderPackage.manifestWriteTime) << "\n"
+		<< fileTimeText(shaderPackage.shaderWriteTime) << "\n";
+	for (const ShaderPassDefinition& pass : shaderPackage.passes)
+	{
+		source << "pass:" << pass.id << ":" << pass.entryPoint << ":" << pass.outputName << ":"
+			<< fileTimeText(pass.sourceWriteTime) << "\n";
+		for (const std::string& inputName : pass.inputNames)
+			source << "input:" << inputName << "\n";
+	}
+	for (const ShaderFontAsset& font : shaderPackage.fontAssets)
+		source << "font:" << font.id << ":" << font.path.string() << ":" << fileTimeText(font.writeTime) << "\n";
+	for (const ShaderParameterDefinition& parameter : shaderPackage.parameters)
+	{
+		source << "param:" << parameter.id << ":" << static_cast<int>(parameter.type) << ":"
+			<< parameter.label << ":" << parameter.description << ":" << parameter.fontId << ":"
+			<< parameter.defaultTextValue << ":" << parameter.defaultBoolean << ":"
+			<< parameter.defaultEnumValue << ":" << parameter.maxLength << "\n";
+		for (double value : parameter.defaultNumbers)
+			source << "default:" << value << "\n";
+		for (double value : parameter.minNumbers)
+			source << "min:" << value << "\n";
+		for (double value : parameter.maxNumbers)
+			source << "max:" << value << "\n";
+		for (const ShaderParameterOption& option : parameter.enumOptions)
+			source << "option:" << option.value << ":" << option.label << "\n";
+	}
+
+	const std::string text = source.str();
+	return std::to_string(text.size()) + ":" + std::to_string(std::hash<std::string>{}(text));
+}
+
 bool SupportedShaderCatalog::Load(const std::filesystem::path& shaderRoot, unsigned maxTemporalHistoryFrames, std::string& error)
 {
 	mShaders.clear();

src/runtime/SupportedShaderCatalog.h

@@ -25,6 +25,7 @@ struct ShaderSupportResult
 };
 
 ShaderSupportResult CheckStatelessSinglePassShaderSupport(const ShaderPackage& shaderPackage);
+std::string ShaderPackageFingerprint(const ShaderPackage& shaderPackage);
 
 class SupportedShaderCatalog
 {

tests/RenderCadenceCompositorHttpControlServerTests.cpp

@@ -1,4 +1,5 @@
 #include "HttpControlServer.h"
+#include "RuntimeControlCommand.h"
 
 #include <filesystem>
 #include <fstream>
@@ -170,6 +171,16 @@ void TestGenericPostCallbackHandlesControlRoutes()
 	Expect(response.body.find("\"ok\":true") != std::string::npos, "generic control callback returns action success");
 }
 
+void TestReloadRouteParsesAsControlCommand()
+{
+	using namespace RenderCadenceCompositor;
+
+	RuntimeControlCommand command;
+	std::string error;
+	Expect(ParseRuntimeControlCommand("/api/reload", "{}", command, error), "reload route parses as a control command");
+	Expect(command.type == RuntimeControlCommandType::ReloadShaders, "reload route maps to reload command type");
+}
+
 void TestUnknownEndpointReturns404()
 {
 	using namespace RenderCadenceCompositor;
@@ -193,6 +204,7 @@ int main()
 	TestKnownPostEndpointReturnsActionError();
 	TestLayerPostEndpointsUseCallbacks();
 	TestGenericPostCallbackHandlesControlRoutes();
+	TestReloadRouteParsesAsControlCommand();
 	TestUnknownEndpointReturns404();
 
 	if (gFailures != 0)

tests/RenderCadenceCompositorRuntimeLayerModelTests.cpp

@@ -34,28 +34,42 @@ void WriteFile(const std::filesystem::path& path, const std::string& contents)
 	output << contents;
 }
 
-RenderCadenceCompositor::SupportedShaderCatalog MakeCatalog(std::filesystem::path& root)
+std::string SolidShaderManifest(double gainDefault, bool includeMix)
 {
-	root = MakeTestRoot();
+	std::string parameters =
-	WriteFile(root / "solid" / "shader.slang", "float4 shadeVideo(float2 uv) { return float4(uv, 0.0, 1.0); }\n");
+		"{ \"id\": \"gain\", \"label\": \"Gain\", \"type\": \"float\", \"default\": " + std::to_string(gainDefault) + " },\n"
-	WriteFile(root / "solid" / "shader.json", R"({
+		"\t\t\t{ \"id\": \"drop\", \"label\": \"Drop\", \"type\": \"trigger\" }";
+	if (includeMix)
+		parameters += ",\n\t\t\t{ \"id\": \"mix\", \"label\": \"Mix\", \"type\": \"float\", \"default\": 0.25 }";
+
+	return R"({
 		"id": "solid",
 		"name": "Solid",
 		"description": "Solid test shader",
 		"category": "Tests",
 		"entryPoint": "shadeVideo",
 		"parameters": [
-			{ "id": "gain", "label": "Gain", "type": "float", "default": 0.5 },
+			)" + parameters + R"(
-			{ "id": "drop", "label": "Drop", "type": "trigger" }
 		]
-	})");
+	})";
+}
 
+RenderCadenceCompositor::SupportedShaderCatalog LoadCatalog(const std::filesystem::path& root)
+{
 	RenderCadenceCompositor::SupportedShaderCatalog catalog;
 	std::string error;
 	Expect(catalog.Load(root, 4, error), error.empty() ? "catalog loads test shader" : error);
 	return catalog;
 }
 
+RenderCadenceCompositor::SupportedShaderCatalog MakeCatalog(std::filesystem::path& root)
+{
+	root = MakeTestRoot();
+	WriteFile(root / "solid" / "shader.slang", "float4 shadeVideo(float2 uv) { return float4(uv, 0.0, 1.0); }\n");
+	WriteFile(root / "solid" / "shader.json", SolidShaderManifest(0.5, false));
+	return LoadCatalog(root);
+}
+
 void TestSingleLayerLifecycle()
 {
 	std::filesystem::path root;
@@ -196,6 +210,31 @@ void TestLayerControlsUpdateDisplayAndRenderModels()
 
 	std::filesystem::remove_all(root);
 }
+
+void TestReloadRefreshesChangedShaderMetadataAndPreservesValues()
+{
+	std::filesystem::path root;
+	RenderCadenceCompositor::SupportedShaderCatalog catalog = MakeCatalog(root);
+
+	RenderCadenceCompositor::RuntimeLayerModel model;
+	std::string error;
+	std::string layerId;
+	Expect(model.AddLayer(catalog, "solid", layerId, error), "reload test layer can be added");
+	Expect(model.UpdateParameter(layerId, "gain", JsonValue(0.75), error), "reload test parameter can be customized");
+
+	WriteFile(root / "solid" / "shader.json", SolidShaderManifest(0.1, true));
+	RenderCadenceCompositor::SupportedShaderCatalog reloadedCatalog = LoadCatalog(root);
+	std::vector<std::pair<std::string, std::string>> buildsToStart;
+	Expect(model.ReloadFromCatalog(reloadedCatalog, buildsToStart, error), "reload refreshes model from changed catalog");
+	Expect(buildsToStart.size() == 1 && buildsToStart[0].first == layerId && buildsToStart[0].second == "solid", "changed shader queues a layer rebuild");
+
+	RenderCadenceCompositor::RuntimeLayerModelSnapshot snapshot = model.Snapshot();
+	Expect(snapshot.displayLayers[0].parameterDefinitions.size() == 3, "reload exposes new JSON parameter definitions to UI");
+	Expect(snapshot.displayLayers[0].parameterValues.at("gain").numberValues.front() == 0.75, "reload preserves compatible parameter values");
+	Expect(snapshot.displayLayers[0].parameterValues.at("mix").numberValues.front() == 0.25, "reload initializes newly added parameters");
+
+	std::filesystem::remove_all(root);
+}
 }
 
 int main()
@@ -205,6 +244,7 @@ int main()
 	TestBuildFailureStaysDisplaySide();
 	TestAddAndRemoveLayers();
 	TestLayerControlsUpdateDisplayAndRenderModels();
+	TestReloadRefreshesChangedShaderMetadataAndPreservesValues();
 
 	if (gFailures != 0)
 	{