reload has a bounded number of shader compilers

src/app/RuntimeLayerController.h

@@ -8,6 +8,7 @@
 #include "../runtime/SupportedShaderCatalog.h"
 #include "../telemetry/CadenceTelemetry.h"
 
+#include <deque>
 #include <functional>
 #include <filesystem>
 #include <map>
@@ -48,6 +49,9 @@ private:
 	void RequestRuntimeStatePersistenceLocked();
 	std::filesystem::path ResolveRuntimeStatePath() const;
 	void StartLayerShaderBuild(const std::string& layerId, const std::string& shaderId, bool preserveExistingRenderArtifact = false);
+	void PumpShaderBuildQueue();
+	void StartQueuedShaderBuild(const std::string& layerId, const std::string& shaderId);
+	void CompleteLayerShaderBuild(const std::string& layerId);
 	void RetireLayerShaderBuild(const std::string& layerId);
 	void CleanupRetiredShaderBuilds();
 	void StopAllRuntimeShaderBuilds();
@@ -67,6 +71,12 @@ private:
 	mutable std::mutex mRuntimeLayerMutex;
 	RuntimeLayerModel mRuntimeLayerModel;
 	std::mutex mShaderBuildMutex;
+	struct QueuedShaderBuild
+	{
+		std::string layerId;
+		std::string shaderId;
+	};
+	std::deque<QueuedShaderBuild> mQueuedShaderBuilds;
 	std::map<std::string, std::unique_ptr<RuntimeShaderBridge>> mShaderBuilds;
 	std::vector<std::unique_ptr<RuntimeShaderBridge>> mRetiredShaderBuilds;
 };

src/app/RuntimeLayerControllerBuild.cpp

@@ -7,9 +7,15 @@
 #include <filesystem>
 #include <fstream>
 #include <sstream>
+#include <algorithm>
 
 namespace RenderCadenceCompositor
 {
+namespace
+{
+constexpr std::size_t kMaxConcurrentRuntimeShaderBuilds = 2;
+}
+
 bool RuntimeLayerController::LoadSupportedShaderCatalog(const std::string& shaderLibrary, unsigned maxTemporalHistoryFrames)
 {
 	const std::filesystem::path shaderRoot = FindRepoPath(shaderLibrary);
@@ -118,31 +124,91 @@ void RuntimeLayerController::StartLayerShaderBuild(const std::string& layerId, c
 		mRuntimeLayerModel.MarkBuildStarted(layerId, "Runtime Slang build started for shader '" + shaderId + "'.", error, preserveExistingRenderArtifact);
 	}
 
+	{
+		std::lock_guard<std::mutex> lock(mShaderBuildMutex);
+		mQueuedShaderBuilds.push_back(QueuedShaderBuild{ layerId, shaderId });
+	}
+	PumpShaderBuildQueue();
+}
+
+void RuntimeLayerController::PumpShaderBuildQueue()
+{
+	for (;;)
+	{
+		QueuedShaderBuild build;
+		{
+			std::lock_guard<std::mutex> lock(mShaderBuildMutex);
+			if (mShaderBuilds.size() >= kMaxConcurrentRuntimeShaderBuilds || mQueuedShaderBuilds.empty())
+				return;
+			build = mQueuedShaderBuilds.front();
+			mQueuedShaderBuilds.pop_front();
+			if (mShaderBuilds.find(build.layerId) != mShaderBuilds.end())
+				continue;
+		}
+
+		StartQueuedShaderBuild(build.layerId, build.shaderId);
+	}
+}
+
+void RuntimeLayerController::StartQueuedShaderBuild(const std::string& layerId, const std::string& shaderId)
+{
 	auto bridge = std::make_unique<RuntimeShaderBridge>();
 	RuntimeShaderBridge* bridgePtr = bridge.get();
 	{
 		std::lock_guard<std::mutex> lock(mShaderBuildMutex);
+		if (mShaderBuilds.size() >= kMaxConcurrentRuntimeShaderBuilds || mShaderBuilds.find(layerId) != mShaderBuilds.end())
+		{
+			mQueuedShaderBuilds.push_front(QueuedShaderBuild{ layerId, shaderId });
+			return;
+		}
 		mShaderBuilds[layerId] = std::move(bridge);
 	}
 
 	bridgePtr->Start(
 		layerId,
 		shaderId,
-		[this](const RuntimeShaderArtifact& artifact) {
+		[this, layerId](const RuntimeShaderArtifact& artifact) {
 			if (MarkRuntimeBuildReady(artifact))
 				PublishRuntimeRenderLayers();
+			CompleteLayerShaderBuild(layerId);
 		},
 		[this, layerId](const std::string& message) {
 			MarkRuntimeBuildFailedForLayer(layerId, message);
 			LogError("runtime-shader", "Runtime Slang build failed: " + message);
+			CompleteLayerShaderBuild(layerId);
 		});
 }
 
+void RuntimeLayerController::CompleteLayerShaderBuild(const std::string& layerId)
+{
+	std::unique_ptr<RuntimeShaderBridge> bridge;
+	{
+		std::lock_guard<std::mutex> lock(mShaderBuildMutex);
+		auto bridgeIt = mShaderBuilds.find(layerId);
+		if (bridgeIt != mShaderBuilds.end())
+		{
+			bridge = std::move(bridgeIt->second);
+			mShaderBuilds.erase(bridgeIt);
+			mRetiredShaderBuilds.push_back(std::move(bridge));
+		}
+	}
+	PumpShaderBuildQueue();
+}
+
 void RuntimeLayerController::RetireLayerShaderBuild(const std::string& layerId)
 {
 	std::unique_ptr<RuntimeShaderBridge> bridge;
 	{
 		std::lock_guard<std::mutex> lock(mShaderBuildMutex);
+		mQueuedShaderBuilds.erase(
+			std::remove_if(
+				mQueuedShaderBuilds.begin(),
+				mQueuedShaderBuilds.end(),
+				[&layerId](const QueuedShaderBuild& build) {
+					return build.layerId == layerId;
+				}),
+			mQueuedShaderBuilds.end());
+
 		auto bridgeIt = mShaderBuilds.find(layerId);
 		if (bridgeIt == mShaderBuilds.end())
 			return;
@@ -180,6 +246,7 @@ void RuntimeLayerController::StopAllRuntimeShaderBuilds()
 	std::vector<std::unique_ptr<RuntimeShaderBridge>> retiredBuilds;
 	{
 		std::lock_guard<std::mutex> lock(mShaderBuildMutex);
+		mQueuedShaderBuilds.clear();
 		builds.swap(mShaderBuilds);
 		retiredBuilds.swap(mRetiredShaderBuilds);
 	}