compute orchestration (#763)

- Added local process runners for Linux/Wine, Mac with some sandboxing support
- Horde & Nomad provisioning for development and testing
- Client session queues with lifecycle management (active/draining/cancelled), automatic retry with configurable limits, and manual reschedule API
- Improved web UI for orchestrator, compute, and hub dashboards with WebSocket push updates
- Some security hardening
- Improved scalability and `zen exec` command

Still experimental - compute support is disabled by default

1 files changed, 97 insertions, 14 deletions

diff --git a/src/zenserver/compute/computeserver.h b/src/zenserver/compute/computeserver.h
index 625140b23..e4a6b01d5 100644
--- a/src/zenserver/compute/computeserver.h
+++ b/src/zenserver/compute/computeserver.h
@@ -6,7 +6,11 @@
 
 #if ZEN_WITH_COMPUTE_SERVICES
 
+#	include <future>
+#	include <zencore/system.h>
+#	include <zenhttp/httpwsclient.h>
 #	include <zenstore/gc.h>
+#	include "frontend/frontend.h"
 
 namespace cxxopts {
 class Options;
@@ -16,19 +20,46 @@ struct Options;
 }
 
 namespace zen::compute {
-class HttpFunctionService;
-}
+class CloudMetadata;
+class HttpComputeService;
+class HttpOrchestratorService;
+}  // namespace zen::compute
+
+#	if ZEN_WITH_HORDE
+#		include <zenhorde/hordeconfig.h>
+namespace zen::horde {
+class HordeProvisioner;
+}  // namespace zen::horde
+#	endif
+
+#	if ZEN_WITH_NOMAD
+#		include <zennomad/nomadconfig.h>
+namespace zen::nomad {
+class NomadProvisioner;
+}  // namespace zen::nomad
+#	endif
 
 namespace zen {
 
 class CidStore;
 class HttpApiService;
-class HttpComputeService;
 
 struct ZenComputeServerConfig : public ZenServerConfig
 {
 	std::string UpstreamNotificationEndpoint;
 	std::string InstanceId;	 // For use in notifications
+	std::string CoordinatorEndpoint;
+	std::string IdmsEndpoint;
+	int32_t		MaxConcurrentActions  = 0;		// 0 = auto (LogicalProcessorCount * 2)
+	bool		EnableWorkerWebSocket = false;	// Use WebSocket for worker↔orchestrator link
+
+#	if ZEN_WITH_HORDE
+	horde::HordeConfig HordeConfig;
+#	endif
+
+#	if ZEN_WITH_NOMAD
+	nomad::NomadConfig NomadConfig;
+#	endif
 };
 
 struct ZenComputeServerConfigurator : public ZenServerConfiguratorBase
@@ -49,6 +80,16 @@ private:
 	virtual void ValidateOptions() override;
 
 	ZenComputeServerConfig& m_ServerOptions;
+
+#	if ZEN_WITH_HORDE
+	std::string m_HordeModeStr		 = "direct";
+	std::string m_HordeEncryptionStr = "none";
+#	endif
+
+#	if ZEN_WITH_NOMAD
+	std::string m_NomadDriverStr	   = "raw_exec";
+	std::string m_NomadDistributionStr = "predeployed";
+#	endif
 };
 
 class ZenComputeServerMain : public ZenServerMain
@@ -88,17 +129,59 @@ public:
 	void Cleanup();
 
 private:
-	HttpStatsService								   m_StatsService;
-	GcManager										   m_GcManager;
-	GcScheduler										   m_GcScheduler{m_GcManager};
-	std::unique_ptr<CidStore>						   m_CidStore;
-	std::unique_ptr<HttpComputeService>				   m_ComputeService;
-	std::unique_ptr<HttpApiService>					   m_ApiService;
-	std::unique_ptr<zen::compute::HttpFunctionService> m_FunctionService;
-
-	void InitializeState(const ZenComputeServerConfig& ServerConfig);
-	void InitializeServices(const ZenComputeServerConfig& ServerConfig);
-	void RegisterServices(const ZenComputeServerConfig& ServerConfig);
+	HttpStatsService										  m_StatsService;
+	GcManager												  m_GcManager;
+	GcScheduler												  m_GcScheduler{m_GcManager};
+	std::unique_ptr<CidStore>								  m_CidStore;
+	std::unique_ptr<HttpApiService>							  m_ApiService;
+	std::unique_ptr<zen::compute::HttpComputeService>		  m_ComputeService;
+	std::unique_ptr<zen::compute::HttpOrchestratorService>	  m_OrchestratorService;
+	std::unique_ptr<zen::compute::CloudMetadata>			  m_CloudMetadata;
+	std::future<std::unique_ptr<zen::compute::CloudMetadata>> m_CloudMetadataFuture;
+	std::unique_ptr<HttpFrontendService>					  m_FrontendService;
+#	if ZEN_WITH_HORDE
+	std::unique_ptr<zen::horde::HordeProvisioner> m_HordeProvisioner;
+#	endif
+#	if ZEN_WITH_NOMAD
+	std::unique_ptr<zen::nomad::NomadProvisioner> m_NomadProvisioner;
+#	endif
+	SystemMetricsTracker m_MetricsTracker;
+	std::string			 m_CoordinatorEndpoint;
+	std::string			 m_InstanceId;
+
+	asio::steady_timer m_AnnounceTimer{m_IoContext};
+	asio::steady_timer m_ProvisionerMaintenanceTimer{m_IoContext};
+
+	void		InitializeState(const ZenComputeServerConfig& ServerConfig);
+	void		InitializeServices(const ZenComputeServerConfig& ServerConfig);
+	void		RegisterServices(const ZenComputeServerConfig& ServerConfig);
+	void		ResolveCloudMetadata();
+	void		PostAnnounce();
+	void		EnqueueAnnounceTimer();
+	void		EnqueueProvisionerMaintenanceTimer();
+	void		ProvisionerMaintenanceTick();
+	std::string GetAnnounceUrl() const;
+	std::string GetInstanceId() const;
+	CbObject	BuildAnnounceBody();
+
+	// Worker→orchestrator WebSocket client
+	struct OrchestratorWsHandler : public IWsClientHandler
+	{
+		ZenComputeServer& Server;
+		explicit OrchestratorWsHandler(ZenComputeServer& S) : Server(S) {}
+
+		void OnWsOpen() override;
+		void OnWsMessage(const WebSocketMessage& Msg) override;
+		void OnWsClose(uint16_t Code, std::string_view Reason) override;
+	};
+
+	std::unique_ptr<OrchestratorWsHandler> m_OrchestratorWsHandler;
+	std::unique_ptr<HttpWsClient>		   m_OrchestratorWsClient;
+	asio::steady_timer					   m_WsReconnectTimer{m_IoContext};
+	bool								   m_EnableWorkerWebSocket = false;
+
+	void InitializeOrchestratorWebSocket();
+	void EnqueueWsReconnect();
 };
 
 }  // namespace zen