compute orchestration (#763)

- Added local process runners for Linux/Wine, Mac with some sandboxing support
- Horde & Nomad provisioning for development and testing
- Client session queues with lifecycle management (active/draining/cancelled), automatic retry with configurable limits, and manual reschedule API
- Improved web UI for orchestrator, compute, and hub dashboards with WebSocket push updates
- Some security hardening
- Improved scalability and `zen exec` command

Still experimental - compute support is disabled by default

1 files changed, 257 insertions, 79 deletions

diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp
index 267c87e12..833d3c04b 100644
--- a/src/zencore/system.cpp
+++ b/src/zencore/system.cpp
@@ -7,6 +7,8 @@
 #include <zencore/memory/memory.h>
 #include <zencore/string.h>
 
+#include <mutex>
+
 #if ZEN_PLATFORM_WINDOWS
 #	include <zencore/windows.h>
 
@@ -133,33 +135,6 @@ GetSystemMetrics()
 		Metrics.AvailPageFileMiB	  = MemStatus.ullAvailPageFile / 1024 / 1024;
 	}
 
-	// Query CPU usage using PDH
-	//
-	// TODO: This should be changed to not require a Sleep, perhaps by using some
-	// background metrics gathering mechanism.
-
-	{
-		PDH_HQUERY	 QueryHandle   = nullptr;
-		PDH_HCOUNTER CounterHandle = nullptr;
-
-		if (PdhOpenQueryW(nullptr, 0, &QueryHandle) == ERROR_SUCCESS)
-		{
-			if (PdhAddEnglishCounterW(QueryHandle, L"\\Processor(_Total)\\% Processor Time", 0, &CounterHandle) == ERROR_SUCCESS)
-			{
-				PdhCollectQueryData(QueryHandle);
-				Sleep(100);
-				PdhCollectQueryData(QueryHandle);
-
-				PDH_FMT_COUNTERVALUE CounterValue;
-				if (PdhGetFormattedCounterValue(CounterHandle, PDH_FMT_DOUBLE, nullptr, &CounterValue) == ERROR_SUCCESS)
-				{
-					Metrics.CpuUsagePercent = static_cast<float>(CounterValue.doubleValue);
-				}
-			}
-			PdhCloseQuery(QueryHandle);
-		}
-	}
-
 	return Metrics;
 }
 #elif ZEN_PLATFORM_LINUX
@@ -235,39 +210,6 @@ GetSystemMetrics()
 		}
 	}
 
-	// Query CPU usage
-	Metrics.CpuUsagePercent = 0.0f;
-	if (FILE* Stat = fopen("/proc/stat", "r"))
-	{
-		char				 Line[256];
-		unsigned long		 User, Nice, System, Idle, IoWait, Irq, SoftIrq;
-		static unsigned long PrevUser = 0, PrevNice = 0, PrevSystem = 0, PrevIdle = 0, PrevIoWait = 0, PrevIrq = 0, PrevSoftIrq = 0;
-
-		if (fgets(Line, sizeof(Line), Stat))
-		{
-			if (sscanf(Line, "cpu %lu %lu %lu %lu %lu %lu %lu", &User, &Nice, &System, &Idle, &IoWait, &Irq, &SoftIrq) == 7)
-			{
-				unsigned long TotalDelta = (User + Nice + System + Idle + IoWait + Irq + SoftIrq) -
-										   (PrevUser + PrevNice + PrevSystem + PrevIdle + PrevIoWait + PrevIrq + PrevSoftIrq);
-				unsigned long IdleDelta = Idle - PrevIdle;
-
-				if (TotalDelta > 0)
-				{
-					Metrics.CpuUsagePercent = 100.0f * (TotalDelta - IdleDelta) / TotalDelta;
-				}
-
-				PrevUser	= User;
-				PrevNice	= Nice;
-				PrevSystem	= System;
-				PrevIdle	= Idle;
-				PrevIoWait	= IoWait;
-				PrevIrq		= Irq;
-				PrevSoftIrq = SoftIrq;
-			}
-		}
-		fclose(Stat);
-	}
-
 	// Get memory information
 	long Pages		= sysconf(_SC_PHYS_PAGES);
 	long PageSize	= sysconf(_SC_PAGE_SIZE);
@@ -348,25 +290,6 @@ GetSystemMetrics()
 	sysctlbyname("hw.packages", &Packages, &Size, nullptr, 0);
 	Metrics.CpuCount = Packages > 0 ? Packages : 1;
 
-	// Query CPU usage using host_statistics64
-	Metrics.CpuUsagePercent = 0.0f;
-	host_cpu_load_info_data_t CpuLoad;
-	mach_msg_type_number_t	  CpuCount = sizeof(CpuLoad) / sizeof(natural_t);
-	if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (host_info_t)&CpuLoad, &CpuCount) == KERN_SUCCESS)
-	{
-		unsigned long TotalTicks = 0;
-		for (int i = 0; i < CPU_STATE_MAX; ++i)
-		{
-			TotalTicks += CpuLoad.cpu_ticks[i];
-		}
-
-		if (TotalTicks > 0)
-		{
-			unsigned long IdleTicks = CpuLoad.cpu_ticks[CPU_STATE_IDLE];
-			Metrics.CpuUsagePercent = 100.0f * (TotalTicks - IdleTicks) / TotalTicks;
-		}
-	}
-
 	// Get memory information
 	uint64_t MemSize = 0;
 	Size			 = sizeof(MemSize);
@@ -401,6 +324,17 @@ GetSystemMetrics()
 #	error "Unknown platform"
 #endif
 
+ExtendedSystemMetrics
+ApplyReportingOverrides(ExtendedSystemMetrics Metrics)
+{
+	if (g_FakeCpuCount)
+	{
+		Metrics.CoreCount			  = g_FakeCpuCount;
+		Metrics.LogicalProcessorCount = g_FakeCpuCount;
+	}
+	return Metrics;
+}
+
 SystemMetrics
 GetSystemMetricsForReporting()
 {
@@ -415,6 +349,225 @@ GetSystemMetricsForReporting()
 	return Sm;
 }
 
+///////////////////////////////////////////////////////////////////////////
+// SystemMetricsTracker
+///////////////////////////////////////////////////////////////////////////
+
+// Per-platform CPU sampling helper. Called with m_Mutex held.
+
+#if ZEN_PLATFORM_WINDOWS || ZEN_PLATFORM_LINUX
+
+// Samples CPU usage by reading /proc/stat. Used natively on Linux and as a
+// Wine fallback on Windows (where /proc/stat is accessible via the Z: drive).
+struct ProcStatCpuSampler
+{
+	const char*	  Path		  = "/proc/stat";
+	unsigned long PrevUser	  = 0;
+	unsigned long PrevNice	  = 0;
+	unsigned long PrevSystem  = 0;
+	unsigned long PrevIdle	  = 0;
+	unsigned long PrevIoWait  = 0;
+	unsigned long PrevIrq	  = 0;
+	unsigned long PrevSoftIrq = 0;
+
+	explicit ProcStatCpuSampler(const char* InPath = "/proc/stat") : Path(InPath) {}
+
+	float Sample()
+	{
+		float CpuUsage = 0.0f;
+
+		if (FILE* Stat = fopen(Path, "r"))
+		{
+			char		  Line[256];
+			unsigned long User, Nice, System, Idle, IoWait, Irq, SoftIrq;
+
+			if (fgets(Line, sizeof(Line), Stat))
+			{
+				if (sscanf(Line, "cpu %lu %lu %lu %lu %lu %lu %lu", &User, &Nice, &System, &Idle, &IoWait, &Irq, &SoftIrq) == 7)
+				{
+					unsigned long TotalDelta = (User + Nice + System + Idle + IoWait + Irq + SoftIrq) -
+											   (PrevUser + PrevNice + PrevSystem + PrevIdle + PrevIoWait + PrevIrq + PrevSoftIrq);
+					unsigned long IdleDelta = Idle - PrevIdle;
+
+					if (TotalDelta > 0)
+					{
+						CpuUsage = 100.0f * (TotalDelta - IdleDelta) / TotalDelta;
+					}
+
+					PrevUser	= User;
+					PrevNice	= Nice;
+					PrevSystem	= System;
+					PrevIdle	= Idle;
+					PrevIoWait	= IoWait;
+					PrevIrq		= Irq;
+					PrevSoftIrq = SoftIrq;
+				}
+			}
+			fclose(Stat);
+		}
+
+		return CpuUsage;
+	}
+};
+
+#endif
+
+#if ZEN_PLATFORM_WINDOWS
+
+struct CpuSampler
+{
+	PDH_HQUERY		   QueryHandle		 = nullptr;
+	PDH_HCOUNTER	   CounterHandle	 = nullptr;
+	bool			   HasPreviousSample = false;
+	bool			   IsWine			 = false;
+	ProcStatCpuSampler ProcStat{"Z:\\proc\\stat"};
+
+	CpuSampler()
+	{
+		IsWine = zen::windows::IsRunningOnWine();
+
+		if (!IsWine)
+		{
+			if (PdhOpenQueryW(nullptr, 0, &QueryHandle) == ERROR_SUCCESS)
+			{
+				if (PdhAddEnglishCounterW(QueryHandle, L"\\Processor(_Total)\\% Processor Time", 0, &CounterHandle) != ERROR_SUCCESS)
+				{
+					CounterHandle = nullptr;
+				}
+			}
+		}
+	}
+
+	~CpuSampler()
+	{
+		if (QueryHandle)
+		{
+			PdhCloseQuery(QueryHandle);
+		}
+	}
+
+	float Sample()
+	{
+		if (IsWine)
+		{
+			return ProcStat.Sample();
+		}
+
+		if (!QueryHandle || !CounterHandle)
+		{
+			return 0.0f;
+		}
+
+		PdhCollectQueryData(QueryHandle);
+
+		if (!HasPreviousSample)
+		{
+			HasPreviousSample = true;
+			return 0.0f;
+		}
+
+		PDH_FMT_COUNTERVALUE CounterValue;
+		if (PdhGetFormattedCounterValue(CounterHandle, PDH_FMT_DOUBLE, nullptr, &CounterValue) == ERROR_SUCCESS)
+		{
+			return static_cast<float>(CounterValue.doubleValue);
+		}
+
+		return 0.0f;
+	}
+};
+
+#elif ZEN_PLATFORM_LINUX
+
+struct CpuSampler
+{
+	ProcStatCpuSampler ProcStat;
+
+	float Sample() { return ProcStat.Sample(); }
+};
+
+#elif ZEN_PLATFORM_MAC
+
+struct CpuSampler
+{
+	unsigned long PrevTotalTicks = 0;
+	unsigned long PrevIdleTicks	 = 0;
+
+	float Sample()
+	{
+		float CpuUsage = 0.0f;
+
+		host_cpu_load_info_data_t CpuLoad;
+		mach_msg_type_number_t	  Count = sizeof(CpuLoad) / sizeof(natural_t);
+		if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (host_info_t)&CpuLoad, &Count) == KERN_SUCCESS)
+		{
+			unsigned long TotalTicks = 0;
+			for (int i = 0; i < CPU_STATE_MAX; ++i)
+			{
+				TotalTicks += CpuLoad.cpu_ticks[i];
+			}
+			unsigned long IdleTicks = CpuLoad.cpu_ticks[CPU_STATE_IDLE];
+
+			unsigned long TotalDelta = TotalTicks - PrevTotalTicks;
+			unsigned long IdleDelta	 = IdleTicks - PrevIdleTicks;
+
+			if (TotalDelta > 0 && PrevTotalTicks > 0)
+			{
+				CpuUsage = 100.0f * (TotalDelta - IdleDelta) / TotalDelta;
+			}
+
+			PrevTotalTicks = TotalTicks;
+			PrevIdleTicks  = IdleTicks;
+		}
+
+		return CpuUsage;
+	}
+};
+
+#endif
+
+struct SystemMetricsTracker::Impl
+{
+	using Clock = std::chrono::steady_clock;
+
+	std::mutex				  Mutex;
+	CpuSampler				  Sampler;
+	float					  CachedCpuPercent = 0.0f;
+	Clock::time_point		  NextSampleTime   = Clock::now();
+	std::chrono::milliseconds MinInterval;
+
+	explicit Impl(std::chrono::milliseconds InMinInterval) : MinInterval(InMinInterval) {}
+
+	float SampleCpu()
+	{
+		const auto Now = Clock::now();
+		if (Now >= NextSampleTime)
+		{
+			CachedCpuPercent = Sampler.Sample();
+			NextSampleTime	 = Now + MinInterval;
+		}
+		return CachedCpuPercent;
+	}
+};
+
+SystemMetricsTracker::SystemMetricsTracker(std::chrono::milliseconds MinInterval) : m_Impl(std::make_unique<Impl>(MinInterval))
+{
+}
+
+SystemMetricsTracker::~SystemMetricsTracker() = default;
+
+ExtendedSystemMetrics
+SystemMetricsTracker::Query()
+{
+	ExtendedSystemMetrics Metrics;
+	static_cast<SystemMetrics&>(Metrics) = GetSystemMetrics();
+
+	std::lock_guard Lock(m_Impl->Mutex);
+	Metrics.CpuUsagePercent = m_Impl->SampleCpu();
+	return Metrics;
+}
+
+///////////////////////////////////////////////////////////////////////////
+
 std::string_view
 GetOperatingSystemName()
 {
@@ -422,6 +575,24 @@ GetOperatingSystemName()
 }
 
 std::string_view
+GetRuntimePlatformName()
+{
+#if ZEN_PLATFORM_WINDOWS
+	if (zen::windows::IsRunningOnWine())
+	{
+		return "wine"sv;
+	}
+	return "windows"sv;
+#elif ZEN_PLATFORM_LINUX
+	return "linux"sv;
+#elif ZEN_PLATFORM_MAC
+	return "macos"sv;
+#else
+	return "unknown"sv;
+#endif
+}
+
+std::string_view
 GetCpuName()
 {
 #if ZEN_ARCH_X64
@@ -440,4 +611,11 @@ Describe(const SystemMetrics& Metrics, CbWriter& Writer)
 		   << "avail_pagefile_mb" << Metrics.AvailPageFileMiB;
 }
 
+void
+Describe(const ExtendedSystemMetrics& Metrics, CbWriter& Writer)
+{
+	Describe(static_cast<const SystemMetrics&>(Metrics), Writer);
+	Writer << "cpu_usage_percent" << Metrics.CpuUsagePercent;
+}
+
 }  // namespace zen