hub resource limits (#900)

- Feature: Hub dashboard now shows a Resources tile with disk and memory usage against configured limits
- Feature: Hub module listing now shows state-change timestamps and duration for each instance
- Improvement: Hub provisioning rejects new instances when disk or memory usage exceeds configurable thresholds; limits are disabled by default (0 = no limit)
  - `--hub-provision-disk-limit-bytes` - Reject provisioning when used disk exceeds this many bytes
  - `--hub-provision-disk-limit-percent` - Reject provisioning when used disk exceeds this percentage of total disk
  - `--hub-provision-memory-limit-bytes` - Reject provisioning when used memory exceeds this many bytes
  - `--hub-provision-memory-limit-percent` - Reject provisioning when used memory exceeds this percentage of total RAM
- Improvement: Hub process metrics are now tracked atomically per active instance slot, eliminating per-query process handle lookups
- Improvement: Hub, Build Store, and Workspaces service stats sections in the dashboard are now collapsible
- Bugfix: Hub watchdog loop did not check `m_ShutdownFlag`, causing it to spin indefinitely on shutdown

9 files changed, 104 insertions, 150 deletions

diff --git a/src/zenserver/frontend/html/pages/builds.js b/src/zenserver/frontend/html/pages/builds.js
index 095f0bf29..6b3426378 100644
--- a/src/zenserver/frontend/html/pages/builds.js
+++ b/src/zenserver/frontend/html/pages/builds.js
@@ -16,7 +16,7 @@ export class Page extends ZenPage
 		this.set_title("build store");
 
 		// Build Store Stats
-		const stats_section = this.add_section("Build Store Stats");
+		const stats_section = this._collapsible_section("Build Store Service Stats");
 		stats_section.tag().classify("dropall").text("raw yaml \u2192").on_click(() => {
 			window.open("/stats/builds.yaml", "_blank");
 		});
diff --git a/src/zenserver/frontend/html/pages/cache.js b/src/zenserver/frontend/html/pages/cache.js
index 1fc8227c8..e0f6f73b6 100644
--- a/src/zenserver/frontend/html/pages/cache.js
+++ b/src/zenserver/frontend/html/pages/cache.js
@@ -95,39 +95,6 @@ export class Page extends ZenPage
 		}
 	}
 
-	_collapsible_section(name)
-	{
-		const section = this.add_section(name);
-		const container = section._parent.inner();
-		const heading = container.firstElementChild;
-
-		heading.style.cursor = "pointer";
-		heading.style.userSelect = "none";
-
-		const indicator = document.createElement("span");
-		indicator.textContent = " \u25BC";
-		indicator.style.fontSize = "0.7em";
-		heading.appendChild(indicator);
-
-		let collapsed = false;
-		heading.addEventListener("click", (e) => {
-			if (e.target !== heading && e.target !== indicator)
-			{
-				return;
-			}
-			collapsed = !collapsed;
-			indicator.textContent = collapsed ? " \u25B6" : " \u25BC";
-			let sibling = heading.nextElementSibling;
-			while (sibling)
-			{
-				sibling.style.display = collapsed ? "none" : "";
-				sibling = sibling.nextElementSibling;
-			}
-		});
-
-		return section;
-	}
-
 	_render_stats(stats)
 	{
 		const safe = (obj, path) => path.split(".").reduce((a, b) => a && a[b], obj);
diff --git a/src/zenserver/frontend/html/pages/compute.js b/src/zenserver/frontend/html/pages/compute.js
index ab3d49c27..d1a880954 100644
--- a/src/zenserver/frontend/html/pages/compute.js
+++ b/src/zenserver/frontend/html/pages/compute.js
@@ -100,39 +100,6 @@ export class Page extends ZenPage
 		}, 2000);
 	}
 
-	_collapsible_section(name)
-	{
-		const section = this.add_section(name);
-		const container = section._parent.inner();
-		const heading = container.firstElementChild;
-
-		heading.style.cursor = "pointer";
-		heading.style.userSelect = "none";
-
-		const indicator = document.createElement("span");
-		indicator.textContent = " \u25BC";
-		indicator.style.fontSize = "0.7em";
-		heading.appendChild(indicator);
-
-		let collapsed = false;
-		heading.addEventListener("click", (e) => {
-			if (e.target !== heading && e.target !== indicator)
-			{
-				return;
-			}
-			collapsed = !collapsed;
-			indicator.textContent = collapsed ? " \u25B6" : " \u25BC";
-			let sibling = heading.nextElementSibling;
-			while (sibling)
-			{
-				sibling.style.display = collapsed ? "none" : "";
-				sibling = sibling.nextElementSibling;
-			}
-		});
-
-		return section;
-	}
-
 	async _load_chartjs()
 	{
 		if (window.Chart)
diff --git a/src/zenserver/frontend/html/pages/hub.js b/src/zenserver/frontend/html/pages/hub.js
index 78e3a090c..c6f96d496 100644
--- a/src/zenserver/frontend/html/pages/hub.js
+++ b/src/zenserver/frontend/html/pages/hub.js
@@ -82,7 +82,7 @@ export class Page extends ZenPage
 		this.set_title("hub");
 
 		// Capacity
-		const stats_section = this.add_section("Capacity");
+		const stats_section = this._collapsible_section("Hub Service Stats");
 		this._stats_grid = stats_section.tag().classify("grid").classify("stats-tiles");
 
 		// Modules
@@ -203,27 +203,46 @@ export class Page extends ZenPage
 
 		{
 			const tile = grid.tag().classify("card").classify("stats-tile");
-			tile.tag().classify("card-title").text("Active Modules");
+			tile.tag().classify("card-title").text("Instances");
 			const body = tile.tag().classify("tile-metrics");
 			this._metric(body, Friendly.sep(current), "currently provisioned", true);
+			this._metric(body, Friendly.sep(max), "high watermark");
+			this._metric(body, Friendly.sep(limit), "maximum allowed");
+			if (limit > 0)
+			{
+				const pct = ((current / limit) * 100).toFixed(0) + "%";
+				this._metric(body, pct, "utilization");
+			}
 		}
 
+		const machine = data.machine || {};
+		const limits  = data.resource_limits || {};
+		if (machine.disk_total_bytes > 0 || machine.memory_total_mib > 0)
 		{
-			const tile = grid.tag().classify("card").classify("stats-tile");
-			tile.tag().classify("card-title").text("Peak Modules");
-			const body = tile.tag().classify("tile-metrics");
-			this._metric(body, Friendly.sep(max), "high watermark", true);
-		}
+			const disk_used  = Math.max(0, (machine.disk_total_bytes  || 0) - (machine.disk_free_bytes  || 0));
+			const mem_used   = Math.max(0, (machine.memory_total_mib || 0) - (machine.memory_avail_mib || 0)) * 1024 * 1024;
+			const vmem_used  = Math.max(0, (machine.virtual_memory_total_mib || 0) - (machine.virtual_memory_avail_mib || 0)) * 1024 * 1024;
+			const disk_limit = limits.disk_bytes   || 0;
+			const mem_limit  = limits.memory_bytes || 0;
+			const disk_over  = disk_limit > 0 && disk_used > disk_limit;
+			const mem_over   = mem_limit  > 0 && mem_used  > mem_limit;
 
-		{
 			const tile = grid.tag().classify("card").classify("stats-tile");
-			tile.tag().classify("card-title").text("Instance Limit");
-			const body = tile.tag().classify("tile-metrics");
-			this._metric(body, Friendly.sep(limit), "maximum allowed", true);
-			if (limit > 0)
+			if (disk_over || mem_over) { tile.inner().setAttribute("data-over", "true"); }
+			tile.tag().classify("card-title").text("Resources");
+			const columns = tile.tag().classify("tile-columns");
+
+			const left = columns.tag().classify("tile-metrics");
+			this._metric(left, Friendly.bytes(disk_used), "disk used", true);
+			if (disk_limit > 0) { this._metric(left, Friendly.bytes(disk_limit), "disk limit"); }
+
+			const right = columns.tag().classify("tile-metrics");
+			this._metric(right, Friendly.bytes(mem_used), "memory used", true);
+			if (mem_limit > 0) { this._metric(right, Friendly.bytes(mem_limit), "memory limit"); }
+			if (machine.virtual_memory_total_mib > 0)
 			{
-				const pct = ((current / limit) * 100).toFixed(0) + "%";
-				this._metric(body, pct, "utilization");
+				this._metric(right, Friendly.bytes(vmem_used), "vmem used", true);
+				this._metric(right, Friendly.bytes(machine.virtual_memory_total_mib * 1024 * 1024), "vmem total");
 			}
 		}
 	}
@@ -284,6 +303,14 @@ export class Page extends ZenPage
 				}
 				row.state_text.nodeValue   = state;
 				row.port_text.nodeValue    = m.port ? String(m.port) : "";
+				if (m.state_change_time)
+				{
+					const state_label = state.charAt(0).toUpperCase() + state.slice(1);
+					row.state_since_label.textContent = state_label + " since";
+					row.state_age_label.textContent   = state_label + " for";
+					row.state_since_node.nodeValue    = m.state_change_time;
+					row.state_age_node.nodeValue      = Friendly.timespan(Date.now() - new Date(m.state_change_time).getTime());
+				}
 				row.btn_open.disabled      = state !== "provisioned";
 				row.btn_hibernate.disabled = !_btn_enabled(state, "hibernate");
 				row.btn_wake.disabled      = !_btn_enabled(state, "wake");
@@ -388,7 +415,7 @@ export class Page extends ZenPage
 				td_action.appendChild(wrap_o);
 				tr.appendChild(td_action);
 
-				// Build metrics grid from process_metrics keys.
+				// Build metrics grid: fixed state-time rows followed by process_metrics keys.
 				// Keys are split into two halves and interleaved so the grid fills
 				// top-to-bottom in the left column before continuing in the right column.
 				const metric_nodes = new Map();
@@ -396,6 +423,28 @@ export class Page extends ZenPage
 				metrics_td.colSpan = 6;
 				const metrics_grid = document.createElement("div");
 				metrics_grid.className = "module-metrics-grid";
+
+				const _add_fixed_pair = (label, value_str) => {
+					const label_el = document.createElement("span");
+					label_el.className = "module-metrics-label";
+					label_el.textContent = label;
+					const value_node = document.createTextNode(value_str);
+					const value_el   = document.createElement("span");
+					value_el.className = "module-metrics-value";
+					value_el.appendChild(value_node);
+					metrics_grid.appendChild(label_el);
+					metrics_grid.appendChild(value_el);
+					return { label_el, value_node };
+				};
+
+				const state_label     = m.state ? m.state.charAt(0).toUpperCase() + m.state.slice(1) : "State";
+				const state_since_str = m.state_change_time || "";
+				const state_age_str   = m.state_change_time
+					? Friendly.timespan(Date.now() - new Date(m.state_change_time).getTime())
+					: "";
+				const { label_el: state_since_label, value_node: state_since_node } = _add_fixed_pair(state_label + " since", state_since_str);
+				const { label_el: state_age_label,   value_node: state_age_node }   = _add_fixed_pair(state_label + " for",   state_age_str);
+
 				const keys = Object.keys(m.process_metrics || {});
 				const half = Math.ceil(keys.length / 2);
 				const add_metric_pair = (key) => {
@@ -423,7 +472,7 @@ export class Page extends ZenPage
 				metrics_td.appendChild(metrics_grid);
 				metrics_tr.appendChild(metrics_td);
 
-				row = { tr, metrics_tr, idx: td_idx, cb, dot, state_text: state_node, port_text: port_node, btn_expand, btn_open: btn_o, btn_hibernate: btn_h, btn_wake: btn_w, btn_deprov: btn_d, metric_nodes };
+				row = { tr, metrics_tr, idx: td_idx, cb, dot, state_text: state_node, port_text: port_node, btn_expand, btn_open: btn_o, btn_hibernate: btn_h, btn_wake: btn_w, btn_deprov: btn_d, metric_nodes, state_since_node, state_age_node, state_since_label, state_age_label };
 				this._row_cache.set(id, row);
 			}
 
diff --git a/src/zenserver/frontend/html/pages/orchestrator.js b/src/zenserver/frontend/html/pages/orchestrator.js
index 4a9290a3c..a280fabdb 100644
--- a/src/zenserver/frontend/html/pages/orchestrator.js
+++ b/src/zenserver/frontend/html/pages/orchestrator.js
@@ -46,39 +46,6 @@ export class Page extends ZenPage
 		this._connect_ws();
 	}
 
-	_collapsible_section(name)
-	{
-		const section = this.add_section(name);
-		const container = section._parent.inner();
-		const heading = container.firstElementChild;
-
-		heading.style.cursor = "pointer";
-		heading.style.userSelect = "none";
-
-		const indicator = document.createElement("span");
-		indicator.textContent = " \u25BC";
-		indicator.style.fontSize = "0.7em";
-		heading.appendChild(indicator);
-
-		let collapsed = false;
-		heading.addEventListener("click", (e) => {
-			if (e.target !== heading && e.target !== indicator)
-			{
-				return;
-			}
-			collapsed = !collapsed;
-			indicator.textContent = collapsed ? " \u25B6" : " \u25BC";
-			let sibling = heading.nextElementSibling;
-			while (sibling)
-			{
-				sibling.style.display = collapsed ? "none" : "";
-				sibling = sibling.nextElementSibling;
-			}
-		});
-
-		return section;
-	}
-
 	async _fetch_all()
 	{
 		try
diff --git a/src/zenserver/frontend/html/pages/page.js b/src/zenserver/frontend/html/pages/page.js
index cf8d3e3dd..ff530ff8e 100644
--- a/src/zenserver/frontend/html/pages/page.js
+++ b/src/zenserver/frontend/html/pages/page.js
@@ -337,4 +337,37 @@ export class ZenPage extends PageBase
 			this._metric(right, Friendly.duration(reqData.t_max), "max");
 		}
 	}
+
+	_collapsible_section(name)
+	{
+		const section = this.add_section(name);
+		const container = section._parent.inner();
+		const heading = container.firstElementChild;
+
+		heading.style.cursor = "pointer";
+		heading.style.userSelect = "none";
+
+		const indicator = document.createElement("span");
+		indicator.textContent = " \u25BC";
+		indicator.style.fontSize = "0.7em";
+		heading.appendChild(indicator);
+
+		let collapsed = false;
+		heading.addEventListener("click", (e) => {
+			if (e.target !== heading && e.target !== indicator)
+			{
+				return;
+			}
+			collapsed = !collapsed;
+			indicator.textContent = collapsed ? " \u25B6" : " \u25BC";
+			let sibling = heading.nextElementSibling;
+			while (sibling)
+			{
+				sibling.style.display = collapsed ? "none" : "";
+				sibling = sibling.nextElementSibling;
+			}
+		});
+
+		return section;
+	}
 }
diff --git a/src/zenserver/frontend/html/pages/projects.js b/src/zenserver/frontend/html/pages/projects.js
index 2469bf70b..dfe4faeb8 100644
--- a/src/zenserver/frontend/html/pages/projects.js
+++ b/src/zenserver/frontend/html/pages/projects.js
@@ -110,39 +110,6 @@ export class Page extends ZenPage
 		}
 	}
 
-	_collapsible_section(name)
-	{
-		const section = this.add_section(name);
-		const container = section._parent.inner();
-		const heading = container.firstElementChild;
-
-		heading.style.cursor = "pointer";
-		heading.style.userSelect = "none";
-
-		const indicator = document.createElement("span");
-		indicator.textContent = " \u25BC";
-		indicator.style.fontSize = "0.7em";
-		heading.appendChild(indicator);
-
-		let collapsed = false;
-		heading.addEventListener("click", (e) => {
-			if (e.target !== heading && e.target !== indicator)
-			{
-				return;
-			}
-			collapsed = !collapsed;
-			indicator.textContent = collapsed ? " \u25B6" : " \u25BC";
-			let sibling = heading.nextElementSibling;
-			while (sibling)
-			{
-				sibling.style.display = collapsed ? "none" : "";
-				sibling = sibling.nextElementSibling;
-			}
-		});
-
-		return section;
-	}
-
 	_clear_param(name)
 	{
 		this._params.delete(name);
diff --git a/src/zenserver/frontend/html/pages/workspaces.js b/src/zenserver/frontend/html/pages/workspaces.js
index d31fd7373..2442fb35b 100644
--- a/src/zenserver/frontend/html/pages/workspaces.js
+++ b/src/zenserver/frontend/html/pages/workspaces.js
@@ -13,7 +13,7 @@ export class Page extends ZenPage
 		this.set_title("workspaces");
 
 		// Workspace Service Stats
-		const stats_section = this.add_section("Workspace Service Stats");
+		const stats_section = this._collapsible_section("Workspace Service Stats");
 		this._stats_grid = stats_section.tag().classify("grid").classify("stats-tiles");
 
 		const stats = await new Fetcher().resource("stats", "ws").json().catch(() => null);
diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css
index d9f7491ea..cb3d78cf2 100644
--- a/src/zenserver/frontend/html/zen.css
+++ b/src/zenserver/frontend/html/zen.css
@@ -816,6 +816,10 @@ zen-banner + zen-nav::part(nav-bar) {
 	border-color:		var(--theme_p0);
 }
 
+.stats-tile[data-over="true"] {
+	border-color:		var(--theme_fail);
+}
+
 .stats-tile-detailed {
 	position:			relative;
 }