aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2026-04-13 16:38:16 +0200
committerGitHub Enterprise <[email protected]>2026-04-13 16:38:16 +0200
commit795345e5fd7974a1f5227d507a58bb3ed75eafd5 (patch)
tree7a0f142bf562c3590400586c82b0e7a1b5ad6493 /src/zenserver
parent5.8.4-pre2 (diff)
downloadzen-795345e5fd7974a1f5227d507a58bb3ed75eafd5.tar.xz
zen-795345e5fd7974a1f5227d507a58bb3ed75eafd5.zip
Compute OIDC auth, async Horde agents, and orchestrator improvements (#913)
Rework of the Horde agent subsystem from synchronous per-thread I/O to an async ASIO-driven architecture, plus provisioner scale-down with graceful draining, OIDC authentication, scheduler improvements, and dashboard UI for provisioner control. ### Async Horde Agent Rewrite - Replace synchronous `HordeAgent` (one thread per agent, blocking I/O) with `AsyncHordeAgent` — an ASIO state machine running on a shared `io_context` thread pool - Replace `TcpComputeTransport`/`AesComputeTransport` with `AsyncTcpComputeTransport`/`AsyncAesComputeTransport` - Replace `AgentMessageChannel` with `AsyncAgentMessageChannel` using frame queuing and ASIO timers - Delete `ComputeBuffer` and `ComputeChannel` ring-buffer classes (no longer needed) ### Provisioner Drain / Scale-Down - `HordeProvisioner` can now drain agents when target core count is lowered: queries each agent's `/compute/session/status` for workload, selects candidates by largest-fit/lowest-workload, and sends `/compute/session/drain` - Configurable `--horde-drain-grace-period` (default 300s) before force-kill - Implement `IProvisionerStateProvider` interface to expose provisioner state to the orchestrator HTTP layer - Forward `--coordinator-session`, `--provision-clean`, and `--provision-tracehost` through both Horde and Nomad provisioners to spawned workers ### OIDC Authentication - `HordeClient` accepts an `AccessTokenProvider` (refreshable token function) as alternative to static `--horde-token` - Wire up `OidcToken.exe` auto-discovery via `httpclientauth::CreateFromOidcTokenExecutable` with `--HordeUrl` mode - New `--horde-oidctoken-exe-path` CLI option for explicit path override ### Orchestrator & Scheduler - Orchestrator generates a session ID at startup; workers include `coordinator_session` in announcements so the orchestrator can reject stale-session workers - New `Rejected` action state — when a remote runner declines at capacity, the action is rescheduled without retry count increment - Reduce scheduler lock contention: snapshot pending actions under shared lock, sort/trim outside the lock - Parallelize remote action submission across runners via `WorkerThreadPool` with slow-submit warnings - New action field `FailureReason` populated by all runner types (exit codes, sandbox failures, exceptions) - New endpoints: `session/drain`, `session/status`, `session/sunset`, `provisioner/status`, `provisioner/target` ### Remote Execution - Eager-attach mode for `RemoteHttpRunner` — bundles all attachments upfront in a `CbPackage` for single-roundtrip submits - Track in-flight submissions to prevent over-queuing - Show remote runner hostname in `GetDisplayName()` - `--announce-url` to override the endpoint announced to the coordinator (e.g. relay-visible address) ### Frontend Dashboard - Delete standalone `compute.html` (925 lines) and `orchestrator.html` (669 lines), consolidated into JS page modules - Add provisioner panel to orchestrator dashboard: target/active/estimated core counts, draining agent count - Editable target-cores input with debounced POST to `/orch/provisioner/target` - Per-agent provisioning status badges (active / draining / deallocated) in the agents table - Active vs total CPU counts in agents summary row ### CLI - New `zen compute record-start` / `record-stop` subcommands - `zen exec` progress bar with submit and completion phases, atomic work counters, `--progress` mode (Pretty/Plain/Quiet) ### Other - `DataDir` supports environment variable expansion - Worker manifest validation checks for `worker.zcb` marker to detect incomplete cached directories - Linux/Mac runners `nice(5)` child processes to avoid starving the main server - `ComputeService::SetShutdownCallback` wired to `RequestExit` via `session/sunset` - Curl HTTP client logs effective URL on failure - `MachineInfo` carries `Pool` and `Mode` from Horde response - Horde bundle creation includes `.pdb` on Windows
Diffstat (limited to 'src/zenserver')
-rw-r--r--src/zenserver/compute/computeserver.cpp108
-rw-r--r--src/zenserver/compute/computeserver.h7
-rw-r--r--src/zenserver/config/config.cpp23
-rw-r--r--src/zenserver/frontend/html/compute/compute.html925
-rw-r--r--src/zenserver/frontend/html/compute/index.html2
-rw-r--r--src/zenserver/frontend/html/compute/orchestrator.html669
-rw-r--r--src/zenserver/frontend/html/pages/orchestrator.js210
-rw-r--r--src/zenserver/main.cpp8
8 files changed, 342 insertions, 1610 deletions
diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp
index 7296098e0..8cd8b4cfe 100644
--- a/src/zenserver/compute/computeserver.cpp
+++ b/src/zenserver/compute/computeserver.cpp
@@ -22,6 +22,8 @@
# if ZEN_WITH_HORDE
# include <zenhorde/hordeconfig.h>
# include <zenhorde/hordeprovisioner.h>
+# include <zenhttp/httpclientauth.h>
+# include <zenutil/authutils.h>
# endif
# if ZEN_WITH_NOMAD
# include <zennomad/nomadconfig.h>
@@ -67,6 +69,20 @@ ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options)
Options.add_option("compute",
"",
+ "coordinator-session",
+ "Session ID of the orchestrator (for stale-instance rejection)",
+ cxxopts::value<std::string>(m_ServerOptions.CoordinatorSession)->default_value(""),
+ "");
+
+ Options.add_option("compute",
+ "",
+ "announce-url",
+ "Override URL announced to the coordinator (e.g. relay-visible endpoint)",
+ cxxopts::value<std::string>(m_ServerOptions.AnnounceUrl)->default_value(""),
+ "");
+
+ Options.add_option("compute",
+ "",
"idms",
"Enable IDMS cloud detection; optionally specify a custom probe endpoint",
cxxopts::value<std::string>(m_ServerOptions.IdmsEndpoint)->default_value("")->implicit_value("auto"),
@@ -79,6 +95,20 @@ ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options)
cxxopts::value<bool>(m_ServerOptions.EnableWorkerWebSocket)->default_value("false"),
"");
+ Options.add_option("compute",
+ "",
+ "provision-clean",
+ "Pass --clean to provisioned worker instances so they wipe state on startup",
+ cxxopts::value<bool>(m_ServerOptions.ProvisionClean)->default_value("false"),
+ "");
+
+ Options.add_option("compute",
+ "",
+ "provision-tracehost",
+ "Pass --tracehost to provisioned worker instances for remote trace collection",
+ cxxopts::value<std::string>(m_ServerOptions.ProvisionTraceHost)->default_value(""),
+ "");
+
# if ZEN_WITH_HORDE
// Horde provisioning options
Options.add_option("horde",
@@ -139,6 +169,13 @@ ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options)
Options.add_option("horde",
"",
+ "horde-drain-grace-period",
+ "Grace period in seconds for draining agents before force-kill",
+ cxxopts::value<int>(m_ServerOptions.HordeConfig.DrainGracePeriodSeconds)->default_value("300"),
+ "");
+
+ Options.add_option("horde",
+ "",
"horde-host",
"Host address for Horde agents to connect back to",
cxxopts::value<std::string>(m_ServerOptions.HordeConfig.HostAddress)->default_value(""),
@@ -164,6 +201,13 @@ ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options)
"Port number for Zen service communication",
cxxopts::value<uint16_t>(m_ServerOptions.HordeConfig.ZenServicePort)->default_value("8558"),
"");
+
+ Options.add_option("horde",
+ "",
+ "horde-oidctoken-exe-path",
+ "Path to OidcToken executable for automatic Horde authentication",
+ cxxopts::value<std::string>(m_HordeOidcTokenExePath)->default_value(""),
+ "");
# endif
# if ZEN_WITH_NOMAD
@@ -313,6 +357,30 @@ ZenComputeServerConfigurator::ValidateOptions()
# if ZEN_WITH_HORDE
horde::FromString(m_ServerOptions.HordeConfig.Mode, m_HordeModeStr);
horde::FromString(m_ServerOptions.HordeConfig.EncryptionMode, m_HordeEncryptionStr);
+
+ // Set up OidcToken-based authentication if no static token was provided
+ if (m_ServerOptions.HordeConfig.AuthToken.empty() && !m_ServerOptions.HordeConfig.ServerUrl.empty())
+ {
+ std::filesystem::path OidcExePath = FindOidcTokenExePath(m_HordeOidcTokenExePath);
+ if (!OidcExePath.empty())
+ {
+ ZEN_INFO("using OidcToken executable for Horde authentication: {}", OidcExePath);
+ auto Provider = httpclientauth::CreateFromOidcTokenExecutable(OidcExePath,
+ m_ServerOptions.HordeConfig.ServerUrl,
+ /*Quiet=*/true,
+ /*Unattended=*/false,
+ /*Hidden=*/true,
+ /*IsHordeUrl=*/true);
+ if (Provider)
+ {
+ m_ServerOptions.HordeConfig.AccessTokenProvider = std::move(*Provider);
+ }
+ else
+ {
+ ZEN_WARN("OidcToken authentication failed; Horde requests will be unauthenticated");
+ }
+ }
+ }
# endif
# if ZEN_WITH_NOMAD
@@ -347,6 +415,8 @@ ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServ
}
m_CoordinatorEndpoint = ServerConfig.CoordinatorEndpoint;
+ m_CoordinatorSession = ServerConfig.CoordinatorSession;
+ m_AnnounceUrl = ServerConfig.AnnounceUrl;
m_InstanceId = ServerConfig.InstanceId;
m_EnableWorkerWebSocket = ServerConfig.EnableWorkerWebSocket;
@@ -379,7 +449,14 @@ ZenComputeServer::Cleanup()
m_AnnounceTimer.cancel();
# if ZEN_WITH_HORDE
- // Shut down Horde provisioner first — this signals all agent threads
+ // Disconnect the provisioner state provider before destroying the
+ // provisioner so the orchestrator HTTP layer cannot call into it.
+ if (m_OrchestratorService)
+ {
+ m_OrchestratorService->SetProvisionerStateProvider(nullptr);
+ }
+
+ // Shut down Horde provisioner — this signals all agent threads
// to exit and joins them before we tear down HTTP services.
m_HordeProvisioner.reset();
# endif
@@ -482,6 +559,7 @@ ZenComputeServer::InitializeServices(const ZenComputeServerConfig& ServerConfig)
m_StatsService,
ServerConfig.DataDir / "functions",
ServerConfig.MaxConcurrentActions);
+ m_ComputeService->SetShutdownCallback([this] { RequestExit(0); });
m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatsService, m_StatusService);
@@ -506,7 +584,11 @@ ZenComputeServer::InitializeServices(const ZenComputeServerConfig& ServerConfig)
OrchestratorEndpoint << '/';
}
- m_NomadProvisioner = std::make_unique<nomad::NomadProvisioner>(NomadCfg, OrchestratorEndpoint);
+ m_NomadProvisioner = std::make_unique<nomad::NomadProvisioner>(NomadCfg,
+ OrchestratorEndpoint,
+ m_OrchestratorService->GetSessionId().ToString(),
+ ServerConfig.ProvisionClean,
+ ServerConfig.ProvisionTraceHost);
}
}
# endif
@@ -537,7 +619,14 @@ ZenComputeServer::InitializeServices(const ZenComputeServerConfig& ServerConfig)
: std::filesystem::path(HordeConfig.BinariesPath);
std::filesystem::path WorkingDir = ServerConfig.DataDir / "horde";
- m_HordeProvisioner = std::make_unique<horde::HordeProvisioner>(HordeConfig, BinariesPath, WorkingDir, OrchestratorEndpoint);
+ m_HordeProvisioner = std::make_unique<horde::HordeProvisioner>(HordeConfig,
+ BinariesPath,
+ WorkingDir,
+ OrchestratorEndpoint,
+ m_OrchestratorService->GetSessionId().ToString(),
+ ServerConfig.ProvisionClean,
+ ServerConfig.ProvisionTraceHost);
+ m_OrchestratorService->SetProvisionerStateProvider(m_HordeProvisioner.get());
}
}
# endif
@@ -565,6 +654,10 @@ ZenComputeServer::GetInstanceId() const
std::string
ZenComputeServer::GetAnnounceUrl() const
{
+ if (!m_AnnounceUrl.empty())
+ {
+ return m_AnnounceUrl;
+ }
return m_Http->GetServiceUri(nullptr);
}
@@ -635,6 +728,11 @@ ZenComputeServer::BuildAnnounceBody()
<< "nomad";
}
+ if (!m_CoordinatorSession.empty())
+ {
+ AnnounceBody << "coordinator_session" << m_CoordinatorSession;
+ }
+
ResolveCloudMetadata();
if (m_CloudMetadata)
{
@@ -781,8 +879,10 @@ ZenComputeServer::ProvisionerMaintenanceTick()
# if ZEN_WITH_HORDE
if (m_HordeProvisioner)
{
- m_HordeProvisioner->SetTargetCoreCount(UINT32_MAX);
+ // Re-apply current target to spawn agent threads for any that have
+ // exited since the last tick, without overwriting a user-set target.
auto Stats = m_HordeProvisioner->GetStats();
+ m_HordeProvisioner->SetTargetCoreCount(Stats.TargetCoreCount);
ZEN_DEBUG("Horde maintenance: target={}, estimated={}, active={}",
Stats.TargetCoreCount,
Stats.EstimatedCoreCount,
diff --git a/src/zenserver/compute/computeserver.h b/src/zenserver/compute/computeserver.h
index 38f93bc36..63db7e9b3 100644
--- a/src/zenserver/compute/computeserver.h
+++ b/src/zenserver/compute/computeserver.h
@@ -49,9 +49,13 @@ struct ZenComputeServerConfig : public ZenServerConfig
std::string UpstreamNotificationEndpoint;
std::string InstanceId; // For use in notifications
std::string CoordinatorEndpoint;
+ std::string CoordinatorSession; ///< Session ID for stale-instance rejection
+ std::string AnnounceUrl; ///< Override for self-announced URL (e.g. relay-visible endpoint)
std::string IdmsEndpoint;
int32_t MaxConcurrentActions = 0; // 0 = auto (LogicalProcessorCount * 2)
bool EnableWorkerWebSocket = false; // Use WebSocket for worker↔orchestrator link
+ bool ProvisionClean = false; // Pass --clean to provisioned workers
+ std::string ProvisionTraceHost; // Pass --tracehost to provisioned workers
# if ZEN_WITH_HORDE
horde::HordeConfig HordeConfig;
@@ -84,6 +88,7 @@ private:
# if ZEN_WITH_HORDE
std::string m_HordeModeStr = "direct";
std::string m_HordeEncryptionStr = "none";
+ std::string m_HordeOidcTokenExePath;
# endif
# if ZEN_WITH_NOMAD
@@ -147,6 +152,8 @@ private:
# endif
SystemMetricsTracker m_MetricsTracker;
std::string m_CoordinatorEndpoint;
+ std::string m_CoordinatorSession;
+ std::string m_AnnounceUrl;
std::string m_InstanceId;
asio::steady_timer m_AnnounceTimer{m_IoContext};
diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp
index daad154bc..6449159fd 100644
--- a/src/zenserver/config/config.cpp
+++ b/src/zenserver/config/config.cpp
@@ -12,6 +12,7 @@
#include <zencore/compactbinaryutil.h>
#include <zencore/compactbinaryvalidation.h>
#include <zencore/except.h>
+#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/iobuffer.h>
#include <zencore/logging.h>
@@ -478,15 +479,27 @@ ZenServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenServerConfig
throw std::runtime_error(fmt::format("'--snapshot-dir' ('{}') must be a directory", ServerOptions.BaseSnapshotDir));
}
- ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir);
- ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir);
- ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir);
- ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile);
- ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir);
+ SystemRootDir = ExpandEnvironmentVariables(SystemRootDir);
+ ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir);
+
+ DataDir = ExpandEnvironmentVariables(DataDir);
+ ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir);
+
+ ContentDir = ExpandEnvironmentVariables(ContentDir);
+ ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir);
+
+ ConfigFile = ExpandEnvironmentVariables(ConfigFile);
+ ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile);
+
+ BaseSnapshotDir = ExpandEnvironmentVariables(BaseSnapshotDir);
+ ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir);
+
+ ExpandEnvironmentVariables(SecurityConfigPath);
ServerOptions.SecurityConfigPath = MakeSafeAbsolutePath(SecurityConfigPath);
if (!UnixSocketPath.empty())
{
+ UnixSocketPath = ExpandEnvironmentVariables(UnixSocketPath);
ServerOptions.HttpConfig.UnixSocketPath = MakeSafeAbsolutePath(UnixSocketPath);
}
diff --git a/src/zenserver/frontend/html/compute/compute.html b/src/zenserver/frontend/html/compute/compute.html
deleted file mode 100644
index c07bbb692..000000000
--- a/src/zenserver/frontend/html/compute/compute.html
+++ /dev/null
@@ -1,925 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
- <meta charset="UTF-8">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
- <title>Zen Compute Dashboard</title>
- <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.umd.min.js"></script>
- <link rel="stylesheet" type="text/css" href="../zen.css" />
- <script src="../util/sanitize.js"></script>
- <script src="../theme.js"></script>
- <script src="../banner.js" defer></script>
- <script src="../nav.js" defer></script>
- <style>
- .grid {
- grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
- }
-
- .chart-container {
- position: relative;
- height: 300px;
- margin-top: 20px;
- }
-
- .stats-row {
- display: flex;
- justify-content: space-between;
- margin-bottom: 12px;
- padding: 8px 0;
- border-bottom: 1px solid var(--theme_border_subtle);
- }
-
- .stats-row:last-child {
- border-bottom: none;
- margin-bottom: 0;
- }
-
- .stats-label {
- color: var(--theme_g1);
- font-size: 13px;
- }
-
- .stats-value {
- color: var(--theme_bright);
- font-weight: 600;
- font-size: 13px;
- }
-
- .rate-stats {
- display: grid;
- grid-template-columns: repeat(3, 1fr);
- gap: 16px;
- margin-top: 16px;
- }
-
- .rate-item {
- text-align: center;
- }
-
- .rate-value {
- font-size: 20px;
- font-weight: 600;
- color: var(--theme_p0);
- }
-
- .rate-label {
- font-size: 11px;
- color: var(--theme_g1);
- margin-top: 4px;
- text-transform: uppercase;
- }
-
- .worker-row {
- cursor: pointer;
- transition: background 0.15s;
- }
-
- .worker-row:hover {
- background: var(--theme_p4);
- }
-
- .worker-row.selected {
- background: var(--theme_p3);
- }
-
- .worker-detail {
- margin-top: 20px;
- border-top: 1px solid var(--theme_g2);
- padding-top: 16px;
- }
-
- .worker-detail-title {
- font-size: 15px;
- font-weight: 600;
- color: var(--theme_bright);
- margin-bottom: 12px;
- }
-
- .detail-section {
- margin-bottom: 16px;
- }
-
- .detail-section-label {
- font-size: 11px;
- font-weight: 600;
- color: var(--theme_g1);
- text-transform: uppercase;
- letter-spacing: 0.5px;
- margin-bottom: 6px;
- }
-
- .detail-table {
- width: 100%;
- border-collapse: collapse;
- font-size: 12px;
- }
-
- .detail-table td {
- padding: 4px 8px;
- color: var(--theme_g0);
- border-bottom: 1px solid var(--theme_border_subtle);
- vertical-align: top;
- }
-
- .detail-table td:first-child {
- color: var(--theme_g1);
- width: 40%;
- font-family: monospace;
- }
-
- .detail-table tr:last-child td {
- border-bottom: none;
- }
-
- .detail-mono {
- font-family: monospace;
- font-size: 11px;
- color: var(--theme_g1);
- }
-
- .detail-tag {
- display: inline-block;
- padding: 2px 8px;
- border-radius: 4px;
- background: var(--theme_border_subtle);
- color: var(--theme_g0);
- font-size: 11px;
- margin: 2px 4px 2px 0;
- }
- </style>
-</head>
-<body>
- <div class="container" style="max-width: 1400px; margin: 0 auto;">
- <zen-banner cluster-status="nominal" load="0" tagline="Node Overview" logo-src="../favicon.ico"></zen-banner>
- <zen-nav>
- <a href="/dashboard/">Home</a>
- <a href="compute.html">Node</a>
- <a href="orchestrator.html">Orchestrator</a>
- </zen-nav>
- <div class="timestamp">Last updated: <span id="last-update">Never</span></div>
-
- <div id="error-container"></div>
-
- <!-- Action Queue Stats -->
- <div class="section-title">Action Queue</div>
- <div class="grid">
- <div class="card">
- <div class="card-title">Pending Actions</div>
- <div class="metric-value" id="actions-pending">-</div>
- <div class="metric-label">Waiting to be scheduled</div>
- </div>
- <div class="card">
- <div class="card-title">Running Actions</div>
- <div class="metric-value" id="actions-running">-</div>
- <div class="metric-label">Currently executing</div>
- </div>
- <div class="card">
- <div class="card-title">Completed Actions</div>
- <div class="metric-value" id="actions-complete">-</div>
- <div class="metric-label">Results available</div>
- </div>
- </div>
-
- <!-- Action Queue Chart -->
- <div class="card" style="margin-bottom: 30px;">
- <div class="card-title">Action Queue History</div>
- <div class="chart-container">
- <canvas id="queue-chart"></canvas>
- </div>
- </div>
-
- <!-- Performance Metrics -->
- <div class="section-title">Performance Metrics</div>
- <div class="card" style="margin-bottom: 30px;">
- <div class="card-title">Completion Rate</div>
- <div class="rate-stats">
- <div class="rate-item">
- <div class="rate-value" id="rate-1">-</div>
- <div class="rate-label">1 min rate</div>
- </div>
- <div class="rate-item">
- <div class="rate-value" id="rate-5">-</div>
- <div class="rate-label">5 min rate</div>
- </div>
- <div class="rate-item">
- <div class="rate-value" id="rate-15">-</div>
- <div class="rate-label">15 min rate</div>
- </div>
- </div>
- <div style="margin-top: 20px;">
- <div class="stats-row">
- <span class="stats-label">Total Retired</span>
- <span class="stats-value" id="retired-count">-</span>
- </div>
- <div class="stats-row">
- <span class="stats-label">Mean Rate</span>
- <span class="stats-value" id="rate-mean">-</span>
- </div>
- </div>
- </div>
-
- <!-- Workers -->
- <div class="section-title">Workers</div>
- <div class="card" style="margin-bottom: 30px;">
- <div class="card-title">Worker Status</div>
- <div class="stats-row">
- <span class="stats-label">Registered Workers</span>
- <span class="stats-value" id="worker-count">-</span>
- </div>
- <div id="worker-table-container" style="margin-top: 16px; display: none;">
- <table id="worker-table">
- <thead>
- <tr>
- <th>Name</th>
- <th>Platform</th>
- <th style="text-align: right;">Cores</th>
- <th style="text-align: right;">Timeout</th>
- <th style="text-align: right;">Functions</th>
- <th>Worker ID</th>
- </tr>
- </thead>
- <tbody id="worker-table-body"></tbody>
- </table>
- <div id="worker-detail" class="worker-detail" style="display: none;"></div>
- </div>
- </div>
-
- <!-- Queues -->
- <div class="section-title">Queues</div>
- <div class="card" style="margin-bottom: 30px;">
- <div class="card-title">Queue Status</div>
- <div id="queue-list-empty" class="empty-state" style="text-align: left;">No queues.</div>
- <div id="queue-list-container" style="display: none;">
- <table id="queue-list-table">
- <thead>
- <tr>
- <th style="text-align: right; width: 60px;">ID</th>
- <th style="text-align: center; width: 80px;">Status</th>
- <th style="text-align: right;">Active</th>
- <th style="text-align: right;">Completed</th>
- <th style="text-align: right;">Failed</th>
- <th style="text-align: right;">Abandoned</th>
- <th style="text-align: right;">Cancelled</th>
- <th>Token</th>
- </tr>
- </thead>
- <tbody id="queue-list-body"></tbody>
- </table>
- </div>
- </div>
-
- <!-- Action History -->
- <div class="section-title">Recent Actions</div>
- <div class="card" style="margin-bottom: 30px;">
- <div class="card-title">Action History</div>
- <div id="action-history-empty" class="empty-state" style="text-align: left;">No actions recorded yet.</div>
- <div id="action-history-container" style="display: none;">
- <table id="action-history-table">
- <thead>
- <tr>
- <th style="text-align: right; width: 60px;">LSN</th>
- <th style="text-align: right; width: 60px;">Queue</th>
- <th style="text-align: center; width: 70px;">Status</th>
- <th>Function</th>
- <th style="text-align: right; width: 80px;">Started</th>
- <th style="text-align: right; width: 80px;">Finished</th>
- <th style="text-align: right; width: 80px;">Duration</th>
- <th>Worker ID</th>
- <th>Action ID</th>
- </tr>
- </thead>
- <tbody id="action-history-body"></tbody>
- </table>
- </div>
- </div>
-
- <!-- System Resources -->
- <div class="section-title">System Resources</div>
- <div class="grid">
- <div class="card">
- <div class="card-title">CPU Usage</div>
- <div class="metric-value" id="cpu-usage">-</div>
- <div class="metric-label">Percent</div>
- <div class="progress-bar">
- <div class="progress-fill" id="cpu-progress" style="width: 0%"></div>
- </div>
- <div style="position: relative; height: 60px; margin-top: 12px;">
- <canvas id="cpu-chart"></canvas>
- </div>
- <div style="margin-top: 12px;">
- <div class="stats-row">
- <span class="stats-label">Packages</span>
- <span class="stats-value" id="cpu-packages">-</span>
- </div>
- <div class="stats-row">
- <span class="stats-label">Physical Cores</span>
- <span class="stats-value" id="cpu-cores">-</span>
- </div>
- <div class="stats-row">
- <span class="stats-label">Logical Processors</span>
- <span class="stats-value" id="cpu-lp">-</span>
- </div>
- </div>
- </div>
- <div class="card">
- <div class="card-title">Memory</div>
- <div class="stats-row">
- <span class="stats-label">Used</span>
- <span class="stats-value" id="memory-used">-</span>
- </div>
- <div class="stats-row">
- <span class="stats-label">Total</span>
- <span class="stats-value" id="memory-total">-</span>
- </div>
- <div class="progress-bar">
- <div class="progress-fill" id="memory-progress" style="width: 0%"></div>
- </div>
- </div>
- <div class="card">
- <div class="card-title">Disk</div>
- <div class="stats-row">
- <span class="stats-label">Used</span>
- <span class="stats-value" id="disk-used">-</span>
- </div>
- <div class="stats-row">
- <span class="stats-label">Total</span>
- <span class="stats-value" id="disk-total">-</span>
- </div>
- <div class="progress-bar">
- <div class="progress-fill" id="disk-progress" style="width: 0%"></div>
- </div>
- </div>
- </div>
- </div>
-
- <script>
- // Configuration
- const BASE_URL = window.location.origin;
- const REFRESH_INTERVAL = 2000; // 2 seconds
- const MAX_HISTORY_POINTS = 60; // Show last 2 minutes
-
- // Data storage
- const history = {
- timestamps: [],
- pending: [],
- running: [],
- completed: [],
- cpu: []
- };
-
- // CPU sparkline chart
- const cpuCtx = document.getElementById('cpu-chart').getContext('2d');
- const cpuChart = new Chart(cpuCtx, {
- type: 'line',
- data: {
- labels: [],
- datasets: [{
- data: [],
- borderColor: '#58a6ff',
- backgroundColor: 'rgba(88, 166, 255, 0.15)',
- borderWidth: 1.5,
- tension: 0.4,
- fill: true,
- pointRadius: 0
- }]
- },
- options: {
- responsive: true,
- maintainAspectRatio: false,
- animation: false,
- plugins: { legend: { display: false }, tooltip: { enabled: false } },
- scales: {
- x: { display: false },
- y: { display: false, min: 0, max: 100 }
- }
- }
- });
-
- // Queue chart setup
- const ctx = document.getElementById('queue-chart').getContext('2d');
- const chart = new Chart(ctx, {
- type: 'line',
- data: {
- labels: [],
- datasets: [
- {
- label: 'Pending',
- data: [],
- borderColor: '#f0883e',
- backgroundColor: 'rgba(240, 136, 62, 0.1)',
- tension: 0.4,
- fill: true
- },
- {
- label: 'Running',
- data: [],
- borderColor: '#58a6ff',
- backgroundColor: 'rgba(88, 166, 255, 0.1)',
- tension: 0.4,
- fill: true
- },
- {
- label: 'Completed',
- data: [],
- borderColor: '#3fb950',
- backgroundColor: 'rgba(63, 185, 80, 0.1)',
- tension: 0.4,
- fill: true
- }
- ]
- },
- options: {
- responsive: true,
- maintainAspectRatio: false,
- plugins: {
- legend: {
- display: true,
- labels: {
- color: '#8b949e'
- }
- }
- },
- scales: {
- x: {
- display: false
- },
- y: {
- beginAtZero: true,
- ticks: {
- color: '#8b949e'
- },
- grid: {
- color: '#21262d'
- }
- }
- }
- }
- });
-
- // Helper functions
-
- function formatBytes(bytes) {
- if (bytes === 0) return '0 B';
- const k = 1024;
- const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
- const i = Math.floor(Math.log(bytes) / Math.log(k));
- return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
- }
-
- function formatRate(rate) {
- return rate.toFixed(2) + '/s';
- }
-
- function showError(message) {
- const container = document.getElementById('error-container');
- container.innerHTML = `<div class="error">Error: ${escapeHtml(message)}</div>`;
- }
-
- function clearError() {
- document.getElementById('error-container').innerHTML = '';
- }
-
- function updateTimestamp() {
- const now = new Date();
- document.getElementById('last-update').textContent = now.toLocaleTimeString();
- }
-
- // Fetch functions
- async function fetchJSON(endpoint) {
- const response = await fetch(`${BASE_URL}${endpoint}`, {
- headers: {
- 'Accept': 'application/json'
- }
- });
- if (!response.ok) {
- throw new Error(`HTTP ${response.status}: ${response.statusText}`);
- }
- return await response.json();
- }
-
- async function fetchHealth() {
- try {
- const response = await fetch(`${BASE_URL}/compute/ready`);
- const isHealthy = response.status === 200;
-
- const banner = document.querySelector('zen-banner');
-
- if (isHealthy) {
- banner.setAttribute('cluster-status', 'nominal');
- banner.setAttribute('load', '0');
- } else {
- banner.setAttribute('cluster-status', 'degraded');
- banner.setAttribute('load', '0');
- }
-
- return isHealthy;
- } catch (error) {
- const banner = document.querySelector('zen-banner');
- banner.setAttribute('cluster-status', 'degraded');
- banner.setAttribute('load', '0');
- throw error;
- }
- }
-
- async function fetchStats() {
- const data = await fetchJSON('/stats/compute');
-
- // Update action counts
- document.getElementById('actions-pending').textContent = data.actions_pending || 0;
- document.getElementById('actions-running').textContent = data.actions_submitted || 0;
- document.getElementById('actions-complete').textContent = data.actions_complete || 0;
-
- // Update completion rates
- if (data.actions_retired) {
- document.getElementById('rate-1').textContent = formatRate(data.actions_retired.rate_1 || 0);
- document.getElementById('rate-5').textContent = formatRate(data.actions_retired.rate_5 || 0);
- document.getElementById('rate-15').textContent = formatRate(data.actions_retired.rate_15 || 0);
- document.getElementById('retired-count').textContent = data.actions_retired.count || 0;
- document.getElementById('rate-mean').textContent = formatRate(data.actions_retired.rate_mean || 0);
- }
-
- // Update chart
- const now = new Date().toLocaleTimeString();
- history.timestamps.push(now);
- history.pending.push(data.actions_pending || 0);
- history.running.push(data.actions_submitted || 0);
- history.completed.push(data.actions_complete || 0);
-
- // Keep only last N points
- if (history.timestamps.length > MAX_HISTORY_POINTS) {
- history.timestamps.shift();
- history.pending.shift();
- history.running.shift();
- history.completed.shift();
- }
-
- chart.data.labels = history.timestamps;
- chart.data.datasets[0].data = history.pending;
- chart.data.datasets[1].data = history.running;
- chart.data.datasets[2].data = history.completed;
- chart.update('none');
- }
-
- async function fetchSysInfo() {
- const data = await fetchJSON('/compute/sysinfo');
-
- // Update CPU
- const cpuUsage = data.cpu_usage || 0;
- document.getElementById('cpu-usage').textContent = cpuUsage.toFixed(1) + '%';
- document.getElementById('cpu-progress').style.width = cpuUsage + '%';
-
- const banner = document.querySelector('zen-banner');
- banner.setAttribute('load', cpuUsage.toFixed(1));
-
- history.cpu.push(cpuUsage);
- if (history.cpu.length > MAX_HISTORY_POINTS) history.cpu.shift();
- cpuChart.data.labels = history.cpu.map(() => '');
- cpuChart.data.datasets[0].data = history.cpu;
- cpuChart.update('none');
-
- document.getElementById('cpu-packages').textContent = data.cpu_count ?? '-';
- document.getElementById('cpu-cores').textContent = data.core_count ?? '-';
- document.getElementById('cpu-lp').textContent = data.lp_count ?? '-';
-
- // Update Memory
- const memUsed = data.memory_used || 0;
- const memTotal = data.memory_total || 1;
- const memPercent = (memUsed / memTotal) * 100;
- document.getElementById('memory-used').textContent = formatBytes(memUsed);
- document.getElementById('memory-total').textContent = formatBytes(memTotal);
- document.getElementById('memory-progress').style.width = memPercent + '%';
-
- // Update Disk
- const diskUsed = data.disk_used || 0;
- const diskTotal = data.disk_total || 1;
- const diskPercent = (diskUsed / diskTotal) * 100;
- document.getElementById('disk-used').textContent = formatBytes(diskUsed);
- document.getElementById('disk-total').textContent = formatBytes(diskTotal);
- document.getElementById('disk-progress').style.width = diskPercent + '%';
- }
-
- // Persists the selected worker ID across refreshes
- let selectedWorkerId = null;
-
- function renderWorkerDetail(id, desc) {
- const panel = document.getElementById('worker-detail');
-
- if (!desc) {
- panel.style.display = 'none';
- return;
- }
-
- function field(label, value) {
- return `<tr><td>${label}</td><td>${value ?? '-'}</td></tr>`;
- }
-
- function monoField(label, value) {
- return `<tr><td>${label}</td><td class="detail-mono">${value ?? '-'}</td></tr>`;
- }
-
- // Functions
- const functions = desc.functions || [];
- const functionsHtml = functions.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
- `<table class="detail-table">${functions.map(f =>
- `<tr><td>${escapeHtml(f.name || '-')}</td><td class="detail-mono">${escapeHtml(f.version || '-')}</td></tr>`
- ).join('')}</table>`;
-
- // Executables
- const executables = desc.executables || [];
- const totalExecSize = executables.reduce((sum, e) => sum + (e.size || 0), 0);
- const execHtml = executables.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
- `<table class="detail-table">
- <tr style="font-size:11px;">
- <td style="color:var(--theme_faint);padding-bottom:4px;">Path</td>
- <td style="color:var(--theme_faint);padding-bottom:4px;">Hash</td>
- <td style="color:var(--theme_faint);padding-bottom:4px;text-align:right;">Size</td>
- </tr>
- ${executables.map(e =>
- `<tr>
- <td>${escapeHtml(e.name || '-')}</td>
- <td class="detail-mono">${escapeHtml(e.hash || '-')}</td>
- <td style="text-align:right;white-space:nowrap;">${e.size != null ? formatBytes(e.size) : '-'}</td>
- </tr>`
- ).join('')}
- <tr style="border-top:1px solid var(--theme_g2);">
- <td style="color:var(--theme_g1);padding-top:6px;">Total</td>
- <td></td>
- <td style="text-align:right;white-space:nowrap;padding-top:6px;color:var(--theme_bright);font-weight:600;">${formatBytes(totalExecSize)}</td>
- </tr>
- </table>`;
-
- // Files
- const files = desc.files || [];
- const filesHtml = files.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
- `<table class="detail-table">${files.map(f =>
- `<tr><td>${escapeHtml(f.name || f)}</td><td class="detail-mono">${escapeHtml(f.hash || '')}</td></tr>`
- ).join('')}</table>`;
-
- // Dirs
- const dirs = desc.dirs || [];
- const dirsHtml = dirs.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
- dirs.map(d => `<span class="detail-tag">${escapeHtml(d)}</span>`).join('');
-
- // Environment
- const env = desc.environment || [];
- const envHtml = env.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
- env.map(e => `<span class="detail-tag">${escapeHtml(e)}</span>`).join('');
-
- panel.innerHTML = `
- <div class="worker-detail-title">${escapeHtml(desc.name || id)}</div>
- <div class="detail-section">
- <table class="detail-table">
- ${field('Worker ID', `<span class="detail-mono">${escapeHtml(id)}</span>`)}
- ${field('Path', escapeHtml(desc.path || '-'))}
- ${field('Platform', escapeHtml(desc.host || '-'))}
- ${monoField('Build System', desc.buildsystem_version)}
- ${field('Cores', desc.cores)}
- ${field('Timeout', desc.timeout != null ? desc.timeout + 's' : null)}
- </table>
- </div>
- <div class="detail-section">
- <div class="detail-section-label">Functions</div>
- ${functionsHtml}
- </div>
- <div class="detail-section">
- <div class="detail-section-label">Executables</div>
- ${execHtml}
- </div>
- <div class="detail-section">
- <div class="detail-section-label">Files</div>
- ${filesHtml}
- </div>
- <div class="detail-section">
- <div class="detail-section-label">Directories</div>
- ${dirsHtml}
- </div>
- <div class="detail-section">
- <div class="detail-section-label">Environment</div>
- ${envHtml}
- </div>
- `;
- panel.style.display = 'block';
- }
-
- async function fetchWorkers() {
- const data = await fetchJSON('/compute/workers');
- const workerIds = data.workers || [];
-
- document.getElementById('worker-count').textContent = workerIds.length;
-
- const container = document.getElementById('worker-table-container');
- const tbody = document.getElementById('worker-table-body');
-
- if (workerIds.length === 0) {
- container.style.display = 'none';
- selectedWorkerId = null;
- return;
- }
-
- const descriptors = await Promise.all(
- workerIds.map(id => fetchJSON(`/compute/workers/${id}`).catch(() => null))
- );
-
- // Build a map for quick lookup by ID
- const descriptorMap = {};
- workerIds.forEach((id, i) => { descriptorMap[id] = descriptors[i]; });
-
- tbody.innerHTML = '';
- descriptors.forEach((desc, i) => {
- const id = workerIds[i];
- const name = desc ? (desc.name || '-') : '-';
- const host = desc ? (desc.host || '-') : '-';
- const cores = desc ? (desc.cores != null ? desc.cores : '-') : '-';
- const timeout = desc ? (desc.timeout != null ? desc.timeout + 's' : '-') : '-';
- const functions = desc ? (desc.functions ? desc.functions.length : 0) : '-';
-
- const tr = document.createElement('tr');
- tr.className = 'worker-row' + (id === selectedWorkerId ? ' selected' : '');
- tr.dataset.workerId = id;
- tr.innerHTML = `
- <td style="color: var(--theme_bright);">${escapeHtml(name)}</td>
- <td>${escapeHtml(host)}</td>
- <td style="text-align: right;">${escapeHtml(String(cores))}</td>
- <td style="text-align: right;">${escapeHtml(String(timeout))}</td>
- <td style="text-align: right;">${escapeHtml(String(functions))}</td>
- <td style="color: var(--theme_g1); font-family: monospace; font-size: 11px;">${escapeHtml(id)}</td>
- `;
- tr.addEventListener('click', () => {
- document.querySelectorAll('.worker-row').forEach(r => r.classList.remove('selected'));
- if (selectedWorkerId === id) {
- // Toggle off
- selectedWorkerId = null;
- document.getElementById('worker-detail').style.display = 'none';
- } else {
- selectedWorkerId = id;
- tr.classList.add('selected');
- renderWorkerDetail(id, descriptorMap[id]);
- }
- });
- tbody.appendChild(tr);
- });
-
- // Re-render detail if selected worker is still present
- if (selectedWorkerId && descriptorMap[selectedWorkerId]) {
- renderWorkerDetail(selectedWorkerId, descriptorMap[selectedWorkerId]);
- } else if (selectedWorkerId && !descriptorMap[selectedWorkerId]) {
- selectedWorkerId = null;
- document.getElementById('worker-detail').style.display = 'none';
- }
-
- container.style.display = 'block';
- }
-
- // Windows FILETIME: 100ns ticks since 1601-01-01. Convert to JS Date.
- const FILETIME_EPOCH_OFFSET_MS = 11644473600000n;
- function filetimeToDate(ticks) {
- if (!ticks) return null;
- const ms = BigInt(ticks) / 10000n - FILETIME_EPOCH_OFFSET_MS;
- return new Date(Number(ms));
- }
-
- function formatTime(date) {
- if (!date) return '-';
- return date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' });
- }
-
- function formatDuration(startDate, endDate) {
- if (!startDate || !endDate) return '-';
- const ms = endDate - startDate;
- if (ms < 0) return '-';
- if (ms < 1000) return ms + ' ms';
- if (ms < 60000) return (ms / 1000).toFixed(2) + ' s';
- const m = Math.floor(ms / 60000);
- const s = ((ms % 60000) / 1000).toFixed(0).padStart(2, '0');
- return `${m}m ${s}s`;
- }
-
- async function fetchQueues() {
- const data = await fetchJSON('/compute/queues');
- const queues = data.queues || [];
-
- const empty = document.getElementById('queue-list-empty');
- const container = document.getElementById('queue-list-container');
- const tbody = document.getElementById('queue-list-body');
-
- if (queues.length === 0) {
- empty.style.display = '';
- container.style.display = 'none';
- return;
- }
-
- empty.style.display = 'none';
- tbody.innerHTML = '';
-
- for (const q of queues) {
- const id = q.queue_id ?? '-';
- const badge = q.state === 'cancelled'
- ? '<span class="status-badge failure">cancelled</span>'
- : q.state === 'draining'
- ? '<span class="status-badge" style="background:color-mix(in srgb, var(--theme_warn) 15%, transparent);color:var(--theme_warn);">draining</span>'
- : q.is_complete
- ? '<span class="status-badge success">complete</span>'
- : '<span class="status-badge" style="background:color-mix(in srgb, var(--theme_p0) 15%, transparent);color:var(--theme_p0);">active</span>';
- const token = q.queue_token
- ? `<span class="detail-mono">${escapeHtml(q.queue_token)}</span>`
- : '<span style="color:var(--theme_faint);">-</span>';
-
- const tr = document.createElement('tr');
- tr.innerHTML = `
- <td style="text-align: right; font-family: monospace; color: var(--theme_bright);">${escapeHtml(String(id))}</td>
- <td style="text-align: center;">${badge}</td>
- <td style="text-align: right;">${q.active_count ?? 0}</td>
- <td style="text-align: right; color: var(--theme_ok);">${q.completed_count ?? 0}</td>
- <td style="text-align: right; color: var(--theme_fail);">${q.failed_count ?? 0}</td>
- <td style="text-align: right; color: var(--theme_warn);">${q.abandoned_count ?? 0}</td>
- <td style="text-align: right; color: var(--theme_warn);">${q.cancelled_count ?? 0}</td>
- <td>${token}</td>
- `;
- tbody.appendChild(tr);
- }
-
- container.style.display = 'block';
- }
-
- async function fetchActionHistory() {
- const data = await fetchJSON('/compute/jobs/history?limit=50');
- const entries = data.history || [];
-
- const empty = document.getElementById('action-history-empty');
- const container = document.getElementById('action-history-container');
- const tbody = document.getElementById('action-history-body');
-
- if (entries.length === 0) {
- empty.style.display = '';
- container.style.display = 'none';
- return;
- }
-
- empty.style.display = 'none';
- tbody.innerHTML = '';
-
- // Entries arrive oldest-first; reverse to show newest at top
- for (const entry of [...entries].reverse()) {
- const lsn = entry.lsn ?? '-';
- const succeeded = entry.succeeded;
- const badge = succeeded == null
- ? '<span class="status-badge" style="background:var(--theme_border_subtle);color:var(--theme_g1);">unknown</span>'
- : succeeded
- ? '<span class="status-badge success">ok</span>'
- : '<span class="status-badge failure">failed</span>';
- const desc = entry.actionDescriptor || {};
- const fn = desc.Function || '-';
- const workerId = entry.workerId || '-';
- const actionId = entry.actionId || '-';
-
- const startDate = filetimeToDate(entry.time_Running);
- const endDate = filetimeToDate(entry.time_Completed ?? entry.time_Failed);
-
- const queueId = entry.queueId || 0;
- const queueCell = queueId
- ? `<a href="/compute/queues/${queueId}" style="color: var(--theme_ln); text-decoration: none; font-family: monospace;">${escapeHtml(String(queueId))}</a>`
- : '<span style="color: var(--theme_faint);">-</span>';
-
- const tr = document.createElement('tr');
- tr.innerHTML = `
- <td style="text-align: right; font-family: monospace; color: var(--theme_g1);">${escapeHtml(String(lsn))}</td>
- <td style="text-align: right;">${queueCell}</td>
- <td style="text-align: center;">${badge}</td>
- <td style="color: var(--theme_bright);">${escapeHtml(fn)}</td>
- <td style="text-align: right; font-size: 12px; white-space: nowrap; color: var(--theme_g1);">${formatTime(startDate)}</td>
- <td style="text-align: right; font-size: 12px; white-space: nowrap; color: var(--theme_g1);">${formatTime(endDate)}</td>
- <td style="text-align: right; font-size: 12px; white-space: nowrap;">${formatDuration(startDate, endDate)}</td>
- <td style="font-family: monospace; font-size: 11px; color: var(--theme_g1);">${escapeHtml(workerId)}</td>
- <td style="font-family: monospace; font-size: 11px; color: var(--theme_g1);">${escapeHtml(actionId)}</td>
- `;
- tbody.appendChild(tr);
- }
-
- container.style.display = 'block';
- }
-
- async function updateDashboard() {
- try {
- await Promise.all([
- fetchHealth(),
- fetchStats(),
- fetchSysInfo(),
- fetchWorkers(),
- fetchQueues(),
- fetchActionHistory()
- ]);
-
- clearError();
- updateTimestamp();
- } catch (error) {
- console.error('Error updating dashboard:', error);
- showError(error.message);
- }
- }
-
- // Start updating
- updateDashboard();
- setInterval(updateDashboard, REFRESH_INTERVAL);
- </script>
-</body>
-</html>
diff --git a/src/zenserver/frontend/html/compute/index.html b/src/zenserver/frontend/html/compute/index.html
index 9597fd7f3..aaa09aec0 100644
--- a/src/zenserver/frontend/html/compute/index.html
+++ b/src/zenserver/frontend/html/compute/index.html
@@ -1 +1 @@
-<meta http-equiv="refresh" content="0; url=compute.html" /> \ No newline at end of file
+<meta http-equiv="refresh" content="0; url=/dashboard/?page=compute" /> \ No newline at end of file
diff --git a/src/zenserver/frontend/html/compute/orchestrator.html b/src/zenserver/frontend/html/compute/orchestrator.html
deleted file mode 100644
index d1a2bb015..000000000
--- a/src/zenserver/frontend/html/compute/orchestrator.html
+++ /dev/null
@@ -1,669 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
- <meta charset="UTF-8">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link rel="stylesheet" type="text/css" href="../zen.css" />
- <script src="../util/sanitize.js"></script>
- <script src="../theme.js"></script>
- <script src="../banner.js" defer></script>
- <script src="../nav.js" defer></script>
- <title>Zen Orchestrator Dashboard</title>
- <style>
- .agent-count {
- display: flex;
- align-items: center;
- gap: 8px;
- font-size: 14px;
- padding: 8px 16px;
- border-radius: 6px;
- background: var(--theme_g3);
- border: 1px solid var(--theme_g2);
- }
-
- .agent-count .count {
- font-size: 20px;
- font-weight: 600;
- color: var(--theme_bright);
- }
- </style>
-</head>
-<body>
- <div class="container" style="max-width: 1400px; margin: 0 auto;">
- <zen-banner cluster-status="nominal" load="0" logo-src="../favicon.ico"></zen-banner>
- <zen-nav>
- <a href="/dashboard/">Home</a>
- <a href="compute.html">Node</a>
- <a href="orchestrator.html">Orchestrator</a>
- </zen-nav>
- <div class="header">
- <div>
- <div class="timestamp">Last updated: <span id="last-update">Never</span></div>
- </div>
- <div class="agent-count">
- <span>Agents:</span>
- <span class="count" id="agent-count">-</span>
- </div>
- </div>
-
- <div id="error-container"></div>
-
- <div class="card">
- <div class="card-title">Compute Agents</div>
- <div id="empty-state" class="empty-state">No agents registered.</div>
- <table id="agent-table" style="display: none;">
- <thead>
- <tr>
- <th style="width: 40px; text-align: center;">Health</th>
- <th>Hostname</th>
- <th style="text-align: right;">CPUs</th>
- <th style="text-align: right;">CPU Usage</th>
- <th style="text-align: right;">Memory</th>
- <th style="text-align: right;">Queues</th>
- <th style="text-align: right;">Pending</th>
- <th style="text-align: right;">Running</th>
- <th style="text-align: right;">Completed</th>
- <th style="text-align: right;">Traffic</th>
- <th style="text-align: right;">Last Seen</th>
- </tr>
- </thead>
- <tbody id="agent-table-body"></tbody>
- </table>
- </div>
- <div class="card" style="margin-top: 20px;">
- <div class="card-title">Connected Clients</div>
- <div id="clients-empty" class="empty-state">No clients connected.</div>
- <table id="clients-table" style="display: none;">
- <thead>
- <tr>
- <th style="width: 40px; text-align: center;">Health</th>
- <th>Client ID</th>
- <th>Hostname</th>
- <th>Address</th>
- <th style="text-align: right;">Last Seen</th>
- </tr>
- </thead>
- <tbody id="clients-table-body"></tbody>
- </table>
- </div>
- <div class="card" style="margin-top: 20px;">
- <div style="display: flex; align-items: center; gap: 12px; margin-bottom: 12px;">
- <div class="card-title" style="margin-bottom: 0;">Event History</div>
- <div class="history-tabs">
- <button class="history-tab active" data-tab="workers" onclick="switchHistoryTab('workers')">Workers</button>
- <button class="history-tab" data-tab="clients" onclick="switchHistoryTab('clients')">Clients</button>
- </div>
- </div>
- <div id="history-panel-workers">
- <div id="history-empty" class="empty-state">No provisioning events recorded.</div>
- <table id="history-table" style="display: none;">
- <thead>
- <tr>
- <th>Time</th>
- <th>Event</th>
- <th>Worker</th>
- <th>Hostname</th>
- </tr>
- </thead>
- <tbody id="history-table-body"></tbody>
- </table>
- </div>
- <div id="history-panel-clients" style="display: none;">
- <div id="client-history-empty" class="empty-state">No client events recorded.</div>
- <table id="client-history-table" style="display: none;">
- <thead>
- <tr>
- <th>Time</th>
- <th>Event</th>
- <th>Client</th>
- <th>Hostname</th>
- </tr>
- </thead>
- <tbody id="client-history-table-body"></tbody>
- </table>
- </div>
- </div>
- </div>
-
- <script>
- const BASE_URL = window.location.origin;
- const REFRESH_INTERVAL = 2000;
-
- function showError(message) {
- document.getElementById('error-container').innerHTML =
- '<div class="error">Error: ' + escapeHtml(message) + '</div>';
- }
-
- function clearError() {
- document.getElementById('error-container').innerHTML = '';
- }
-
- function formatLastSeen(dtMs) {
- if (dtMs == null) return '-';
- var seconds = Math.floor(dtMs / 1000);
- if (seconds < 60) return seconds + 's ago';
- var minutes = Math.floor(seconds / 60);
- if (minutes < 60) return minutes + 'm ' + (seconds % 60) + 's ago';
- var hours = Math.floor(minutes / 60);
- return hours + 'h ' + (minutes % 60) + 'm ago';
- }
-
- function healthClass(dtMs, reachable) {
- if (reachable === false) return 'health-red';
- if (dtMs == null) return 'health-red';
- var seconds = dtMs / 1000;
- if (seconds < 30 && reachable === true) return 'health-green';
- if (seconds < 120) return 'health-yellow';
- return 'health-red';
- }
-
- function healthTitle(dtMs, reachable) {
- var seenStr = dtMs != null ? 'Last seen ' + formatLastSeen(dtMs) : 'Never seen';
- if (reachable === true) return seenStr + ' · Reachable';
- if (reachable === false) return seenStr + ' · Unreachable';
- return seenStr + ' · Reachability unknown';
- }
-
- function formatCpuUsage(percent) {
- if (percent == null || percent === 0) return '-';
- return percent.toFixed(1) + '%';
- }
-
- function formatMemory(usedBytes, totalBytes) {
- if (!totalBytes) return '-';
- var usedGiB = usedBytes / (1024 * 1024 * 1024);
- var totalGiB = totalBytes / (1024 * 1024 * 1024);
- return usedGiB.toFixed(1) + ' / ' + totalGiB.toFixed(1) + ' GiB';
- }
-
- function formatBytes(bytes) {
- if (!bytes) return '-';
- if (bytes < 1024) return bytes + ' B';
- if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KiB';
- if (bytes < 1024 * 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(1) + ' MiB';
- if (bytes < 1024 * 1024 * 1024 * 1024) return (bytes / (1024 * 1024 * 1024)).toFixed(1) + ' GiB';
- return (bytes / (1024 * 1024 * 1024 * 1024)).toFixed(1) + ' TiB';
- }
-
- function formatTraffic(recv, sent) {
- if (!recv && !sent) return '-';
- return formatBytes(recv) + ' / ' + formatBytes(sent);
- }
-
- function parseIpFromUri(uri) {
- try {
- var url = new URL(uri);
- var host = url.hostname;
- // Strip IPv6 brackets
- if (host.startsWith('[') && host.endsWith(']')) host = host.slice(1, -1);
- // Only handle IPv4
- var parts = host.split('.');
- if (parts.length !== 4) return null;
- var octets = parts.map(Number);
- if (octets.some(function(o) { return isNaN(o) || o < 0 || o > 255; })) return null;
- return octets;
- } catch (e) {
- return null;
- }
- }
-
- function computeCidr(ips) {
- if (ips.length === 0) return null;
- if (ips.length === 1) return ips[0].join('.') + '/32';
-
- // Convert each IP to a 32-bit integer
- var ints = ips.map(function(o) {
- return ((o[0] << 24) | (o[1] << 16) | (o[2] << 8) | o[3]) >>> 0;
- });
-
- // Find common prefix length by ANDing all identical high bits
- var common = ~0 >>> 0;
- for (var i = 1; i < ints.length; i++) {
- // XOR to find differing bits, then mask away everything from the first difference down
- var diff = (ints[0] ^ ints[i]) >>> 0;
- if (diff !== 0) {
- var bit = 31 - Math.floor(Math.log2(diff));
- var mask = bit > 0 ? ((~0 << (32 - bit)) >>> 0) : 0;
- common = (common & mask) >>> 0;
- }
- }
-
- // Count leading ones in the common mask
- var prefix = 0;
- for (var b = 31; b >= 0; b--) {
- if ((common >>> b) & 1) prefix++;
- else break;
- }
-
- // Network address
- var net = (ints[0] & common) >>> 0;
- var a = (net >>> 24) & 0xff;
- var bv = (net >>> 16) & 0xff;
- var c = (net >>> 8) & 0xff;
- var d = net & 0xff;
- return a + '.' + bv + '.' + c + '.' + d + '/' + prefix;
- }
-
- function renderDashboard(data) {
- var banner = document.querySelector('zen-banner');
- if (data.hostname) {
- banner.setAttribute('tagline', 'Orchestrator \u2014 ' + data.hostname);
- }
- var workers = data.workers || [];
-
- document.getElementById('agent-count').textContent = workers.length;
-
- if (workers.length === 0) {
- banner.setAttribute('cluster-status', 'degraded');
- banner.setAttribute('load', '0');
- } else {
- banner.setAttribute('cluster-status', 'nominal');
- }
-
- var emptyState = document.getElementById('empty-state');
- var table = document.getElementById('agent-table');
- var tbody = document.getElementById('agent-table-body');
-
- if (workers.length === 0) {
- emptyState.style.display = '';
- table.style.display = 'none';
- } else {
- emptyState.style.display = 'none';
- table.style.display = '';
-
- tbody.innerHTML = '';
- var totalCpus = 0;
- var totalWeightedCpuUsage = 0;
- var totalMemUsed = 0;
- var totalMemTotal = 0;
- var totalQueues = 0;
- var totalPending = 0;
- var totalRunning = 0;
- var totalCompleted = 0;
- var totalBytesRecv = 0;
- var totalBytesSent = 0;
- var allIps = [];
- for (var i = 0; i < workers.length; i++) {
- var w = workers[i];
- var uri = w.uri || '';
- var dt = w.dt;
- var dashboardUrl = uri + '/dashboard/compute/';
-
- var id = w.id || '';
-
- var hostname = w.hostname || '';
- var cpus = w.cpus || 0;
- totalCpus += cpus;
- if (cpus > 0 && typeof w.cpu_usage === 'number') {
- totalWeightedCpuUsage += w.cpu_usage * cpus;
- }
-
- var memTotal = w.memory_total || 0;
- var memUsed = w.memory_used || 0;
- totalMemTotal += memTotal;
- totalMemUsed += memUsed;
-
- var activeQueues = w.active_queues || 0;
- totalQueues += activeQueues;
-
- var actionsPending = w.actions_pending || 0;
- var actionsRunning = w.actions_running || 0;
- var actionsCompleted = w.actions_completed || 0;
- totalPending += actionsPending;
- totalRunning += actionsRunning;
- totalCompleted += actionsCompleted;
-
- var bytesRecv = w.bytes_received || 0;
- var bytesSent = w.bytes_sent || 0;
- totalBytesRecv += bytesRecv;
- totalBytesSent += bytesSent;
-
- var ip = parseIpFromUri(uri);
- if (ip) allIps.push(ip);
-
- var reachable = w.reachable;
- var hClass = healthClass(dt, reachable);
- var hTitle = healthTitle(dt, reachable);
-
- var platform = w.platform || '';
- var badges = '';
- if (platform) {
- var platColors = { windows: '#0078d4', wine: '#722f37', linux: '#e95420', macos: '#a2aaad' };
- var platColor = platColors[platform] || '#8b949e';
- badges += ' <span style="display:inline-block;padding:1px 6px;border-radius:10px;font-size:10px;font-weight:600;color:#fff;background:' + platColor + ';vertical-align:middle;margin-left:4px;">' + escapeHtml(platform) + '</span>';
- }
- var provisioner = w.provisioner || '';
- if (provisioner) {
- var provColors = { horde: '#8957e5', nomad: '#3fb950' };
- var provColor = provColors[provisioner] || '#8b949e';
- badges += ' <span style="display:inline-block;padding:1px 6px;border-radius:10px;font-size:10px;font-weight:600;color:#fff;background:' + provColor + ';vertical-align:middle;margin-left:4px;">' + escapeHtml(provisioner) + '</span>';
- }
-
- var tr = document.createElement('tr');
- tr.title = id;
- tr.innerHTML =
- '<td style="text-align: center;"><span class="health-dot ' + hClass + '" title="' + escapeHtml(hTitle) + '"></span></td>' +
- '<td><a href="' + escapeHtml(dashboardUrl) + '" target="_blank">' + escapeHtml(hostname) + '</a>' + badges + '</td>' +
- '<td style="text-align: right;">' + (cpus > 0 ? cpus : '-') + '</td>' +
- '<td style="text-align: right;">' + formatCpuUsage(w.cpu_usage) + '</td>' +
- '<td style="text-align: right;">' + formatMemory(memUsed, memTotal) + '</td>' +
- '<td style="text-align: right;">' + (activeQueues > 0 ? activeQueues : '-') + '</td>' +
- '<td style="text-align: right;">' + actionsPending + '</td>' +
- '<td style="text-align: right;">' + actionsRunning + '</td>' +
- '<td style="text-align: right;">' + actionsCompleted + '</td>' +
- '<td style="text-align: right; font-size: 11px; color: var(--theme_g1);">' + formatTraffic(bytesRecv, bytesSent) + '</td>' +
- '<td style="text-align: right; color: var(--theme_g1);">' + formatLastSeen(dt) + '</td>';
- tbody.appendChild(tr);
- }
-
- var clusterLoad = totalCpus > 0 ? (totalWeightedCpuUsage / totalCpus) : 0;
- banner.setAttribute('load', clusterLoad.toFixed(1));
-
- // Total row
- var cidr = computeCidr(allIps);
- var totalTr = document.createElement('tr');
- totalTr.className = 'total-row';
- totalTr.innerHTML =
- '<td></td>' +
- '<td style="text-align: right; color: var(--theme_g1); text-transform: uppercase; font-size: 11px;">Total' + (cidr ? ' <span style="font-family: monospace; font-weight: normal;">' + escapeHtml(cidr) + '</span>' : '') + '</td>' +
- '<td style="text-align: right;">' + totalCpus + '</td>' +
- '<td></td>' +
- '<td style="text-align: right;">' + formatMemory(totalMemUsed, totalMemTotal) + '</td>' +
- '<td style="text-align: right;">' + totalQueues + '</td>' +
- '<td style="text-align: right;">' + totalPending + '</td>' +
- '<td style="text-align: right;">' + totalRunning + '</td>' +
- '<td style="text-align: right;">' + totalCompleted + '</td>' +
- '<td style="text-align: right; font-size: 11px;">' + formatTraffic(totalBytesRecv, totalBytesSent) + '</td>' +
- '<td></td>';
- tbody.appendChild(totalTr);
- }
-
- clearError();
- document.getElementById('last-update').textContent = new Date().toLocaleTimeString();
-
- // Render provisioning history if present in WebSocket payload
- if (data.events) {
- renderProvisioningHistory(data.events);
- }
-
- // Render connected clients if present
- if (data.clients) {
- renderClients(data.clients);
- }
-
- // Render client history if present
- if (data.client_events) {
- renderClientHistory(data.client_events);
- }
- }
-
- function eventBadge(type) {
- var colors = { joined: 'var(--theme_ok)', left: 'var(--theme_fail)', returned: 'var(--theme_warn)' };
- var labels = { joined: 'Joined', left: 'Left', returned: 'Returned' };
- var color = colors[type] || 'var(--theme_g1)';
- var label = labels[type] || type;
- return '<span style="display:inline-block;padding:2px 8px;border-radius:4px;font-size:11px;font-weight:600;color:var(--theme_g4);background:' + color + ';">' + escapeHtml(label) + '</span>';
- }
-
- function formatTimestamp(ts) {
- if (!ts) return '-';
- // CbObject DateTime serialized as ticks (100ns since 0001-01-01) or ISO string
- var date;
- if (typeof ts === 'number') {
- // .NET-style ticks: convert to Unix ms
- var unixMs = (ts - 621355968000000000) / 10000;
- date = new Date(unixMs);
- } else {
- date = new Date(ts);
- }
- if (isNaN(date.getTime())) return '-';
- return date.toLocaleTimeString();
- }
-
- var activeHistoryTab = 'workers';
-
- function switchHistoryTab(tab) {
- activeHistoryTab = tab;
- var tabs = document.querySelectorAll('.history-tab');
- for (var i = 0; i < tabs.length; i++) {
- tabs[i].classList.toggle('active', tabs[i].getAttribute('data-tab') === tab);
- }
- document.getElementById('history-panel-workers').style.display = tab === 'workers' ? '' : 'none';
- document.getElementById('history-panel-clients').style.display = tab === 'clients' ? '' : 'none';
- }
-
- function renderProvisioningHistory(events) {
- var emptyState = document.getElementById('history-empty');
- var table = document.getElementById('history-table');
- var tbody = document.getElementById('history-table-body');
-
- if (!events || events.length === 0) {
- emptyState.style.display = '';
- table.style.display = 'none';
- return;
- }
-
- emptyState.style.display = 'none';
- table.style.display = '';
- tbody.innerHTML = '';
-
- for (var i = 0; i < events.length; i++) {
- var evt = events[i];
- var tr = document.createElement('tr');
- tr.innerHTML =
- '<td style="color: var(--theme_g1);">' + formatTimestamp(evt.ts) + '</td>' +
- '<td>' + eventBadge(evt.type) + '</td>' +
- '<td>' + escapeHtml(evt.worker_id || '') + '</td>' +
- '<td>' + escapeHtml(evt.hostname || '') + '</td>';
- tbody.appendChild(tr);
- }
- }
-
- function clientHealthClass(dtMs) {
- if (dtMs == null) return 'health-red';
- var seconds = dtMs / 1000;
- if (seconds < 30) return 'health-green';
- if (seconds < 120) return 'health-yellow';
- return 'health-red';
- }
-
- function renderClients(clients) {
- var emptyState = document.getElementById('clients-empty');
- var table = document.getElementById('clients-table');
- var tbody = document.getElementById('clients-table-body');
-
- if (!clients || clients.length === 0) {
- emptyState.style.display = '';
- table.style.display = 'none';
- return;
- }
-
- emptyState.style.display = 'none';
- table.style.display = '';
- tbody.innerHTML = '';
-
- for (var i = 0; i < clients.length; i++) {
- var c = clients[i];
- var dt = c.dt;
- var hClass = clientHealthClass(dt);
- var hTitle = dt != null ? 'Last seen ' + formatLastSeen(dt) : 'Never seen';
-
- var sessionBadge = '';
- if (c.session_id) {
- sessionBadge = ' <span style="font-family:monospace;font-size:10px;color:var(--theme_faint);" title="Session ' + escapeHtml(c.session_id) + '">' + escapeHtml(c.session_id.substring(0, 8)) + '</span>';
- }
-
- var tr = document.createElement('tr');
- tr.innerHTML =
- '<td style="text-align: center;"><span class="health-dot ' + hClass + '" title="' + escapeHtml(hTitle) + '"></span></td>' +
- '<td>' + escapeHtml(c.id || '') + sessionBadge + '</td>' +
- '<td>' + escapeHtml(c.hostname || '') + '</td>' +
- '<td style="font-family: monospace; font-size: 12px; color: var(--theme_g1);">' + escapeHtml(c.address || '') + '</td>' +
- '<td style="text-align: right; color: var(--theme_g1);">' + formatLastSeen(dt) + '</td>';
- tbody.appendChild(tr);
- }
- }
-
- function clientEventBadge(type) {
- var colors = { connected: 'var(--theme_ok)', disconnected: 'var(--theme_fail)', updated: 'var(--theme_warn)' };
- var labels = { connected: 'Connected', disconnected: 'Disconnected', updated: 'Updated' };
- var color = colors[type] || 'var(--theme_g1)';
- var label = labels[type] || type;
- return '<span style="display:inline-block;padding:2px 8px;border-radius:4px;font-size:11px;font-weight:600;color:var(--theme_g4);background:' + color + ';">' + escapeHtml(label) + '</span>';
- }
-
- function renderClientHistory(events) {
- var emptyState = document.getElementById('client-history-empty');
- var table = document.getElementById('client-history-table');
- var tbody = document.getElementById('client-history-table-body');
-
- if (!events || events.length === 0) {
- emptyState.style.display = '';
- table.style.display = 'none';
- return;
- }
-
- emptyState.style.display = 'none';
- table.style.display = '';
- tbody.innerHTML = '';
-
- for (var i = 0; i < events.length; i++) {
- var evt = events[i];
- var tr = document.createElement('tr');
- tr.innerHTML =
- '<td style="color: var(--theme_g1);">' + formatTimestamp(evt.ts) + '</td>' +
- '<td>' + clientEventBadge(evt.type) + '</td>' +
- '<td>' + escapeHtml(evt.client_id || '') + '</td>' +
- '<td>' + escapeHtml(evt.hostname || '') + '</td>';
- tbody.appendChild(tr);
- }
- }
-
- // Fetch-based polling fallback
- var pollTimer = null;
-
- async function fetchProvisioningHistory() {
- try {
- var response = await fetch(BASE_URL + '/orch/history?limit=50', {
- headers: { 'Accept': 'application/json' }
- });
- if (response.ok) {
- var data = await response.json();
- renderProvisioningHistory(data.events || []);
- }
- } catch (e) {
- console.error('Error fetching provisioning history:', e);
- }
- }
-
- async function fetchClients() {
- try {
- var response = await fetch(BASE_URL + '/orch/clients', {
- headers: { 'Accept': 'application/json' }
- });
- if (response.ok) {
- var data = await response.json();
- renderClients(data.clients || []);
- }
- } catch (e) {
- console.error('Error fetching clients:', e);
- }
- }
-
- async function fetchClientHistory() {
- try {
- var response = await fetch(BASE_URL + '/orch/clients/history?limit=50', {
- headers: { 'Accept': 'application/json' }
- });
- if (response.ok) {
- var data = await response.json();
- renderClientHistory(data.client_events || []);
- }
- } catch (e) {
- console.error('Error fetching client history:', e);
- }
- }
-
- async function fetchDashboard() {
- var banner = document.querySelector('zen-banner');
- try {
- var response = await fetch(BASE_URL + '/orch/agents', {
- headers: { 'Accept': 'application/json' }
- });
-
- if (!response.ok) {
- banner.setAttribute('cluster-status', 'degraded');
- throw new Error('HTTP ' + response.status + ': ' + response.statusText);
- }
-
- renderDashboard(await response.json());
- fetchProvisioningHistory();
- fetchClients();
- fetchClientHistory();
- } catch (error) {
- console.error('Error updating dashboard:', error);
- showError(error.message);
- banner.setAttribute('cluster-status', 'offline');
- }
- }
-
- function startPolling() {
- if (pollTimer) return;
- fetchDashboard();
- pollTimer = setInterval(fetchDashboard, REFRESH_INTERVAL);
- }
-
- function stopPolling() {
- if (pollTimer) {
- clearInterval(pollTimer);
- pollTimer = null;
- }
- }
-
- // WebSocket connection with automatic reconnect and polling fallback
- var ws = null;
-
- function connectWebSocket() {
- var proto = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
- ws = new WebSocket(proto + '//' + window.location.host + '/orch/ws');
-
- ws.onopen = function() {
- stopPolling();
- clearError();
- };
-
- ws.onmessage = function(event) {
- try {
- renderDashboard(JSON.parse(event.data));
- } catch (e) {
- console.error('WebSocket message parse error:', e);
- }
- };
-
- ws.onclose = function() {
- ws = null;
- startPolling();
- setTimeout(connectWebSocket, 3000);
- };
-
- ws.onerror = function() {
- // onclose will fire after onerror
- };
- }
-
- // Fetch orchestrator hostname for the banner
- fetch(BASE_URL + '/orch/status', { headers: { 'Accept': 'application/json' } })
- .then(function(r) { return r.ok ? r.json() : null; })
- .then(function(d) {
- if (d && d.hostname) {
- document.querySelector('zen-banner').setAttribute('tagline', 'Orchestrator \u2014 ' + d.hostname);
- }
- })
- .catch(function() {});
-
- // Initial load via fetch, then try WebSocket
- fetchDashboard();
- connectWebSocket();
- </script>
-</body>
-</html>
diff --git a/src/zenserver/frontend/html/pages/orchestrator.js b/src/zenserver/frontend/html/pages/orchestrator.js
index a280fabdb..30f6a8122 100644
--- a/src/zenserver/frontend/html/pages/orchestrator.js
+++ b/src/zenserver/frontend/html/pages/orchestrator.js
@@ -14,6 +14,14 @@ export class Page extends ZenPage
{
this.set_title("orchestrator");
+ // Provisioner section (hidden until data arrives)
+ this._prov_section = this._collapsible_section("Provisioner");
+ this._prov_section._parent.inner().style.display = "none";
+ this._prov_grid = null;
+ this._prov_target_dirty = false;
+ this._prov_commit_timer = null;
+ this._prov_last_target = null;
+
// Agents section
const agents_section = this._collapsible_section("Compute Agents");
this._agents_host = agents_section;
@@ -50,11 +58,12 @@ export class Page extends ZenPage
{
try
{
- const [agents, history, clients, client_history] = await Promise.all([
+ const [agents, history, clients, client_history, prov] = await Promise.all([
new Fetcher().resource("/orch/agents").json(),
new Fetcher().resource("/orch/history").param("limit", "50").json().catch(() => null),
new Fetcher().resource("/orch/clients").json().catch(() => null),
new Fetcher().resource("/orch/clients/history").param("limit", "50").json().catch(() => null),
+ new Fetcher().resource("/orch/provisioner/status").json().catch(() => null),
]);
this._render_agents(agents);
@@ -70,6 +79,7 @@ export class Page extends ZenPage
{
this._render_client_history(client_history.client_events || []);
}
+ this._render_provisioner(prov);
}
catch (e) { /* service unavailable */ }
}
@@ -109,6 +119,7 @@ export class Page extends ZenPage
{
this._render_client_history(data.client_events);
}
+ this._render_provisioner(data.provisioner);
}
catch (e) { /* ignore parse errors */ }
};
@@ -156,7 +167,7 @@ export class Page extends ZenPage
return;
}
- let totalCpus = 0, totalWeightedCpu = 0;
+ let totalCpus = 0, activeCpus = 0, totalWeightedCpu = 0;
let totalMemUsed = 0, totalMemTotal = 0;
let totalQueues = 0, totalPending = 0, totalRunning = 0, totalCompleted = 0;
let totalRecv = 0, totalSent = 0;
@@ -173,8 +184,14 @@ export class Page extends ZenPage
const completed = w.actions_completed || 0;
const recv = w.bytes_received || 0;
const sent = w.bytes_sent || 0;
+ const provisioner = w.provisioner || "";
+ const isProvisioned = provisioner !== "";
totalCpus += cpus;
+ if (w.provisioner_status === "active")
+ {
+ activeCpus += cpus;
+ }
if (cpus > 0 && typeof cpuUsage === "number")
{
totalWeightedCpu += cpuUsage * cpus;
@@ -209,12 +226,49 @@ export class Page extends ZenPage
cell.inner().textContent = "";
cell.tag("a").text(hostname).attr("href", w.uri + "/dashboard/compute/").attr("target", "_blank");
}
+
+ // Visual treatment based on provisioner status
+ const provStatus = w.provisioner_status || "";
+ if (!isProvisioned)
+ {
+ row.inner().style.opacity = "0.45";
+ }
+ else
+ {
+ const hostCell = row.get_cell(0);
+ const el = hostCell.inner();
+ const badge = document.createElement("span");
+ const badgeBase = "display:inline-block;margin-left:6px;padding:1px 5px;border-radius:8px;" +
+ "font-size:9px;font-weight:600;color:#fff;vertical-align:middle;";
+
+ if (provStatus === "draining")
+ {
+ badge.textContent = "draining";
+ badge.style.cssText = badgeBase + "background:var(--theme_warn);";
+ row.inner().style.opacity = "0.6";
+ }
+ else if (provStatus === "active")
+ {
+ badge.textContent = provisioner;
+ badge.style.cssText = badgeBase + "background:#8957e5;";
+ }
+ else
+ {
+ badge.textContent = "deallocated";
+ badge.style.cssText = badgeBase + "background:var(--theme_fail);";
+ row.inner().style.opacity = "0.45";
+ }
+ el.appendChild(badge);
+ }
}
- // Total row
+ // Total row — show active / total in CPUs column
+ const cpuLabel = activeCpus < totalCpus
+ ? Friendly.sep(activeCpus) + " / " + Friendly.sep(totalCpus)
+ : Friendly.sep(totalCpus);
const total = this._agents_table.add_row(
"TOTAL",
- Friendly.sep(totalCpus),
+ cpuLabel,
"",
totalMemTotal > 0 ? Friendly.bytes(totalMemUsed) + " / " + Friendly.bytes(totalMemTotal) : "-",
Friendly.sep(totalQueues),
@@ -305,6 +359,154 @@ export class Page extends ZenPage
}
}
+ _render_provisioner(prov)
+ {
+ const container = this._prov_section._parent.inner();
+
+ if (!prov || !prov.name)
+ {
+ container.style.display = "none";
+ return;
+ }
+ container.style.display = "";
+
+ if (!this._prov_grid)
+ {
+ this._prov_grid = this._prov_section.tag().classify("grid").classify("stats-tiles");
+ this._prov_tiles = {};
+
+ // Target cores tile with editable input
+ const target_tile = this._prov_grid.tag().classify("card").classify("stats-tile");
+ target_tile.tag().classify("card-title").text("Target Cores");
+ const target_body = target_tile.tag().classify("tile-metrics");
+ const target_m = target_body.tag().classify("tile-metric").classify("tile-metric-hero");
+ const input = document.createElement("input");
+ input.type = "number";
+ input.min = "0";
+ input.style.cssText = "width:100px;padding:4px 8px;border:1px solid var(--theme_g2);border-radius:4px;" +
+ "background:var(--theme_g4);color:var(--theme_bright);font-size:20px;font-weight:600;text-align:right;";
+ target_m.inner().appendChild(input);
+ target_m.tag().classify("metric-label").text("target");
+ this._prov_tiles.target_input = input;
+
+ input.addEventListener("focus", () => { this._prov_target_dirty = true; });
+ input.addEventListener("input", () => {
+ this._prov_target_dirty = true;
+ if (this._prov_commit_timer)
+ {
+ clearTimeout(this._prov_commit_timer);
+ }
+ this._prov_commit_timer = setTimeout(() => this._commit_provisioner_target(), 800);
+ });
+ input.addEventListener("keydown", (e) => {
+ if (e.key === "Enter")
+ {
+ if (this._prov_commit_timer)
+ {
+ clearTimeout(this._prov_commit_timer);
+ }
+ this._commit_provisioner_target();
+ input.blur();
+ }
+ });
+ input.addEventListener("blur", () => {
+ if (this._prov_commit_timer)
+ {
+ clearTimeout(this._prov_commit_timer);
+ }
+ this._commit_provisioner_target();
+ });
+
+ // Active cores
+ const active_tile = this._prov_grid.tag().classify("card").classify("stats-tile");
+ active_tile.tag().classify("card-title").text("Active Cores");
+ const active_body = active_tile.tag().classify("tile-metrics");
+ this._prov_tiles.active = active_body;
+
+ // Estimated cores
+ const est_tile = this._prov_grid.tag().classify("card").classify("stats-tile");
+ est_tile.tag().classify("card-title").text("Estimated Cores");
+ const est_body = est_tile.tag().classify("tile-metrics");
+ this._prov_tiles.estimated = est_body;
+
+ // Agents
+ const agents_tile = this._prov_grid.tag().classify("card").classify("stats-tile");
+ agents_tile.tag().classify("card-title").text("Agents");
+ const agents_body = agents_tile.tag().classify("tile-metrics");
+ this._prov_tiles.agents = agents_body;
+
+ // Draining
+ const drain_tile = this._prov_grid.tag().classify("card").classify("stats-tile");
+ drain_tile.tag().classify("card-title").text("Draining");
+ const drain_body = drain_tile.tag().classify("tile-metrics");
+ this._prov_tiles.draining = drain_body;
+ }
+
+ // Update values
+ const input = this._prov_tiles.target_input;
+ if (!this._prov_target_dirty && document.activeElement !== input)
+ {
+ input.value = prov.target_cores;
+ }
+ this._prov_last_target = prov.target_cores;
+
+ // Re-render metric tiles (clear and recreate content)
+ for (const key of ["active", "estimated", "agents", "draining"])
+ {
+ this._prov_tiles[key].inner().innerHTML = "";
+ }
+ this._metric(this._prov_tiles.active, Friendly.sep(prov.active_cores), "cores", true);
+ this._metric(this._prov_tiles.estimated, Friendly.sep(prov.estimated_cores), "cores", true);
+ this._metric(this._prov_tiles.agents, Friendly.sep(prov.agents), "active", true);
+ this._metric(this._prov_tiles.draining, Friendly.sep(prov.agents_draining || 0), "agents", true);
+ }
+
+ async _commit_provisioner_target()
+ {
+ const input = this._prov_tiles?.target_input;
+ if (!input || this._prov_committing)
+ {
+ return;
+ }
+ const value = parseInt(input.value, 10);
+ if (isNaN(value) || value < 0)
+ {
+ return;
+ }
+ if (value === this._prov_last_target)
+ {
+ this._prov_target_dirty = false;
+ return;
+ }
+ this._prov_committing = true;
+ try
+ {
+ const resp = await fetch("/orch/provisioner/target", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ target_cores: value }),
+ });
+ if (resp.ok)
+ {
+ this._prov_target_dirty = false;
+ console.log("Target cores set to", value);
+ }
+ else
+ {
+ const text = await resp.text();
+ console.error("Failed to set target cores: HTTP", resp.status, text);
+ }
+ }
+ catch (e)
+ {
+ console.error("Failed to set target cores:", e);
+ }
+ finally
+ {
+ this._prov_committing = false;
+ }
+ }
+
_metric(parent, value, label, hero = false)
{
const m = parent.tag().classify("tile-metric");
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index c5f8724ca..108685eb9 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -14,7 +14,6 @@
#include <zencore/memory/memorytrace.h>
#include <zencore/memory/newdelete.h>
#include <zencore/scopeguard.h>
-#include <zencore/sentryintegration.h>
#include <zencore/session.h>
#include <zencore/string.h>
#include <zencore/thread.h>
@@ -169,7 +168,12 @@ AppMain(int argc, char* argv[])
if (IsDir(ServerOptions.DataDir))
{
ZEN_CONSOLE_INFO("Deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason);
- DeleteDirectories(ServerOptions.DataDir);
+ std::error_code Ec;
+ DeleteDirectories(ServerOptions.DataDir, Ec);
+ if (Ec)
+ {
+ ZEN_WARN("could not fully clean '{}': {} (continuing anyway)", ServerOptions.DataDir, Ec.message());
+ }
}
}