aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2026-03-04 14:13:46 +0100
committerGitHub Enterprise <[email protected]>2026-03-04 14:13:46 +0100
commit0763d09a81e5a1d3df11763a7ec75e7860c9510a (patch)
tree074575ba6ea259044a179eab0bb396d37268fb09 /src/zenserver
parentnative xmake toolchain definition for UE-clang (#805) (diff)
downloadzen-0763d09a81e5a1d3df11763a7ec75e7860c9510a.tar.xz
zen-0763d09a81e5a1d3df11763a7ec75e7860c9510a.zip
compute orchestration (#763)
- Added local process runners for Linux/Wine, Mac with some sandboxing support - Horde & Nomad provisioning for development and testing - Client session queues with lifecycle management (active/draining/cancelled), automatic retry with configurable limits, and manual reschedule API - Improved web UI for orchestrator, compute, and hub dashboards with WebSocket push updates - Some security hardening - Improved scalability and `zen exec` command Still experimental - compute support is disabled by default
Diffstat (limited to 'src/zenserver')
-rw-r--r--src/zenserver/compute/computeserver.cpp725
-rw-r--r--src/zenserver/compute/computeserver.h111
-rw-r--r--src/zenserver/compute/computeservice.cpp100
-rw-r--r--src/zenserver/compute/computeservice.h36
-rw-r--r--src/zenserver/frontend/html.zipbin238188 -> 319315 bytes
-rw-r--r--src/zenserver/frontend/html/404.html486
-rw-r--r--src/zenserver/frontend/html/compute/banner.js321
-rw-r--r--src/zenserver/frontend/html/compute/compute.html (renamed from src/zenserver/frontend/html/compute.html)181
-rw-r--r--src/zenserver/frontend/html/compute/hub.html310
-rw-r--r--src/zenserver/frontend/html/compute/index.html1
-rw-r--r--src/zenserver/frontend/html/compute/nav.js79
-rw-r--r--src/zenserver/frontend/html/compute/orchestrator.html831
-rw-r--r--src/zenserver/frontend/html/pages/page.js36
-rw-r--r--src/zenserver/frontend/html/zen.css27
-rw-r--r--src/zenserver/hub/hubservice.cpp2
-rw-r--r--src/zenserver/hub/zenhubserver.cpp7
-rw-r--r--src/zenserver/hub/zenhubserver.h6
-rw-r--r--src/zenserver/storage/zenstorageserver.cpp17
-rw-r--r--src/zenserver/storage/zenstorageserver.h4
-rw-r--r--src/zenserver/trace/tracerecorder.cpp565
-rw-r--r--src/zenserver/trace/tracerecorder.h46
-rw-r--r--src/zenserver/xmake.lua19
22 files changed, 3683 insertions, 227 deletions
diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp
index 0f9ef0287..802d06caf 100644
--- a/src/zenserver/compute/computeserver.cpp
+++ b/src/zenserver/compute/computeserver.cpp
@@ -1,9 +1,9 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include "computeserver.h"
-#include <zencompute/httpfunctionservice.h>
-#include "computeservice.h"
-
+#include <zencompute/cloudmetadata.h>
+#include <zencompute/httpcomputeservice.h>
+#include <zencompute/httporchestrator.h>
#if ZEN_WITH_COMPUTE_SERVICES
# include <zencore/fmtutils.h>
@@ -13,10 +13,20 @@
# include <zencore/scopeguard.h>
# include <zencore/sentryintegration.h>
# include <zencore/system.h>
+# include <zencore/compactbinarybuilder.h>
# include <zencore/windows.h>
+# include <zenhttp/httpclient.h>
# include <zenhttp/httpapiservice.h>
# include <zenstore/cidstore.h>
# include <zenutil/service.h>
+# if ZEN_WITH_HORDE
+# include <zenhorde/hordeconfig.h>
+# include <zenhorde/hordeprovisioner.h>
+# endif
+# if ZEN_WITH_NOMAD
+# include <zennomad/nomadconfig.h>
+# include <zennomad/nomadprovisioner.h>
+# endif
ZEN_THIRD_PARTY_INCLUDES_START
# include <cxxopts.hpp>
@@ -29,6 +39,13 @@ ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options)
{
Options.add_option("compute",
"",
+ "max-actions",
+ "Maximum number of concurrent local actions (0 = auto)",
+ cxxopts::value<int32_t>(m_ServerOptions.MaxConcurrentActions)->default_value("0"),
+ "");
+
+ Options.add_option("compute",
+ "",
"upstream-notification-endpoint",
"Endpoint URL for upstream notifications",
cxxopts::value<std::string>(m_ServerOptions.UpstreamNotificationEndpoint)->default_value(""),
@@ -40,6 +57,236 @@ ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options)
"Instance ID for use in notifications",
cxxopts::value<std::string>(m_ServerOptions.InstanceId)->default_value(""),
"");
+
+ Options.add_option("compute",
+ "",
+ "coordinator-endpoint",
+ "Endpoint URL for coordinator service",
+ cxxopts::value<std::string>(m_ServerOptions.CoordinatorEndpoint)->default_value(""),
+ "");
+
+ Options.add_option("compute",
+ "",
+ "idms",
+ "Enable IDMS cloud detection; optionally specify a custom probe endpoint",
+ cxxopts::value<std::string>(m_ServerOptions.IdmsEndpoint)->default_value("")->implicit_value("auto"),
+ "");
+
+ Options.add_option("compute",
+ "",
+ "worker-websocket",
+ "Use WebSocket for worker-orchestrator link (instant reachability detection)",
+ cxxopts::value<bool>(m_ServerOptions.EnableWorkerWebSocket)->default_value("false"),
+ "");
+
+# if ZEN_WITH_HORDE
+ // Horde provisioning options
+ Options.add_option("horde",
+ "",
+ "horde-enabled",
+ "Enable Horde worker provisioning",
+ cxxopts::value<bool>(m_ServerOptions.HordeConfig.Enabled)->default_value("false"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-server",
+ "Horde server URL",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.ServerUrl)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-token",
+ "Horde authentication token",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.AuthToken)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-pool",
+ "Horde pool name",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.Pool)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-cluster",
+ "Horde cluster ID ('default' or '_auto' for auto-resolve)",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.Cluster)->default_value("default"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-mode",
+ "Horde connection mode (direct, tunnel, relay)",
+ cxxopts::value<std::string>(m_HordeModeStr)->default_value("direct"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-encryption",
+ "Horde transport encryption (none, aes)",
+ cxxopts::value<std::string>(m_HordeEncryptionStr)->default_value("none"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-max-cores",
+ "Maximum number of Horde cores to provision",
+ cxxopts::value<int>(m_ServerOptions.HordeConfig.MaxCores)->default_value("2048"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-host",
+ "Host address for Horde agents to connect back to",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.HostAddress)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-condition",
+ "Additional Horde agent filter condition",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.Condition)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-binaries",
+ "Path to directory containing zenserver binary for remote upload",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.BinariesPath)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-zen-service-port",
+ "Port number for Zen service communication",
+ cxxopts::value<uint16_t>(m_ServerOptions.HordeConfig.ZenServicePort)->default_value("8558"),
+ "");
+# endif
+
+# if ZEN_WITH_NOMAD
+ // Nomad provisioning options
+ Options.add_option("nomad",
+ "",
+ "nomad-enabled",
+ "Enable Nomad worker provisioning",
+ cxxopts::value<bool>(m_ServerOptions.NomadConfig.Enabled)->default_value("false"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-server",
+ "Nomad HTTP API URL",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.ServerUrl)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-token",
+ "Nomad ACL token",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.AclToken)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-datacenter",
+ "Nomad target datacenter",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.Datacenter)->default_value("dc1"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-namespace",
+ "Nomad namespace",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.Namespace)->default_value("default"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-region",
+ "Nomad region (empty for server default)",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.Region)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-driver",
+ "Nomad task driver (raw_exec, docker)",
+ cxxopts::value<std::string>(m_NomadDriverStr)->default_value("raw_exec"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-distribution",
+ "Binary distribution mode (predeployed, artifact)",
+ cxxopts::value<std::string>(m_NomadDistributionStr)->default_value("predeployed"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-binary-path",
+ "Path to zenserver on Nomad clients (predeployed mode)",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.BinaryPath)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-artifact-source",
+ "URL to download zenserver binary (artifact mode)",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.ArtifactSource)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-docker-image",
+ "Docker image for zenserver (docker driver)",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.DockerImage)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-max-jobs",
+ "Maximum concurrent Nomad jobs",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.MaxJobs)->default_value("64"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-cpu-mhz",
+ "CPU MHz allocated per Nomad task",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.CpuMhz)->default_value("1000"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-memory-mb",
+ "Memory MB allocated per Nomad task",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.MemoryMb)->default_value("2048"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-cores-per-job",
+ "Estimated cores per Nomad job (for scaling)",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.CoresPerJob)->default_value("32"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-max-cores",
+ "Maximum total cores to provision via Nomad",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.MaxCores)->default_value("2048"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-job-prefix",
+ "Prefix for generated Nomad job IDs",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.JobPrefix)->default_value("zenserver-worker"),
+ "");
+# endif
}
void
@@ -63,6 +310,15 @@ ZenComputeServerConfigurator::OnConfigFileParsed(LuaConfig::Options& LuaOptions)
void
ZenComputeServerConfigurator::ValidateOptions()
{
+# if ZEN_WITH_HORDE
+ horde::FromString(m_ServerOptions.HordeConfig.Mode, m_HordeModeStr);
+ horde::FromString(m_ServerOptions.HordeConfig.EncryptionMode, m_HordeEncryptionStr);
+# endif
+
+# if ZEN_WITH_NOMAD
+ nomad::FromString(m_ServerOptions.NomadConfig.TaskDriver, m_NomadDriverStr);
+ nomad::FromString(m_ServerOptions.NomadConfig.BinDistribution, m_NomadDistributionStr);
+# endif
}
///////////////////////////////////////////////////////////////////////////
@@ -90,10 +346,14 @@ ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServ
return EffectiveBasePort;
}
+ m_CoordinatorEndpoint = ServerConfig.CoordinatorEndpoint;
+ m_InstanceId = ServerConfig.InstanceId;
+ m_EnableWorkerWebSocket = ServerConfig.EnableWorkerWebSocket;
+
// This is a workaround to make sure we can have automated tests. Without
// this the ranges for different child zen compute processes could overlap with
// the main test range.
- ZenServerEnvironment::SetBaseChildId(1000);
+ ZenServerEnvironment::SetBaseChildId(2000);
m_DebugOptionForcedCrash = ServerConfig.ShouldCrash;
@@ -113,6 +373,46 @@ ZenComputeServer::Cleanup()
ZEN_INFO(ZEN_APP_NAME " cleaning up");
try
{
+ // Cancel the maintenance timer so it stops re-enqueuing before we
+ // tear down the provisioners it references.
+ m_ProvisionerMaintenanceTimer.cancel();
+ m_AnnounceTimer.cancel();
+
+# if ZEN_WITH_HORDE
+ // Shut down Horde provisioner first — this signals all agent threads
+ // to exit and joins them before we tear down HTTP services.
+ m_HordeProvisioner.reset();
+# endif
+
+# if ZEN_WITH_NOMAD
+ // Shut down Nomad provisioner — stops the management thread and
+ // sends stop requests for all tracked jobs.
+ m_NomadProvisioner.reset();
+# endif
+
+ // Close the orchestrator WebSocket client before stopping the io_context
+ m_WsReconnectTimer.cancel();
+ if (m_OrchestratorWsClient)
+ {
+ m_OrchestratorWsClient->Close();
+ m_OrchestratorWsClient.reset();
+ }
+ m_OrchestratorWsHandler.reset();
+
+ ResolveCloudMetadata();
+ m_CloudMetadata.reset();
+
+ // Shut down services that own threads or use the io_context before we
+ // stop the io_context and close the HTTP server.
+ if (m_OrchestratorService)
+ {
+ m_OrchestratorService->Shutdown();
+ }
+ if (m_ComputeService)
+ {
+ m_ComputeService->Shutdown();
+ }
+
m_IoContext.stop();
if (m_IoRunner.joinable())
{
@@ -139,7 +439,8 @@ ZenComputeServer::InitializeState(const ZenComputeServerConfig& ServerConfig)
void
ZenComputeServer::InitializeServices(const ZenComputeServerConfig& ServerConfig)
{
- ZEN_INFO("initializing storage");
+ ZEN_TRACE_CPU("ZenComputeServer::InitializeServices");
+ ZEN_INFO("initializing compute services");
CidStoreConfiguration Config;
Config.RootDirectory = m_DataRoot / "cas";
@@ -147,46 +448,405 @@ ZenComputeServer::InitializeServices(const ZenComputeServerConfig& ServerConfig)
m_CidStore = std::make_unique<CidStore>(m_GcManager);
m_CidStore->Initialize(Config);
+ if (!ServerConfig.IdmsEndpoint.empty())
+ {
+ ZEN_INFO("detecting cloud environment (async)");
+ if (ServerConfig.IdmsEndpoint == "auto")
+ {
+ m_CloudMetadataFuture = std::async(std::launch::async, [DataDir = ServerConfig.DataDir] {
+ return std::make_unique<zen::compute::CloudMetadata>(DataDir / "cloud");
+ });
+ }
+ else
+ {
+ ZEN_INFO("using custom IDMS endpoint: {}", ServerConfig.IdmsEndpoint);
+ m_CloudMetadataFuture = std::async(std::launch::async, [DataDir = ServerConfig.DataDir, Endpoint = ServerConfig.IdmsEndpoint] {
+ return std::make_unique<zen::compute::CloudMetadata>(DataDir / "cloud", Endpoint);
+ });
+ }
+ }
+
ZEN_INFO("instantiating API service");
m_ApiService = std::make_unique<zen::HttpApiService>(*m_Http);
- ZEN_INFO("instantiating compute service");
- m_ComputeService = std::make_unique<HttpComputeService>(ServerConfig.DataDir / "compute");
+ ZEN_INFO("instantiating orchestrator service");
+ m_OrchestratorService =
+ std::make_unique<zen::compute::HttpOrchestratorService>(ServerConfig.DataDir / "orch", ServerConfig.EnableWorkerWebSocket);
+
+ ZEN_INFO("instantiating function service");
+ m_ComputeService = std::make_unique<zen::compute::HttpComputeService>(*m_CidStore,
+ m_StatsService,
+ ServerConfig.DataDir / "functions",
+ ServerConfig.MaxConcurrentActions);
- // Ref<zen::compute::FunctionRunner> Runner;
- // Runner = zen::compute::CreateLocalRunner(*m_CidStore, ServerConfig.DataDir / "runner");
+ m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
- // TODO: (re)implement default configuration here
+# if ZEN_WITH_NOMAD
+ // Nomad provisioner
+ if (ServerConfig.NomadConfig.Enabled && !ServerConfig.NomadConfig.ServerUrl.empty())
+ {
+ ZEN_INFO("instantiating Nomad provisioner (server: {})", ServerConfig.NomadConfig.ServerUrl);
- ZEN_INFO("instantiating function service");
- m_FunctionService =
- std::make_unique<zen::compute::HttpFunctionService>(*m_CidStore, m_StatsService, ServerConfig.DataDir / "functions");
+ const auto& NomadCfg = ServerConfig.NomadConfig;
+
+ if (!NomadCfg.Validate())
+ {
+ ZEN_ERROR("invalid Nomad configuration");
+ }
+ else
+ {
+ ExtendableStringBuilder<256> OrchestratorEndpoint;
+ OrchestratorEndpoint << m_Http->GetServiceUri(m_OrchestratorService.get());
+ if (auto View = OrchestratorEndpoint.ToView(); !View.empty() && View.back() != '/')
+ {
+ OrchestratorEndpoint << '/';
+ }
+
+ m_NomadProvisioner = std::make_unique<nomad::NomadProvisioner>(NomadCfg, OrchestratorEndpoint);
+ }
+ }
+# endif
+
+# if ZEN_WITH_HORDE
+ // Horde provisioner
+ if (ServerConfig.HordeConfig.Enabled && !ServerConfig.HordeConfig.ServerUrl.empty())
+ {
+ ZEN_INFO("instantiating Horde provisioner (server: {})", ServerConfig.HordeConfig.ServerUrl);
+
+ const auto& HordeConfig = ServerConfig.HordeConfig;
+
+ if (!HordeConfig.Validate())
+ {
+ ZEN_ERROR("invalid Horde configuration");
+ }
+ else
+ {
+ ExtendableStringBuilder<256> OrchestratorEndpoint;
+ OrchestratorEndpoint << m_Http->GetServiceUri(m_OrchestratorService.get());
+ if (auto View = OrchestratorEndpoint.ToView(); !View.empty() && View.back() != '/')
+ {
+ OrchestratorEndpoint << '/';
+ }
+
+ // If no binaries path is specified, just use the running executable's directory
+ std::filesystem::path BinariesPath = HordeConfig.BinariesPath.empty() ? GetRunningExecutablePath().parent_path()
+ : std::filesystem::path(HordeConfig.BinariesPath);
+ std::filesystem::path WorkingDir = ServerConfig.DataDir / "horde";
+
+ m_HordeProvisioner = std::make_unique<horde::HordeProvisioner>(HordeConfig, BinariesPath, WorkingDir, OrchestratorEndpoint);
+ }
+ }
+# endif
+}
+
+void
+ZenComputeServer::ResolveCloudMetadata()
+{
+ if (m_CloudMetadataFuture.valid())
+ {
+ m_CloudMetadata = m_CloudMetadataFuture.get();
+ }
+}
+
+std::string
+ZenComputeServer::GetInstanceId() const
+{
+ if (!m_InstanceId.empty())
+ {
+ return m_InstanceId;
+ }
+ return fmt::format("{}-{}", GetMachineName(), GetCurrentProcessId());
+}
+
+std::string
+ZenComputeServer::GetAnnounceUrl() const
+{
+ return m_Http->GetServiceUri(nullptr);
}
void
ZenComputeServer::RegisterServices(const ZenComputeServerConfig& ServerConfig)
{
+ ZEN_TRACE_CPU("ZenComputeServer::RegisterServices");
ZEN_UNUSED(ServerConfig);
+ m_Http->RegisterService(m_StatsService);
+
+ if (m_ApiService)
+ {
+ m_Http->RegisterService(*m_ApiService);
+ }
+
+ if (m_OrchestratorService)
+ {
+ m_Http->RegisterService(*m_OrchestratorService);
+ }
+
if (m_ComputeService)
{
m_Http->RegisterService(*m_ComputeService);
}
- if (m_ApiService)
+ if (m_FrontendService)
{
- m_Http->RegisterService(*m_ApiService);
+ m_Http->RegisterService(*m_FrontendService);
+ }
+}
+
+CbObject
+ZenComputeServer::BuildAnnounceBody()
+{
+ CbObjectWriter AnnounceBody;
+ AnnounceBody << "id" << GetInstanceId();
+ AnnounceBody << "uri" << GetAnnounceUrl();
+ AnnounceBody << "hostname" << GetMachineName();
+ AnnounceBody << "platform" << GetRuntimePlatformName();
+
+ ExtendedSystemMetrics Sm = ApplyReportingOverrides(m_MetricsTracker.Query());
+
+ AnnounceBody.BeginObject("metrics");
+ Describe(Sm, AnnounceBody);
+ AnnounceBody.EndObject();
+
+ AnnounceBody << "cpu_usage" << Sm.CpuUsagePercent;
+ AnnounceBody << "memory_total" << Sm.SystemMemoryMiB * 1024 * 1024;
+ AnnounceBody << "memory_used" << (Sm.SystemMemoryMiB - Sm.AvailSystemMemoryMiB) * 1024 * 1024;
+
+ AnnounceBody << "bytes_received" << m_Http->GetTotalBytesReceived();
+ AnnounceBody << "bytes_sent" << m_Http->GetTotalBytesSent();
+
+ auto Actions = m_ComputeService->GetActionCounts();
+ AnnounceBody << "actions_pending" << Actions.Pending;
+ AnnounceBody << "actions_running" << Actions.Running;
+ AnnounceBody << "actions_completed" << Actions.Completed;
+ AnnounceBody << "active_queues" << Actions.ActiveQueues;
+
+ // Derive provisioner from instance ID prefix (e.g. "horde-xxx" or "nomad-xxx")
+ if (m_InstanceId.starts_with("horde-"))
+ {
+ AnnounceBody << "provisioner"
+ << "horde";
+ }
+ else if (m_InstanceId.starts_with("nomad-"))
+ {
+ AnnounceBody << "provisioner"
+ << "nomad";
+ }
+
+ ResolveCloudMetadata();
+ if (m_CloudMetadata)
+ {
+ m_CloudMetadata->Describe(AnnounceBody);
+ }
+
+ return AnnounceBody.Save();
+}
+
+void
+ZenComputeServer::PostAnnounce()
+{
+ ZEN_TRACE_CPU("ZenComputeServer::PostAnnounce");
+
+ if (!m_ComputeService || m_CoordinatorEndpoint.empty())
+ {
+ return;
+ }
+
+ ZEN_INFO("notifying coordinator at '{}' of our availability at '{}'", m_CoordinatorEndpoint, GetAnnounceUrl());
+
+ try
+ {
+ CbObject Body = BuildAnnounceBody();
+
+ // If we have an active WebSocket connection, send via that instead of HTTP POST
+ if (m_OrchestratorWsClient && m_OrchestratorWsClient->IsOpen())
+ {
+ MemoryView View = Body.GetView();
+ m_OrchestratorWsClient->SendBinary(std::span<const uint8_t>(reinterpret_cast<const uint8_t*>(View.GetData()), View.GetSize()));
+ ZEN_INFO("announced to coordinator via WebSocket");
+ return;
+ }
+
+ HttpClient CoordinatorHttp(m_CoordinatorEndpoint);
+ HttpClient::Response Result = CoordinatorHttp.Post("announce", std::move(Body));
+
+ if (Result.Error)
+ {
+ ZEN_ERROR("failed to notify coordinator at '{}': HTTP error {} - {}",
+ m_CoordinatorEndpoint,
+ Result.Error->ErrorCode,
+ Result.Error->ErrorMessage);
+ }
+ else if (!IsHttpOk(Result.StatusCode))
+ {
+ ZEN_ERROR("failed to notify coordinator at '{}': unexpected HTTP status code {}",
+ m_CoordinatorEndpoint,
+ static_cast<int>(Result.StatusCode));
+ }
+ else
+ {
+ ZEN_INFO("successfully notified coordinator at '{}'", m_CoordinatorEndpoint);
+ }
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("failed to notify coordinator at '{}': {}", m_CoordinatorEndpoint, Ex.what());
+ }
+}
+
+void
+ZenComputeServer::EnqueueAnnounceTimer()
+{
+ if (!m_ComputeService || m_CoordinatorEndpoint.empty())
+ {
+ return;
+ }
+
+ m_AnnounceTimer.expires_after(std::chrono::seconds(15));
+ m_AnnounceTimer.async_wait([this](const asio::error_code& Ec) {
+ if (!Ec)
+ {
+ PostAnnounce();
+ EnqueueAnnounceTimer();
+ }
+ });
+ EnsureIoRunner();
+}
+
+void
+ZenComputeServer::InitializeOrchestratorWebSocket()
+{
+ if (!m_EnableWorkerWebSocket || m_CoordinatorEndpoint.empty())
+ {
+ return;
+ }
+
+ // Convert http://host:port → ws://host:port/orch/ws
+ std::string WsUrl = m_CoordinatorEndpoint;
+ if (WsUrl.starts_with("http://"))
+ {
+ WsUrl = "ws://" + WsUrl.substr(7);
+ }
+ else if (WsUrl.starts_with("https://"))
+ {
+ WsUrl = "wss://" + WsUrl.substr(8);
+ }
+ if (!WsUrl.empty() && WsUrl.back() != '/')
+ {
+ WsUrl += '/';
+ }
+ WsUrl += "orch/ws";
+
+ ZEN_INFO("establishing WebSocket link to orchestrator at {}", WsUrl);
+
+ m_OrchestratorWsHandler = std::make_unique<OrchestratorWsHandler>(*this);
+ m_OrchestratorWsClient =
+ std::make_unique<HttpWsClient>(WsUrl, *m_OrchestratorWsHandler, m_IoContext, HttpWsClientSettings{.LogCategory = "orch_ws"});
+
+ m_OrchestratorWsClient->Connect();
+ EnsureIoRunner();
+}
+
+void
+ZenComputeServer::EnqueueWsReconnect()
+{
+ m_WsReconnectTimer.expires_after(std::chrono::seconds(5));
+ m_WsReconnectTimer.async_wait([this](const asio::error_code& Ec) {
+ if (!Ec && m_OrchestratorWsClient)
+ {
+ ZEN_INFO("attempting WebSocket reconnect to orchestrator");
+ m_OrchestratorWsClient->Connect();
+ }
+ });
+ EnsureIoRunner();
+}
+
+void
+ZenComputeServer::OrchestratorWsHandler::OnWsOpen()
+{
+ ZEN_INFO("WebSocket link to orchestrator established");
+
+ // Send initial announce immediately over the WebSocket
+ Server.PostAnnounce();
+}
+
+void
+ZenComputeServer::OrchestratorWsHandler::OnWsMessage([[maybe_unused]] const WebSocketMessage& Msg)
+{
+ // Orchestrator does not push messages to workers; ignore
+}
+
+void
+ZenComputeServer::OrchestratorWsHandler::OnWsClose([[maybe_unused]] uint16_t Code, [[maybe_unused]] std::string_view Reason)
+{
+ ZEN_WARN("WebSocket link to orchestrator closed (code {}), falling back to HTTP announce", Code);
+
+ // Trigger an immediate HTTP announce so the orchestrator has fresh state,
+ // then schedule a reconnect attempt.
+ Server.PostAnnounce();
+ Server.EnqueueWsReconnect();
+}
+
+void
+ZenComputeServer::ProvisionerMaintenanceTick()
+{
+# if ZEN_WITH_HORDE
+ if (m_HordeProvisioner)
+ {
+ m_HordeProvisioner->SetTargetCoreCount(UINT32_MAX);
+ auto Stats = m_HordeProvisioner->GetStats();
+ ZEN_DEBUG("Horde maintenance: target={}, estimated={}, active={}",
+ Stats.TargetCoreCount,
+ Stats.EstimatedCoreCount,
+ Stats.ActiveCoreCount);
+ }
+# endif
+
+# if ZEN_WITH_NOMAD
+ if (m_NomadProvisioner)
+ {
+ m_NomadProvisioner->SetTargetCoreCount(UINT32_MAX);
+ auto Stats = m_NomadProvisioner->GetStats();
+ ZEN_DEBUG("Nomad maintenance: target={}, estimated={}, running jobs={}",
+ Stats.TargetCoreCount,
+ Stats.EstimatedCoreCount,
+ Stats.RunningJobCount);
}
+# endif
+}
+
+void
+ZenComputeServer::EnqueueProvisionerMaintenanceTimer()
+{
+ bool HasProvisioner = false;
+# if ZEN_WITH_HORDE
+ HasProvisioner = HasProvisioner || (m_HordeProvisioner != nullptr);
+# endif
+# if ZEN_WITH_NOMAD
+ HasProvisioner = HasProvisioner || (m_NomadProvisioner != nullptr);
+# endif
- if (m_FunctionService)
+ if (!HasProvisioner)
{
- m_Http->RegisterService(*m_FunctionService);
+ return;
}
+
+ m_ProvisionerMaintenanceTimer.expires_after(std::chrono::seconds(15));
+ m_ProvisionerMaintenanceTimer.async_wait([this](const asio::error_code& Ec) {
+ if (!Ec)
+ {
+ ProvisionerMaintenanceTick();
+ EnqueueProvisionerMaintenanceTimer();
+ }
+ });
+ EnsureIoRunner();
}
void
ZenComputeServer::Run()
{
+ ZEN_TRACE_CPU("ZenComputeServer::Run");
+
if (m_ProcessMonitor.IsActive())
{
CheckOwnerPid();
@@ -236,6 +896,35 @@ ZenComputeServer::Run()
OnReady();
+ PostAnnounce();
+ EnqueueAnnounceTimer();
+ InitializeOrchestratorWebSocket();
+
+# if ZEN_WITH_HORDE
+ // Start Horde provisioning if configured — request maximum allowed cores.
+ // SetTargetCoreCount clamps to HordeConfig::MaxCores internally.
+ if (m_HordeProvisioner)
+ {
+ ZEN_INFO("Horde provisioning starting");
+ m_HordeProvisioner->SetTargetCoreCount(UINT32_MAX);
+ auto Stats = m_HordeProvisioner->GetStats();
+ ZEN_INFO("Horde provisioning started (target cores: {})", Stats.TargetCoreCount);
+ }
+# endif
+
+# if ZEN_WITH_NOMAD
+ // Start Nomad provisioning if configured — request maximum allowed cores.
+ // SetTargetCoreCount clamps to NomadConfig::MaxCores internally.
+ if (m_NomadProvisioner)
+ {
+ m_NomadProvisioner->SetTargetCoreCount(UINT32_MAX);
+ auto Stats = m_NomadProvisioner->GetStats();
+ ZEN_INFO("Nomad provisioning started (target cores: {})", Stats.TargetCoreCount);
+ }
+# endif
+
+ EnqueueProvisionerMaintenanceTimer();
+
m_Http->Run(IsInteractiveMode);
SetNewState(kShuttingDown);
@@ -254,6 +943,8 @@ ZenComputeServerMain::ZenComputeServerMain(ZenComputeServerConfig& ServerOptions
void
ZenComputeServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
{
+ ZEN_TRACE_CPU("ZenComputeServerMain::DoRun");
+
ZenComputeServer Server;
Server.SetDataRoot(m_ServerOptions.DataDir);
Server.SetContentRoot(m_ServerOptions.ContentDir);
diff --git a/src/zenserver/compute/computeserver.h b/src/zenserver/compute/computeserver.h
index 625140b23..e4a6b01d5 100644
--- a/src/zenserver/compute/computeserver.h
+++ b/src/zenserver/compute/computeserver.h
@@ -6,7 +6,11 @@
#if ZEN_WITH_COMPUTE_SERVICES
+# include <future>
+# include <zencore/system.h>
+# include <zenhttp/httpwsclient.h>
# include <zenstore/gc.h>
+# include "frontend/frontend.h"
namespace cxxopts {
class Options;
@@ -16,19 +20,46 @@ struct Options;
}
namespace zen::compute {
-class HttpFunctionService;
-}
+class CloudMetadata;
+class HttpComputeService;
+class HttpOrchestratorService;
+} // namespace zen::compute
+
+# if ZEN_WITH_HORDE
+# include <zenhorde/hordeconfig.h>
+namespace zen::horde {
+class HordeProvisioner;
+} // namespace zen::horde
+# endif
+
+# if ZEN_WITH_NOMAD
+# include <zennomad/nomadconfig.h>
+namespace zen::nomad {
+class NomadProvisioner;
+} // namespace zen::nomad
+# endif
namespace zen {
class CidStore;
class HttpApiService;
-class HttpComputeService;
struct ZenComputeServerConfig : public ZenServerConfig
{
std::string UpstreamNotificationEndpoint;
std::string InstanceId; // For use in notifications
+ std::string CoordinatorEndpoint;
+ std::string IdmsEndpoint;
+ int32_t MaxConcurrentActions = 0; // 0 = auto (LogicalProcessorCount * 2)
+ bool EnableWorkerWebSocket = false; // Use WebSocket for worker↔orchestrator link
+
+# if ZEN_WITH_HORDE
+ horde::HordeConfig HordeConfig;
+# endif
+
+# if ZEN_WITH_NOMAD
+ nomad::NomadConfig NomadConfig;
+# endif
};
struct ZenComputeServerConfigurator : public ZenServerConfiguratorBase
@@ -49,6 +80,16 @@ private:
virtual void ValidateOptions() override;
ZenComputeServerConfig& m_ServerOptions;
+
+# if ZEN_WITH_HORDE
+ std::string m_HordeModeStr = "direct";
+ std::string m_HordeEncryptionStr = "none";
+# endif
+
+# if ZEN_WITH_NOMAD
+ std::string m_NomadDriverStr = "raw_exec";
+ std::string m_NomadDistributionStr = "predeployed";
+# endif
};
class ZenComputeServerMain : public ZenServerMain
@@ -88,17 +129,59 @@ public:
void Cleanup();
private:
- HttpStatsService m_StatsService;
- GcManager m_GcManager;
- GcScheduler m_GcScheduler{m_GcManager};
- std::unique_ptr<CidStore> m_CidStore;
- std::unique_ptr<HttpComputeService> m_ComputeService;
- std::unique_ptr<HttpApiService> m_ApiService;
- std::unique_ptr<zen::compute::HttpFunctionService> m_FunctionService;
-
- void InitializeState(const ZenComputeServerConfig& ServerConfig);
- void InitializeServices(const ZenComputeServerConfig& ServerConfig);
- void RegisterServices(const ZenComputeServerConfig& ServerConfig);
+ HttpStatsService m_StatsService;
+ GcManager m_GcManager;
+ GcScheduler m_GcScheduler{m_GcManager};
+ std::unique_ptr<CidStore> m_CidStore;
+ std::unique_ptr<HttpApiService> m_ApiService;
+ std::unique_ptr<zen::compute::HttpComputeService> m_ComputeService;
+ std::unique_ptr<zen::compute::HttpOrchestratorService> m_OrchestratorService;
+ std::unique_ptr<zen::compute::CloudMetadata> m_CloudMetadata;
+ std::future<std::unique_ptr<zen::compute::CloudMetadata>> m_CloudMetadataFuture;
+ std::unique_ptr<HttpFrontendService> m_FrontendService;
+# if ZEN_WITH_HORDE
+ std::unique_ptr<zen::horde::HordeProvisioner> m_HordeProvisioner;
+# endif
+# if ZEN_WITH_NOMAD
+ std::unique_ptr<zen::nomad::NomadProvisioner> m_NomadProvisioner;
+# endif
+ SystemMetricsTracker m_MetricsTracker;
+ std::string m_CoordinatorEndpoint;
+ std::string m_InstanceId;
+
+ asio::steady_timer m_AnnounceTimer{m_IoContext};
+ asio::steady_timer m_ProvisionerMaintenanceTimer{m_IoContext};
+
+ void InitializeState(const ZenComputeServerConfig& ServerConfig);
+ void InitializeServices(const ZenComputeServerConfig& ServerConfig);
+ void RegisterServices(const ZenComputeServerConfig& ServerConfig);
+ void ResolveCloudMetadata();
+ void PostAnnounce();
+ void EnqueueAnnounceTimer();
+ void EnqueueProvisionerMaintenanceTimer();
+ void ProvisionerMaintenanceTick();
+ std::string GetAnnounceUrl() const;
+ std::string GetInstanceId() const;
+ CbObject BuildAnnounceBody();
+
+ // Worker→orchestrator WebSocket client
+ struct OrchestratorWsHandler : public IWsClientHandler
+ {
+ ZenComputeServer& Server;
+ explicit OrchestratorWsHandler(ZenComputeServer& S) : Server(S) {}
+
+ void OnWsOpen() override;
+ void OnWsMessage(const WebSocketMessage& Msg) override;
+ void OnWsClose(uint16_t Code, std::string_view Reason) override;
+ };
+
+ std::unique_ptr<OrchestratorWsHandler> m_OrchestratorWsHandler;
+ std::unique_ptr<HttpWsClient> m_OrchestratorWsClient;
+ asio::steady_timer m_WsReconnectTimer{m_IoContext};
+ bool m_EnableWorkerWebSocket = false;
+
+ void InitializeOrchestratorWebSocket();
+ void EnqueueWsReconnect();
};
} // namespace zen
diff --git a/src/zenserver/compute/computeservice.cpp b/src/zenserver/compute/computeservice.cpp
deleted file mode 100644
index 2c0bc0ae9..000000000
--- a/src/zenserver/compute/computeservice.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "computeservice.h"
-
-#if ZEN_WITH_COMPUTE_SERVICES
-
-# include <zencore/compactbinarybuilder.h>
-# include <zencore/filesystem.h>
-# include <zencore/fmtutils.h>
-# include <zencore/logging.h>
-# include <zencore/system.h>
-# include <zenutil/zenserverprocess.h>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-# include <EASTL/fixed_vector.h>
-# include <asio.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
-
-# include <unordered_map>
-
-namespace zen {
-
-//////////////////////////////////////////////////////////////////////////
-
-struct ResourceMetrics
-{
- uint64_t DiskUsageBytes = 0;
- uint64_t MemoryUsageBytes = 0;
-};
-
-//////////////////////////////////////////////////////////////////////////
-
-struct HttpComputeService::Impl
-{
- Impl(const Impl&) = delete;
- Impl& operator=(const Impl&) = delete;
-
- Impl();
- ~Impl();
-
- void Initialize(std::filesystem::path BaseDir) { ZEN_UNUSED(BaseDir); }
-
- void Cleanup() {}
-
-private:
-};
-
-HttpComputeService::Impl::Impl()
-{
-}
-
-HttpComputeService::Impl::~Impl()
-{
-}
-
-///////////////////////////////////////////////////////////////////////////
-
-HttpComputeService::HttpComputeService(std::filesystem::path BaseDir) : m_Impl(std::make_unique<Impl>())
-{
- using namespace std::literals;
-
- m_Impl->Initialize(BaseDir);
-
- m_Router.RegisterRoute(
- "status",
- [this](HttpRouterRequest& Req) {
- CbObjectWriter Obj;
- Obj.BeginArray("modules");
- Obj.EndArray();
- Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
- },
- HttpVerb::kGet);
-
- m_Router.RegisterRoute(
- "stats",
- [this](HttpRouterRequest& Req) {
- CbObjectWriter Obj;
- Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
- },
- HttpVerb::kGet);
-}
-
-HttpComputeService::~HttpComputeService()
-{
-}
-
-const char*
-HttpComputeService::BaseUri() const
-{
- return "/compute/";
-}
-
-void
-HttpComputeService::HandleRequest(zen::HttpServerRequest& Request)
-{
- m_Router.HandleRequest(Request);
-}
-
-} // namespace zen
-#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/src/zenserver/compute/computeservice.h b/src/zenserver/compute/computeservice.h
deleted file mode 100644
index 339200dd8..000000000
--- a/src/zenserver/compute/computeservice.h
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zenhttp/httpserver.h>
-
-#if ZEN_WITH_COMPUTE_SERVICES
-namespace zen {
-
-/** ZenServer Compute Service
- *
- * Manages a set of compute workers for use in UEFN content worker
- *
- */
-class HttpComputeService : public zen::HttpService
-{
-public:
- HttpComputeService(std::filesystem::path BaseDir);
- ~HttpComputeService();
-
- HttpComputeService(const HttpComputeService&) = delete;
- HttpComputeService& operator=(const HttpComputeService&) = delete;
-
- virtual const char* BaseUri() const override;
- virtual void HandleRequest(zen::HttpServerRequest& Request) override;
-
-private:
- HttpRequestRouter m_Router;
-
- struct Impl;
-
- std::unique_ptr<Impl> m_Impl;
-};
-
-} // namespace zen
-#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index 4767029c0..c167cc70e 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
Binary files differ
diff --git a/src/zenserver/frontend/html/404.html b/src/zenserver/frontend/html/404.html
new file mode 100644
index 000000000..829ef2097
--- /dev/null
+++ b/src/zenserver/frontend/html/404.html
@@ -0,0 +1,486 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Ooops</title>
+<style>
+ * { margin: 0; padding: 0; box-sizing: border-box; }
+
+ :root {
+ --deep-space: #00000f;
+ --nebula-blue: #0a0a2e;
+ --star-white: #ffffff;
+ --star-blue: #c8d8ff;
+ --star-yellow: #fff3c0;
+ --star-red: #ffd0c0;
+ --nebula-glow: rgba(60, 80, 180, 0.12);
+ }
+
+ body {
+ background: var(--deep-space);
+ min-height: 100vh;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ font-family: 'Courier New', monospace;
+ overflow: hidden;
+ }
+
+ starfield-bg {
+ display: block;
+ position: fixed;
+ inset: 0;
+ z-index: 0;
+ }
+
+ canvas {
+ display: block;
+ width: 100%;
+ height: 100%;
+ }
+
+ .page-content {
+ position: relative;
+ z-index: 1;
+ text-align: center;
+ color: rgba(200, 216, 255, 0.85);
+ letter-spacing: 0.25em;
+ text-transform: uppercase;
+ pointer-events: none;
+ user-select: none;
+ }
+
+ .page-content h1 {
+ font-size: clamp(1.2rem, 4vw, 2.4rem);
+ font-weight: 300;
+ letter-spacing: 0.6em;
+ text-shadow: 0 0 40px rgba(120, 160, 255, 0.6), 0 0 80px rgba(80, 120, 255, 0.3);
+ animation: pulse 6s ease-in-out infinite;
+ }
+
+ .page-content p {
+ margin-top: 1.2rem;
+ font-size: clamp(0.55rem, 1.5vw, 0.75rem);
+ letter-spacing: 0.4em;
+ opacity: 0.45;
+ }
+
+ @keyframes pulse {
+ 0%, 100% { opacity: 0.7; }
+ 50% { opacity: 1; }
+ }
+
+ .globe-link {
+ display: block;
+ margin: 0 auto 2rem;
+ width: 160px;
+ height: 160px;
+ pointer-events: auto;
+ cursor: pointer;
+ border-radius: 50%;
+ position: relative;
+ }
+
+ .globe-link:hover .globe-glow {
+ opacity: 0.6;
+ }
+
+ .globe-glow {
+ position: absolute;
+ inset: -18px;
+ border-radius: 50%;
+ background: radial-gradient(circle, rgba(80, 140, 255, 0.35) 0%, transparent 70%);
+ opacity: 0.35;
+ transition: opacity 0.4s;
+ pointer-events: none;
+ }
+
+ .globe-link canvas {
+ display: block;
+ width: 160px;
+ height: 160px;
+ border-radius: 50%;
+ }
+</style>
+</head>
+<body>
+
+<starfield-bg
+ star-count="380"
+ speed="0.6"
+ depth="true"
+ nebula="true"
+ shooting-stars="true"
+></starfield-bg>
+
+<div class="page-content">
+ <a class="globe-link" href="/dashboard/" title="Back to Dashboard">
+ <div class="globe-glow"></div>
+ <canvas id="globe" width="320" height="320"></canvas>
+ </a>
+ <h1>404 NOT FOUND</h1>
+</div>
+
+<script>
+class StarfieldBg extends HTMLElement {
+ constructor() {
+ super();
+ this.attachShadow({ mode: 'open' });
+ }
+
+ connectedCallback() {
+ this.shadowRoot.innerHTML = `
+ <style>
+ :host { display: block; position: absolute; inset: 0; overflow: hidden; }
+ canvas { width: 100%; height: 100%; display: block; }
+ </style>
+ <canvas></canvas>
+ `;
+
+ this.canvas = this.shadowRoot.querySelector('canvas');
+ this.ctx = this.canvas.getContext('2d');
+
+ this.starCount = parseInt(this.getAttribute('star-count') || '350');
+ this.speed = parseFloat(this.getAttribute('speed') || '0.6');
+ this.useDepth = this.getAttribute('depth') !== 'false';
+ this.useNebula = this.getAttribute('nebula') !== 'false';
+ this.useShooting = this.getAttribute('shooting-stars') !== 'false';
+
+ this.stars = [];
+ this.shooters = [];
+ this.nebulaTime = 0;
+ this.frame = 0;
+
+ this.resize();
+ this.init();
+
+ this._ro = new ResizeObserver(() => { this.resize(); this.init(); });
+ this._ro.observe(this);
+
+ this.raf = requestAnimationFrame(this.tick.bind(this));
+ }
+
+ disconnectedCallback() {
+ cancelAnimationFrame(this.raf);
+ this._ro.disconnect();
+ }
+
+ resize() {
+ const dpr = window.devicePixelRatio || 1;
+ const rect = this.getBoundingClientRect();
+ this.W = rect.width || window.innerWidth;
+ this.H = rect.height || window.innerHeight;
+ this.canvas.width = this.W * dpr;
+ this.canvas.height = this.H * dpr;
+ this.ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
+ }
+
+ init() {
+ const COLORS = ['#ffffff', '#c8d8ff', '#d0e8ff', '#fff3c0', '#ffd0c0', '#e0f0ff'];
+ this.stars = Array.from({ length: this.starCount }, () => ({
+ x: Math.random() * this.W,
+ y: Math.random() * this.H,
+ z: this.useDepth ? Math.random() : 1, // depth: 0=far, 1=near
+ r: Math.random() * 1.4 + 0.2,
+ color: COLORS[Math.floor(Math.random() * COLORS.length)],
+ twinkleOffset: Math.random() * Math.PI * 2,
+ twinkleSpeed: 0.008 + Math.random() * 0.012,
+ }));
+ }
+
+ spawnShooter() {
+ const edge = Math.random() < 0.7 ? 'top' : 'left';
+ const angle = (Math.random() * 30 + 15) * (Math.PI / 180);
+ this.shooters.push({
+ x: edge === 'top' ? Math.random() * this.W : -10,
+ y: edge === 'top' ? -10 : Math.random() * this.H * 0.5,
+ vx: Math.cos(angle) * (6 + Math.random() * 6),
+ vy: Math.sin(angle) * (6 + Math.random() * 6),
+ len: 80 + Math.random() * 120,
+ life: 1,
+ decay: 0.012 + Math.random() * 0.018,
+ });
+ }
+
+ tick() {
+ this.raf = requestAnimationFrame(this.tick.bind(this));
+ this.frame++;
+ const ctx = this.ctx;
+ const W = this.W, H = this.H;
+
+ // Background
+ ctx.fillStyle = '#00000f';
+ ctx.fillRect(0, 0, W, H);
+
+ // Nebula clouds (subtle)
+ if (this.useNebula) {
+ this.nebulaTime += 0.003;
+ this.drawNebula(ctx, W, H);
+ }
+
+ // Stars
+ for (const s of this.stars) {
+ const twinkle = 0.55 + 0.45 * Math.sin(this.frame * s.twinkleSpeed + s.twinkleOffset);
+ const radius = s.r * (this.useDepth ? (0.3 + s.z * 0.7) : 1);
+ const alpha = (this.useDepth ? (0.25 + s.z * 0.75) : 1) * twinkle;
+
+ // Tiny drift
+ s.x += (s.z * this.speed * 0.08) * (this.useDepth ? 1 : 0);
+ s.y += (s.z * this.speed * 0.04) * (this.useDepth ? 1 : 0);
+ if (s.x > W + 2) s.x = -2;
+ if (s.y > H + 2) s.y = -2;
+
+ // Glow for bright stars
+ if (radius > 1.1 && alpha > 0.6) {
+ const grd = ctx.createRadialGradient(s.x, s.y, 0, s.x, s.y, radius * 3.5);
+ grd.addColorStop(0, s.color.replace(')', `, ${alpha * 0.5})`).replace('rgb', 'rgba'));
+ grd.addColorStop(1, 'transparent');
+ ctx.beginPath();
+ ctx.arc(s.x, s.y, radius * 3.5, 0, Math.PI * 2);
+ ctx.fillStyle = grd;
+ ctx.fill();
+ }
+
+ ctx.beginPath();
+ ctx.arc(s.x, s.y, radius, 0, Math.PI * 2);
+ ctx.fillStyle = hexToRgba(s.color, alpha);
+ ctx.fill();
+ }
+
+ // Shooting stars
+ if (this.useShooting) {
+ if (this.frame % 140 === 0 && Math.random() < 0.65) this.spawnShooter();
+ for (let i = this.shooters.length - 1; i >= 0; i--) {
+ const s = this.shooters[i];
+ const tailX = s.x - s.vx * (s.len / Math.hypot(s.vx, s.vy));
+ const tailY = s.y - s.vy * (s.len / Math.hypot(s.vx, s.vy));
+
+ const grd = ctx.createLinearGradient(tailX, tailY, s.x, s.y);
+ grd.addColorStop(0, `rgba(255,255,255,0)`);
+ grd.addColorStop(0.7, `rgba(200,220,255,${s.life * 0.5})`);
+ grd.addColorStop(1, `rgba(255,255,255,${s.life})`);
+
+ ctx.beginPath();
+ ctx.moveTo(tailX, tailY);
+ ctx.lineTo(s.x, s.y);
+ ctx.strokeStyle = grd;
+ ctx.lineWidth = 1.5 * s.life;
+ ctx.lineCap = 'round';
+ ctx.stroke();
+
+ // Head dot
+ ctx.beginPath();
+ ctx.arc(s.x, s.y, 1.5 * s.life, 0, Math.PI * 2);
+ ctx.fillStyle = `rgba(255,255,255,${s.life})`;
+ ctx.fill();
+
+ s.x += s.vx;
+ s.y += s.vy;
+ s.life -= s.decay;
+
+ if (s.life <= 0 || s.x > W + 200 || s.y > H + 200) {
+ this.shooters.splice(i, 1);
+ }
+ }
+ }
+ }
+
+ drawNebula(ctx, W, H) {
+ const t = this.nebulaTime;
+ const blobs = [
+ { x: W * 0.25, y: H * 0.3, rx: W * 0.35, ry: H * 0.25, color: '40,60,180', a: 0.055 },
+ { x: W * 0.75, y: H * 0.65, rx: W * 0.30, ry: H * 0.22, color: '100,40,160', a: 0.04 },
+ { x: W * 0.5, y: H * 0.5, rx: W * 0.45, ry: H * 0.35, color: '20,50,120', a: 0.035 },
+ ];
+ ctx.save();
+ for (const b of blobs) {
+ const ox = Math.sin(t * 0.7 + b.x) * 30;
+ const oy = Math.cos(t * 0.5 + b.y) * 20;
+ const grd = ctx.createRadialGradient(b.x + ox, b.y + oy, 0, b.x + ox, b.y + oy, Math.max(b.rx, b.ry));
+ grd.addColorStop(0, `rgba(${b.color}, ${b.a})`);
+ grd.addColorStop(0.5, `rgba(${b.color}, ${b.a * 0.4})`);
+ grd.addColorStop(1, `rgba(${b.color}, 0)`);
+ ctx.save();
+ ctx.scale(b.rx / Math.max(b.rx, b.ry), b.ry / Math.max(b.rx, b.ry));
+ ctx.beginPath();
+ const scale = Math.max(b.rx, b.ry);
+ ctx.arc((b.x + ox) / (b.rx / scale), (b.y + oy) / (b.ry / scale), scale, 0, Math.PI * 2);
+ ctx.fillStyle = grd;
+ ctx.fill();
+ ctx.restore();
+ }
+ ctx.restore();
+ }
+}
+
+function hexToRgba(hex, alpha) {
+ // Handle named-ish values or full hex
+ const c = hex.startsWith('#') ? hex : '#ffffff';
+ const r = parseInt(c.slice(1,3), 16);
+ const g = parseInt(c.slice(3,5), 16);
+ const b = parseInt(c.slice(5,7), 16);
+ return `rgba(${r},${g},${b},${alpha.toFixed(3)})`;
+}
+
+customElements.define('starfield-bg', StarfieldBg);
+</script>
+
+<script>
+(function() {
+ const canvas = document.getElementById('globe');
+ const ctx = canvas.getContext('2d');
+ const W = canvas.width, H = canvas.height;
+ const R = W * 0.44;
+ const cx = W / 2, cy = H / 2;
+
+ // Simplified continent outlines as lon/lat polygon chains (degrees).
+ // Each continent is an array of [lon, lat] points.
+ const continents = [
+ // North America
+ [[-130,50],[-125,55],[-120,60],[-115,65],[-100,68],[-85,70],[-75,65],[-60,52],[-65,45],[-70,42],[-75,35],[-80,30],[-85,28],[-90,28],[-95,25],[-100,20],[-105,20],[-110,25],[-115,30],[-120,35],[-125,42],[-130,50]],
+ // South America
+ [[-80,10],[-75,5],[-70,5],[-65,0],[-60,-5],[-55,-5],[-50,-10],[-45,-15],[-40,-20],[-40,-25],[-42,-30],[-48,-32],[-52,-34],[-55,-38],[-60,-42],[-65,-50],[-68,-55],[-70,-48],[-72,-40],[-75,-30],[-78,-15],[-80,-5],[-80,5],[-80,10]],
+ // Europe
+ [[-10,36],[-5,38],[0,40],[2,43],[5,44],[8,46],[10,48],[15,50],[18,54],[20,56],[25,58],[28,60],[30,62],[35,65],[40,68],[38,60],[35,55],[30,50],[28,48],[25,45],[22,40],[20,38],[15,36],[10,36],[5,36],[0,36],[-5,36],[-10,36]],
+ // Africa
+ [[-15,14],[-17,16],[-15,22],[-12,28],[-5,32],[0,35],[5,37],[10,35],[15,32],[20,30],[25,30],[30,28],[35,25],[38,18],[40,12],[42,5],[44,0],[42,-5],[40,-12],[38,-18],[35,-25],[32,-30],[30,-34],[25,-33],[20,-30],[15,-28],[12,-20],[10,-10],[8,-5],[5,0],[2,5],[0,5],[-5,5],[-10,6],[-15,10],[-15,14]],
+ // Asia (simplified)
+ [[30,35],[35,38],[40,40],[45,42],[50,45],[55,48],[60,50],[65,55],[70,60],[75,65],[80,68],[90,70],[100,68],[110,65],[120,60],[125,55],[130,50],[135,45],[140,40],[138,35],[130,30],[120,25],[110,20],[105,15],[100,10],[95,12],[90,20],[85,22],[80,25],[75,28],[70,30],[65,35],[55,35],[45,35],[40,35],[35,35],[30,35]],
+ // Australia
+ [[115,-12],[120,-14],[125,-15],[130,-14],[135,-13],[138,-16],[140,-18],[145,-20],[148,-22],[150,-25],[152,-28],[150,-33],[148,-35],[145,-37],[140,-38],[135,-36],[130,-33],[125,-30],[120,-25],[118,-22],[116,-20],[114,-18],[115,-15],[115,-12]],
+ ];
+
+ function project(lon, lat, rotation) {
+ // Convert to radians and apply rotation
+ var lonR = (lon + rotation) * Math.PI / 180;
+ var latR = lat * Math.PI / 180;
+
+ var x3 = Math.cos(latR) * Math.sin(lonR);
+ var y3 = -Math.sin(latR);
+ var z3 = Math.cos(latR) * Math.cos(lonR);
+
+ // Only visible if facing us
+ if (z3 < 0) return null;
+
+ return { x: cx + x3 * R, y: cy + y3 * R, z: z3 };
+ }
+
+ var rotation = 0;
+
+ function draw() {
+ requestAnimationFrame(draw);
+ rotation += 0.15;
+ ctx.clearRect(0, 0, W, H);
+
+ // Atmosphere glow
+ var atm = ctx.createRadialGradient(cx, cy, R * 0.85, cx, cy, R * 1.15);
+ atm.addColorStop(0, 'rgba(60,130,255,0.12)');
+ atm.addColorStop(0.5, 'rgba(60,130,255,0.06)');
+ atm.addColorStop(1, 'rgba(60,130,255,0)');
+ ctx.beginPath();
+ ctx.arc(cx, cy, R * 1.15, 0, Math.PI * 2);
+ ctx.fillStyle = atm;
+ ctx.fill();
+
+ // Ocean sphere
+ var oceanGrad = ctx.createRadialGradient(cx - R * 0.3, cy - R * 0.3, R * 0.1, cx, cy, R);
+ oceanGrad.addColorStop(0, '#1a4a8a');
+ oceanGrad.addColorStop(0.5, '#0e2d5e');
+ oceanGrad.addColorStop(1, '#071838');
+ ctx.beginPath();
+ ctx.arc(cx, cy, R, 0, Math.PI * 2);
+ ctx.fillStyle = oceanGrad;
+ ctx.fill();
+
+ // Draw continents
+ for (var c = 0; c < continents.length; c++) {
+ var pts = continents[c];
+ var projected = [];
+ var allVisible = true;
+
+ for (var i = 0; i < pts.length; i++) {
+ var p = project(pts[i][0], pts[i][1], rotation);
+ if (!p) { allVisible = false; break; }
+ projected.push(p);
+ }
+
+ if (!allVisible || projected.length < 3) continue;
+
+ ctx.beginPath();
+ ctx.moveTo(projected[0].x, projected[0].y);
+ for (var i = 1; i < projected.length; i++) {
+ ctx.lineTo(projected[i].x, projected[i].y);
+ }
+ ctx.closePath();
+
+ // Shade based on average depth
+ var avgZ = 0;
+ for (var i = 0; i < projected.length; i++) avgZ += projected[i].z;
+ avgZ /= projected.length;
+ var brightness = 0.3 + avgZ * 0.7;
+
+ var r = Math.round(30 * brightness);
+ var g = Math.round(100 * brightness);
+ var b = Math.round(50 * brightness);
+ ctx.fillStyle = 'rgb(' + r + ',' + g + ',' + b + ')';
+ ctx.fill();
+ }
+
+ // Grid lines (longitude)
+ ctx.strokeStyle = 'rgba(100,160,255,0.08)';
+ ctx.lineWidth = 0.7;
+ for (var lon = -180; lon < 180; lon += 30) {
+ ctx.beginPath();
+ var started = false;
+ for (var lat = -90; lat <= 90; lat += 3) {
+ var p = project(lon, lat, rotation);
+ if (p) {
+ if (!started) { ctx.moveTo(p.x, p.y); started = true; }
+ else ctx.lineTo(p.x, p.y);
+ } else {
+ started = false;
+ }
+ }
+ ctx.stroke();
+ }
+
+ // Grid lines (latitude)
+ for (var lat = -60; lat <= 60; lat += 30) {
+ ctx.beginPath();
+ var started = false;
+ for (var lon = -180; lon <= 180; lon += 3) {
+ var p = project(lon, lat, rotation);
+ if (p) {
+ if (!started) { ctx.moveTo(p.x, p.y); started = true; }
+ else ctx.lineTo(p.x, p.y);
+ } else {
+ started = false;
+ }
+ }
+ ctx.stroke();
+ }
+
+ // Specular highlight
+ var spec = ctx.createRadialGradient(cx - R * 0.35, cy - R * 0.35, 0, cx - R * 0.35, cy - R * 0.35, R * 0.8);
+ spec.addColorStop(0, 'rgba(180,210,255,0.18)');
+ spec.addColorStop(0.4, 'rgba(120,160,255,0.05)');
+ spec.addColorStop(1, 'rgba(0,0,0,0)');
+ ctx.beginPath();
+ ctx.arc(cx, cy, R, 0, Math.PI * 2);
+ ctx.fillStyle = spec;
+ ctx.fill();
+
+ // Rim light
+ ctx.beginPath();
+ ctx.arc(cx, cy, R, 0, Math.PI * 2);
+ ctx.strokeStyle = 'rgba(80,140,255,0.2)';
+ ctx.lineWidth = 1.5;
+ ctx.stroke();
+ }
+
+ draw();
+})();
+</script>
+</body>
+</html>
diff --git a/src/zenserver/frontend/html/compute/banner.js b/src/zenserver/frontend/html/compute/banner.js
new file mode 100644
index 000000000..61c7ce21f
--- /dev/null
+++ b/src/zenserver/frontend/html/compute/banner.js
@@ -0,0 +1,321 @@
+/**
+ * zen-banner.js — Zen Compute dashboard banner Web Component
+ *
+ * Usage:
+ * <script src="/components/zen-banner.js" defer></script>
+ *
+ * <zen-banner></zen-banner>
+ * <zen-banner variant="compact"></zen-banner>
+ * <zen-banner cluster-status="degraded" load="78"></zen-banner>
+ *
+ * Attributes:
+ * variant "full" (default) | "compact"
+ * cluster-status "nominal" (default) | "degraded" | "offline"
+ * load 0–100 integer, shown as a percentage (default: hidden)
+ * tagline custom tagline text (default: "Orchestrator Overview" / "Orchestrator")
+ * subtitle text after "ZEN" in the wordmark (default: "COMPUTE")
+ */
+
+class ZenBanner extends HTMLElement {
+
+ static get observedAttributes() {
+ return ['variant', 'cluster-status', 'load', 'tagline', 'subtitle'];
+ }
+
+ attributeChangedCallback() {
+ if (this.shadowRoot) this._render();
+ }
+
+ connectedCallback() {
+ if (!this.shadowRoot) this.attachShadow({ mode: 'open' });
+ this._render();
+ }
+
+ // ─────────────────────────────────────────────
+ // Derived values
+ // ─────────────────────────────────────────────
+
+ get _variant() { return this.getAttribute('variant') || 'full'; }
+ get _status() { return (this.getAttribute('cluster-status') || 'nominal').toLowerCase(); }
+ get _load() { return this.getAttribute('load'); } // null → hidden
+ get _tagline() { return this.getAttribute('tagline'); } // null → default
+ get _subtitle() { return this.getAttribute('subtitle'); } // null → "COMPUTE"
+
+ get _statusColor() {
+ return { nominal: '#7ecfb8', degraded: '#d4a84b', offline: '#c0504d' }[this._status] ?? '#7ecfb8';
+ }
+
+ get _statusLabel() {
+ return { nominal: 'NOMINAL', degraded: 'DEGRADED', offline: 'OFFLINE' }[this._status] ?? 'NOMINAL';
+ }
+
+ get _loadColor() {
+ const v = parseInt(this._load, 10);
+ if (isNaN(v)) return '#7ecfb8';
+ if (v >= 85) return '#c0504d';
+ if (v >= 60) return '#d4a84b';
+ return '#7ecfb8';
+ }
+
+ // ─────────────────────────────────────────────
+ // Render
+ // ─────────────────────────────────────────────
+
+ _render() {
+ const compact = this._variant === 'compact';
+ this.shadowRoot.innerHTML = `
+ <style>${this._css(compact)}</style>
+ ${this._html(compact)}
+ `;
+ }
+
+ // ─────────────────────────────────────────────
+ // CSS
+ // ─────────────────────────────────────────────
+
+ _css(compact) {
+ const height = compact ? '60px' : '100px';
+ const padding = compact ? '0 24px' : '0 32px';
+ const gap = compact ? '16px' : '24px';
+ const markSize = compact ? '34px' : '52px';
+ const divH = compact ? '32px' : '48px';
+ const nameSize = compact ? '15px' : '22px';
+ const tagSize = compact ? '9px' : '11px';
+ const sc = this._statusColor;
+ const lc = this._loadColor;
+
+ return `
+ @import url('https://fonts.googleapis.com/css2?family=Noto+Serif+JP:wght@300;400&family=Space+Mono:wght@400;700&display=swap');
+
+ *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+ :host {
+ display: block;
+ font-family: 'Space Mono', monospace;
+ }
+
+ .banner {
+ width: 100%;
+ height: ${height};
+ background: #0b0d10;
+ border: 1px solid #1e2330;
+ border-radius: 6px;
+ display: flex;
+ align-items: center;
+ padding: ${padding};
+ gap: ${gap};
+ position: relative;
+ overflow: hidden;
+ }
+
+ /* scan-line texture */
+ .banner::before {
+ content: '';
+ position: absolute;
+ inset: 0;
+ background: repeating-linear-gradient(
+ 0deg,
+ transparent, transparent 3px,
+ rgba(255,255,255,0.012) 3px, rgba(255,255,255,0.012) 4px
+ );
+ pointer-events: none;
+ }
+
+ /* ambient glow */
+ .banner::after {
+ content: '';
+ position: absolute;
+ right: -60px;
+ top: 50%;
+ transform: translateY(-50%);
+ width: 280px;
+ height: 280px;
+ background: radial-gradient(circle, rgba(130,200,180,0.06) 0%, transparent 70%);
+ pointer-events: none;
+ }
+
+ .logo-mark {
+ flex-shrink: 0;
+ width: ${markSize};
+ height: ${markSize};
+ }
+
+ .logo-mark svg { width: 100%; height: 100%; }
+
+ .divider {
+ width: 1px;
+ height: ${divH};
+ background: linear-gradient(to bottom, transparent, #2a3040, transparent);
+ flex-shrink: 0;
+ }
+
+ .text-block {
+ display: flex;
+ flex-direction: column;
+ gap: 4px;
+ }
+
+ .wordmark {
+ font-weight: 700;
+ font-size: ${nameSize};
+ letter-spacing: 0.12em;
+ color: #e8e4dc;
+ text-transform: uppercase;
+ line-height: 1;
+ }
+
+ .wordmark span { color: #7ecfb8; }
+
+ .tagline {
+ font-family: 'Noto Serif JP', serif;
+ font-weight: 300;
+ font-size: ${tagSize};
+ letter-spacing: 0.3em;
+ color: #4a5a68;
+ text-transform: uppercase;
+ }
+
+ .spacer { flex: 1; }
+
+ /* ── right-side decorative circuit ── */
+ .circuit { flex-shrink: 0; opacity: 0.22; }
+
+ /* ── status cluster ── */
+ .status-cluster {
+ display: flex;
+ flex-direction: column;
+ align-items: flex-end;
+ gap: 6px;
+ }
+
+ .status-row {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ }
+
+ .status-lbl {
+ font-size: 9px;
+ letter-spacing: 0.18em;
+ color: #3a4555;
+ text-transform: uppercase;
+ }
+
+ .pill {
+ display: flex;
+ align-items: center;
+ gap: 5px;
+ border-radius: 20px;
+ padding: 2px 10px;
+ font-size: 10px;
+ letter-spacing: 0.1em;
+ }
+
+ .pill.cluster {
+ color: ${sc};
+ background: color-mix(in srgb, ${sc} 8%, transparent);
+ border: 1px solid color-mix(in srgb, ${sc} 28%, transparent);
+ }
+
+ .pill.load-pill {
+ color: ${lc};
+ background: color-mix(in srgb, ${lc} 8%, transparent);
+ border: 1px solid color-mix(in srgb, ${lc} 28%, transparent);
+ }
+
+ .dot {
+ width: 5px;
+ height: 5px;
+ border-radius: 50%;
+ animation: pulse 2.4s ease-in-out infinite;
+ }
+
+ .dot.cluster { background: ${sc}; }
+ .dot.load-dot { background: ${lc}; animation-delay: 0.5s; }
+
+ @keyframes pulse {
+ 0%, 100% { opacity: 1; }
+ 50% { opacity: 0.25; }
+ }
+ `;
+ }
+
+ // ─────────────────────────────────────────────
+ // HTML template
+ // ─────────────────────────────────────────────
+
+ _html(compact) {
+ const loadAttr = this._load;
+ const showStatus = !compact;
+
+ const rightSide = showStatus ? `
+ <svg class="circuit" width="60" height="60" viewBox="0 0 60 60" fill="none">
+ <path d="M5 30 H22 L28 18 H60" stroke="#7ecfb8" stroke-width="0.8"/>
+ <path d="M5 38 H18 L24 46 H60" stroke="#7ecfb8" stroke-width="0.8"/>
+ <circle cx="22" cy="30" r="2" fill="none" stroke="#7ecfb8" stroke-width="0.8"/>
+ <circle cx="18" cy="38" r="2" fill="none" stroke="#7ecfb8" stroke-width="0.8"/>
+ <circle cx="10" cy="30" r="1.2" fill="#7ecfb8"/>
+ <circle cx="10" cy="38" r="1.2" fill="#7ecfb8"/>
+ </svg>
+
+ <div class="status-cluster">
+ <div class="status-row">
+ <span class="status-lbl">Cluster</span>
+ <div class="pill cluster">
+ <div class="dot cluster"></div>
+ ${this._statusLabel}
+ </div>
+ </div>
+ ${loadAttr !== null ? `
+ <div class="status-row">
+ <span class="status-lbl">Load</span>
+ <div class="pill load-pill">
+ <div class="dot load-dot"></div>
+ ${parseInt(loadAttr, 10)} %
+ </div>
+ </div>` : ''}
+ </div>
+ ` : '';
+
+ return `
+ <div class="banner">
+ <div class="logo-mark">${this._svgMark()}</div>
+ <div class="divider"></div>
+ <div class="text-block">
+ <div class="wordmark">ZEN<span> ${this._subtitle ?? 'COMPUTE'}</span></div>
+ <div class="tagline">${this._tagline ?? (compact ? 'Orchestrator' : 'Orchestrator Overview')}</div>
+ </div>
+ <div class="spacer"></div>
+ ${rightSide}
+ </div>
+ `;
+ }
+
+ // ─────────────────────────────────────────────
+ // SVG logo mark
+ // ─────────────────────────────────────────────
+
+ _svgMark() {
+ return `
+ <svg viewBox="0 0 52 52" fill="none" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="26" cy="26" r="22" stroke="#2a3a48" stroke-width="1.5"/>
+ <path d="M26 4 A22 22 0 1 1 12 43.1" stroke="#7ecfb8" stroke-width="2" stroke-linecap="round" fill="none"/>
+ <circle cx="17" cy="17" r="1.6" fill="#7ecfb8" />
+ <circle cx="26" cy="17" r="1.6" fill="#7ecfb8" />
+ <circle cx="35" cy="17" r="1.6" fill="#7ecfb8" />
+ <circle cx="17" cy="26" r="1.6" fill="#7ecfb8" opacity="0.6"/>
+ <circle cx="26" cy="26" r="2.2" fill="#7ecfb8"/>
+ <circle cx="35" cy="26" r="1.6" fill="#7ecfb8" opacity="0.6"/>
+ <circle cx="17" cy="35" r="1.6" fill="#7ecfb8"/>
+ <circle cx="26" cy="35" r="1.6" fill="#7ecfb8"/>
+ <circle cx="35" cy="35" r="1.6" fill="#7ecfb8"/>
+ <line x1="17" y1="17" x2="35" y2="17" stroke="#7ecfb8" stroke-width="0.7" stroke-opacity="0.25"/>
+ <line x1="35" y1="17" x2="17" y2="35" stroke="#7ecfb8" stroke-width="0.7" stroke-opacity="0.25"/>
+ <line x1="17" y1="35" x2="35" y2="35" stroke="#7ecfb8" stroke-width="0.7" stroke-opacity="0.2"/>
+ <line x1="26" y1="17" x2="26" y2="35" stroke="#7ecfb8" stroke-width="0.7" stroke-opacity="0.2"/>
+ </svg>
+ `;
+ }
+}
+
+customElements.define('zen-banner', ZenBanner);
diff --git a/src/zenserver/frontend/html/compute.html b/src/zenserver/frontend/html/compute/compute.html
index 668189fe5..1e101d839 100644
--- a/src/zenserver/frontend/html/compute.html
+++ b/src/zenserver/frontend/html/compute/compute.html
@@ -5,6 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Zen Compute Dashboard</title>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.umd.min.js"></script>
+ <script src="banner.js" defer></script>
+ <script src="nav.js" defer></script>
<style>
* {
margin: 0;
@@ -291,16 +293,12 @@
</head>
<body>
<div class="container">
- <div class="header">
- <div>
- <h1>Zen Compute Dashboard</h1>
- <div class="timestamp">Last updated: <span id="last-update">Never</span></div>
- </div>
- <div class="health-indicator" id="health-indicator">
- <div class="status-dot"></div>
- <span id="health-text">Checking...</span>
- </div>
- </div>
+ <zen-banner cluster-status="nominal" load="0" tagline="Node Overview"></zen-banner>
+ <zen-nav>
+ <a href="compute.html">Node</a>
+ <a href="orchestrator.html">Orchestrator</a>
+ </zen-nav>
+ <div class="timestamp">Last updated: <span id="last-update">Never</span></div>
<div id="error-container"></div>
@@ -388,6 +386,30 @@
</div>
</div>
+ <!-- Queues -->
+ <div class="section-title">Queues</div>
+ <div class="card" style="margin-bottom: 30px;">
+ <div class="card-title">Queue Status</div>
+ <div id="queue-list-empty" style="color: #6e7681; font-size: 13px;">No queues.</div>
+ <div id="queue-list-container" style="display: none;">
+ <table id="queue-list-table" style="width: 100%; border-collapse: collapse; font-size: 13px;">
+ <thead>
+ <tr>
+ <th style="text-align: right; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px; width: 60px;">ID</th>
+ <th style="text-align: center; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px; width: 80px;">Status</th>
+ <th style="text-align: right; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px;">Active</th>
+ <th style="text-align: right; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px;">Completed</th>
+ <th style="text-align: right; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px;">Failed</th>
+ <th style="text-align: right; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px;">Abandoned</th>
+ <th style="text-align: right; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px;">Cancelled</th>
+ <th style="text-align: left; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px;">Token</th>
+ </tr>
+ </thead>
+ <tbody id="queue-list-body"></tbody>
+ </table>
+ </div>
+ </div>
+
<!-- Action History -->
<div class="section-title">Recent Actions</div>
<div class="card" style="margin-bottom: 30px;">
@@ -398,6 +420,7 @@
<thead>
<tr>
<th style="text-align: right; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px; width: 60px;">LSN</th>
+ <th style="text-align: right; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px; width: 60px;">Queue</th>
<th style="text-align: center; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px; width: 70px;">Status</th>
<th style="text-align: left; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px;">Function</th>
<th style="text-align: right; color: #8b949e; padding: 6px 8px; border-bottom: 1px solid #30363d; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; font-size: 11px; width: 80px;">Started</th>
@@ -576,6 +599,12 @@
});
// Helper functions
+ function escapeHtml(text) {
+ var div = document.createElement('div');
+ div.textContent = text;
+ return div.innerHTML;
+ }
+
function formatBytes(bytes) {
if (bytes === 0) return '0 B';
const k = 1024;
@@ -590,7 +619,7 @@
function showError(message) {
const container = document.getElementById('error-container');
- container.innerHTML = `<div class="error">Error: ${message}</div>`;
+ container.innerHTML = `<div class="error">Error: ${escapeHtml(message)}</div>`;
}
function clearError() {
@@ -617,35 +646,30 @@
async function fetchHealth() {
try {
- const response = await fetch(`${BASE_URL}/apply/ready`);
+ const response = await fetch(`${BASE_URL}/compute/ready`);
const isHealthy = response.status === 200;
- const indicator = document.getElementById('health-indicator');
- const text = document.getElementById('health-text');
+ const banner = document.querySelector('zen-banner');
if (isHealthy) {
- indicator.classList.add('healthy');
- indicator.classList.remove('unhealthy');
- text.textContent = 'Healthy';
+ banner.setAttribute('cluster-status', 'nominal');
+ banner.setAttribute('load', '0');
} else {
- indicator.classList.add('unhealthy');
- indicator.classList.remove('healthy');
- text.textContent = 'Unhealthy';
+ banner.setAttribute('cluster-status', 'degraded');
+ banner.setAttribute('load', '0');
}
return isHealthy;
} catch (error) {
- const indicator = document.getElementById('health-indicator');
- const text = document.getElementById('health-text');
- indicator.classList.add('unhealthy');
- indicator.classList.remove('healthy');
- text.textContent = 'Error';
+ const banner = document.querySelector('zen-banner');
+ banner.setAttribute('cluster-status', 'degraded');
+ banner.setAttribute('load', '0');
throw error;
}
}
async function fetchStats() {
- const data = await fetchJSON('/stats/apply');
+ const data = await fetchJSON('/stats/compute');
// Update action counts
document.getElementById('actions-pending').textContent = data.actions_pending || 0;
@@ -684,13 +708,16 @@
}
async function fetchSysInfo() {
- const data = await fetchJSON('/apply/sysinfo');
+ const data = await fetchJSON('/compute/sysinfo');
// Update CPU
const cpuUsage = data.cpu_usage || 0;
document.getElementById('cpu-usage').textContent = cpuUsage.toFixed(1) + '%';
document.getElementById('cpu-progress').style.width = cpuUsage + '%';
+ const banner = document.querySelector('zen-banner');
+ banner.setAttribute('load', cpuUsage.toFixed(1));
+
history.cpu.push(cpuUsage);
if (history.cpu.length > MAX_HISTORY_POINTS) history.cpu.shift();
cpuChart.data.labels = history.cpu.map(() => '');
@@ -741,7 +768,7 @@
const functions = desc.functions || [];
const functionsHtml = functions.length === 0 ? '<span style="color:#6e7681;font-size:12px;">none</span>' :
`<table class="detail-table">${functions.map(f =>
- `<tr><td>${f.name || '-'}</td><td class="detail-mono">${f.version || '-'}</td></tr>`
+ `<tr><td>${escapeHtml(f.name || '-')}</td><td class="detail-mono">${escapeHtml(f.version || '-')}</td></tr>`
).join('')}</table>`;
// Executables
@@ -756,8 +783,8 @@
</tr>
${executables.map(e =>
`<tr>
- <td>${e.name || '-'}</td>
- <td class="detail-mono">${e.hash || '-'}</td>
+ <td>${escapeHtml(e.name || '-')}</td>
+ <td class="detail-mono">${escapeHtml(e.hash || '-')}</td>
<td style="text-align:right;white-space:nowrap;">${e.size != null ? formatBytes(e.size) : '-'}</td>
</tr>`
).join('')}
@@ -772,26 +799,26 @@
const files = desc.files || [];
const filesHtml = files.length === 0 ? '<span style="color:#6e7681;font-size:12px;">none</span>' :
`<table class="detail-table">${files.map(f =>
- `<tr><td>${f.name || f}</td><td class="detail-mono">${f.hash || ''}</td></tr>`
+ `<tr><td>${escapeHtml(f.name || f)}</td><td class="detail-mono">${escapeHtml(f.hash || '')}</td></tr>`
).join('')}</table>`;
// Dirs
const dirs = desc.dirs || [];
const dirsHtml = dirs.length === 0 ? '<span style="color:#6e7681;font-size:12px;">none</span>' :
- dirs.map(d => `<span class="detail-tag">${d}</span>`).join('');
+ dirs.map(d => `<span class="detail-tag">${escapeHtml(d)}</span>`).join('');
// Environment
const env = desc.environment || [];
const envHtml = env.length === 0 ? '<span style="color:#6e7681;font-size:12px;">none</span>' :
- env.map(e => `<span class="detail-tag">${e}</span>`).join('');
+ env.map(e => `<span class="detail-tag">${escapeHtml(e)}</span>`).join('');
panel.innerHTML = `
- <div class="worker-detail-title">${desc.name || id}</div>
+ <div class="worker-detail-title">${escapeHtml(desc.name || id)}</div>
<div class="detail-section">
<table class="detail-table">
- ${field('Worker ID', `<span class="detail-mono">${id}</span>`)}
- ${field('Path', desc.path)}
- ${field('Platform', desc.host)}
+ ${field('Worker ID', `<span class="detail-mono">${escapeHtml(id)}</span>`)}
+ ${field('Path', escapeHtml(desc.path || '-'))}
+ ${field('Platform', escapeHtml(desc.host || '-'))}
${monoField('Build System', desc.buildsystem_version)}
${field('Cores', desc.cores)}
${field('Timeout', desc.timeout != null ? desc.timeout + 's' : null)}
@@ -822,7 +849,7 @@
}
async function fetchWorkers() {
- const data = await fetchJSON('/apply/workers');
+ const data = await fetchJSON('/compute/workers');
const workerIds = data.workers || [];
document.getElementById('worker-count').textContent = workerIds.length;
@@ -837,7 +864,7 @@
}
const descriptors = await Promise.all(
- workerIds.map(id => fetchJSON(`/apply/workers/${id}`).catch(() => null))
+ workerIds.map(id => fetchJSON(`/compute/workers/${id}`).catch(() => null))
);
// Build a map for quick lookup by ID
@@ -857,12 +884,12 @@
tr.className = 'worker-row' + (id === selectedWorkerId ? ' selected' : '');
tr.dataset.workerId = id;
tr.innerHTML = `
- <td style="padding: 6px 8px; color: #f0f6fc; border-bottom: 1px solid #21262d;">${name}</td>
- <td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d;">${host}</td>
- <td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d; text-align: right;">${cores}</td>
- <td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d; text-align: right;">${timeout}</td>
- <td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d; text-align: right;">${functions}</td>
- <td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; font-family: monospace; font-size: 11px;">${id}</td>
+ <td style="padding: 6px 8px; color: #f0f6fc; border-bottom: 1px solid #21262d;">${escapeHtml(name)}</td>
+ <td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d;">${escapeHtml(host)}</td>
+ <td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d; text-align: right;">${escapeHtml(String(cores))}</td>
+ <td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d; text-align: right;">${escapeHtml(String(timeout))}</td>
+ <td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d; text-align: right;">${escapeHtml(String(functions))}</td>
+ <td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; font-family: monospace; font-size: 11px;">${escapeHtml(id)}</td>
`;
tr.addEventListener('click', () => {
document.querySelectorAll('.worker-row').forEach(r => r.classList.remove('selected'));
@@ -914,8 +941,55 @@
return `${m}m ${s}s`;
}
+ async function fetchQueues() {
+ const data = await fetchJSON('/compute/queues');
+ const queues = data.queues || [];
+
+ const empty = document.getElementById('queue-list-empty');
+ const container = document.getElementById('queue-list-container');
+ const tbody = document.getElementById('queue-list-body');
+
+ if (queues.length === 0) {
+ empty.style.display = '';
+ container.style.display = 'none';
+ return;
+ }
+
+ empty.style.display = 'none';
+ tbody.innerHTML = '';
+
+ for (const q of queues) {
+ const id = q.queue_id ?? '-';
+ const badge = q.state === 'cancelled'
+ ? '<span class="status-badge failure">cancelled</span>'
+ : q.state === 'draining'
+ ? '<span class="status-badge" style="background:rgba(210,153,34,0.15);color:#d29922;">draining</span>'
+ : q.is_complete
+ ? '<span class="status-badge success">complete</span>'
+ : '<span class="status-badge" style="background:rgba(88,166,255,0.15);color:#58a6ff;">active</span>';
+ const token = q.queue_token
+ ? `<span class="detail-mono">${escapeHtml(q.queue_token)}</span>`
+ : '<span style="color:#6e7681;">-</span>';
+
+ const tr = document.createElement('tr');
+ tr.innerHTML = `
+ <td style="padding: 6px 8px; color: #f0f6fc; border-bottom: 1px solid #21262d; text-align: right; font-family: monospace;">${escapeHtml(String(id))}</td>
+ <td style="padding: 6px 8px; border-bottom: 1px solid #21262d; text-align: center;">${badge}</td>
+ <td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d; text-align: right;">${q.active_count ?? 0}</td>
+ <td style="padding: 6px 8px; color: #3fb950; border-bottom: 1px solid #21262d; text-align: right;">${q.completed_count ?? 0}</td>
+ <td style="padding: 6px 8px; color: #f85149; border-bottom: 1px solid #21262d; text-align: right;">${q.failed_count ?? 0}</td>
+ <td style="padding: 6px 8px; color: #d29922; border-bottom: 1px solid #21262d; text-align: right;">${q.abandoned_count ?? 0}</td>
+ <td style="padding: 6px 8px; color: #f0883e; border-bottom: 1px solid #21262d; text-align: right;">${q.cancelled_count ?? 0}</td>
+ <td style="padding: 6px 8px; border-bottom: 1px solid #21262d;">${token}</td>
+ `;
+ tbody.appendChild(tr);
+ }
+
+ container.style.display = 'block';
+ }
+
async function fetchActionHistory() {
- const data = await fetchJSON('/apply/jobs/history?limit=50');
+ const data = await fetchJSON('/compute/jobs/history?limit=50');
const entries = data.history || [];
const empty = document.getElementById('action-history-empty');
@@ -948,16 +1022,22 @@
const startDate = filetimeToDate(entry.time_Running);
const endDate = filetimeToDate(entry.time_Completed ?? entry.time_Failed);
+ const queueId = entry.queueId || 0;
+ const queueCell = queueId
+ ? `<a href="/compute/queues/${queueId}" style="color: #58a6ff; text-decoration: none; font-family: monospace;">${escapeHtml(String(queueId))}</a>`
+ : '<span style="color: #6e7681;">-</span>';
+
const tr = document.createElement('tr');
tr.innerHTML = `
- <td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; text-align: right; font-family: monospace;">${lsn}</td>
+ <td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; text-align: right; font-family: monospace;">${escapeHtml(String(lsn))}</td>
+ <td style="padding: 6px 8px; border-bottom: 1px solid #21262d; text-align: right;">${queueCell}</td>
<td style="padding: 6px 8px; border-bottom: 1px solid #21262d; text-align: center;">${badge}</td>
- <td style="padding: 6px 8px; color: #f0f6fc; border-bottom: 1px solid #21262d;">${fn}</td>
+ <td style="padding: 6px 8px; color: #f0f6fc; border-bottom: 1px solid #21262d;">${escapeHtml(fn)}</td>
<td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; text-align: right; font-size: 12px; white-space: nowrap;">${formatTime(startDate)}</td>
<td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; text-align: right; font-size: 12px; white-space: nowrap;">${formatTime(endDate)}</td>
<td style="padding: 6px 8px; color: #c9d1d9; border-bottom: 1px solid #21262d; text-align: right; font-size: 12px; white-space: nowrap;">${formatDuration(startDate, endDate)}</td>
- <td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; font-family: monospace; font-size: 11px;">${workerId}</td>
- <td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; font-family: monospace; font-size: 11px;">${actionId}</td>
+ <td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; font-family: monospace; font-size: 11px;">${escapeHtml(workerId)}</td>
+ <td style="padding: 6px 8px; color: #8b949e; border-bottom: 1px solid #21262d; font-family: monospace; font-size: 11px;">${escapeHtml(actionId)}</td>
`;
tbody.appendChild(tr);
}
@@ -972,6 +1052,7 @@
fetchStats(),
fetchSysInfo(),
fetchWorkers(),
+ fetchQueues(),
fetchActionHistory()
]);
diff --git a/src/zenserver/frontend/html/compute/hub.html b/src/zenserver/frontend/html/compute/hub.html
new file mode 100644
index 000000000..f66ba94d5
--- /dev/null
+++ b/src/zenserver/frontend/html/compute/hub.html
@@ -0,0 +1,310 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <script src="banner.js" defer></script>
+ <script src="nav.js" defer></script>
+ <title>Zen Hub Dashboard</title>
+ <style>
+ * {
+ margin: 0;
+ padding: 0;
+ box-sizing: border-box;
+ }
+
+ body {
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+ background: #0d1117;
+ color: #c9d1d9;
+ padding: 20px;
+ }
+
+ .container {
+ max-width: 1400px;
+ margin: 0 auto;
+ }
+
+ .timestamp {
+ font-size: 12px;
+ color: #6e7681;
+ }
+
+ .grid {
+ display: grid;
+ grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
+ gap: 20px;
+ margin-bottom: 30px;
+ }
+
+ .card {
+ background: #161b22;
+ border: 1px solid #30363d;
+ border-radius: 6px;
+ padding: 20px;
+ }
+
+ .card-title {
+ font-size: 14px;
+ font-weight: 600;
+ color: #8b949e;
+ margin-bottom: 12px;
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+ }
+
+ .metric-value {
+ font-size: 36px;
+ font-weight: 600;
+ color: #f0f6fc;
+ line-height: 1;
+ }
+
+ .metric-label {
+ font-size: 12px;
+ color: #8b949e;
+ margin-top: 4px;
+ }
+
+ .progress-bar {
+ width: 100%;
+ height: 8px;
+ background: #21262d;
+ border-radius: 4px;
+ overflow: hidden;
+ margin-top: 8px;
+ }
+
+ .progress-fill {
+ height: 100%;
+ background: #58a6ff;
+ transition: width 0.3s ease;
+ }
+
+ .error {
+ color: #f85149;
+ padding: 12px;
+ background: #1c1c1c;
+ border-radius: 6px;
+ margin: 20px 0;
+ font-size: 13px;
+ }
+
+ .section-title {
+ font-size: 20px;
+ font-weight: 600;
+ margin-bottom: 20px;
+ color: #f0f6fc;
+ }
+
+ table {
+ width: 100%;
+ border-collapse: collapse;
+ font-size: 13px;
+ }
+
+ th {
+ text-align: left;
+ color: #8b949e;
+ padding: 8px 12px;
+ border-bottom: 1px solid #30363d;
+ font-weight: 600;
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+ font-size: 11px;
+ }
+
+ td {
+ padding: 8px 12px;
+ border-bottom: 1px solid #21262d;
+ color: #c9d1d9;
+ }
+
+ tr:last-child td {
+ border-bottom: none;
+ }
+
+ .status-badge {
+ display: inline-block;
+ padding: 2px 8px;
+ border-radius: 4px;
+ font-size: 11px;
+ font-weight: 600;
+ }
+
+ .status-badge.active {
+ background: rgba(63, 185, 80, 0.15);
+ color: #3fb950;
+ }
+
+ .status-badge.inactive {
+ background: rgba(139, 148, 158, 0.15);
+ color: #8b949e;
+ }
+
+ .empty-state {
+ color: #6e7681;
+ font-size: 13px;
+ padding: 20px 0;
+ text-align: center;
+ }
+ </style>
+</head>
+<body>
+ <div class="container">
+ <zen-banner cluster-status="nominal" subtitle="HUB" tagline="Overview"></zen-banner>
+ <zen-nav>
+ <a href="hub.html">Hub</a>
+ </zen-nav>
+ <div class="timestamp">Last updated: <span id="last-update">Never</span></div>
+
+ <div id="error-container"></div>
+
+ <div class="section-title">Capacity</div>
+ <div class="grid">
+ <div class="card">
+ <div class="card-title">Active Modules</div>
+ <div class="metric-value" id="instance-count">-</div>
+ <div class="metric-label">Currently provisioned</div>
+ </div>
+ <div class="card">
+ <div class="card-title">Peak Modules</div>
+ <div class="metric-value" id="max-instance-count">-</div>
+ <div class="metric-label">High watermark</div>
+ </div>
+ <div class="card">
+ <div class="card-title">Instance Limit</div>
+ <div class="metric-value" id="instance-limit">-</div>
+ <div class="metric-label">Maximum allowed</div>
+ <div class="progress-bar">
+ <div class="progress-fill" id="capacity-progress" style="width: 0%"></div>
+ </div>
+ </div>
+ </div>
+
+ <div class="section-title">Modules</div>
+ <div class="card">
+ <div class="card-title">Storage Server Instances</div>
+ <div id="empty-state" class="empty-state">No modules provisioned.</div>
+ <table id="module-table" style="display: none;">
+ <thead>
+ <tr>
+ <th>Module ID</th>
+ <th style="text-align: center;">Status</th>
+ </tr>
+ </thead>
+ <tbody id="module-table-body"></tbody>
+ </table>
+ </div>
+ </div>
+
+ <script>
+ const BASE_URL = window.location.origin;
+ const REFRESH_INTERVAL = 2000;
+
+ function escapeHtml(text) {
+ var div = document.createElement('div');
+ div.textContent = text;
+ return div.innerHTML;
+ }
+
+ function showError(message) {
+ document.getElementById('error-container').innerHTML =
+ '<div class="error">Error: ' + escapeHtml(message) + '</div>';
+ }
+
+ function clearError() {
+ document.getElementById('error-container').innerHTML = '';
+ }
+
+ async function fetchJSON(endpoint) {
+ var response = await fetch(BASE_URL + endpoint, {
+ headers: { 'Accept': 'application/json' }
+ });
+ if (!response.ok) {
+ throw new Error('HTTP ' + response.status + ': ' + response.statusText);
+ }
+ return await response.json();
+ }
+
+ async function fetchStats() {
+ var data = await fetchJSON('/hub/stats');
+
+ var current = data.currentInstanceCount || 0;
+ var max = data.maxInstanceCount || 0;
+ var limit = data.instanceLimit || 0;
+
+ document.getElementById('instance-count').textContent = current;
+ document.getElementById('max-instance-count').textContent = max;
+ document.getElementById('instance-limit').textContent = limit;
+
+ var pct = limit > 0 ? (current / limit) * 100 : 0;
+ document.getElementById('capacity-progress').style.width = pct + '%';
+
+ var banner = document.querySelector('zen-banner');
+ if (current === 0) {
+ banner.setAttribute('cluster-status', 'nominal');
+ } else if (limit > 0 && current >= limit * 0.9) {
+ banner.setAttribute('cluster-status', 'degraded');
+ } else {
+ banner.setAttribute('cluster-status', 'nominal');
+ }
+ }
+
+ async function fetchModules() {
+ var data = await fetchJSON('/hub/status');
+ var modules = data.modules || [];
+
+ var emptyState = document.getElementById('empty-state');
+ var table = document.getElementById('module-table');
+ var tbody = document.getElementById('module-table-body');
+
+ if (modules.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ return;
+ }
+
+ emptyState.style.display = 'none';
+ table.style.display = '';
+
+ tbody.innerHTML = '';
+ for (var i = 0; i < modules.length; i++) {
+ var m = modules[i];
+ var moduleId = m.moduleId || '';
+ var provisioned = m.provisioned;
+
+ var badge = provisioned
+ ? '<span class="status-badge active">Provisioned</span>'
+ : '<span class="status-badge inactive">Inactive</span>';
+
+ var tr = document.createElement('tr');
+ tr.innerHTML =
+ '<td style="font-family: monospace; font-size: 12px;">' + escapeHtml(moduleId) + '</td>' +
+ '<td style="text-align: center;">' + badge + '</td>';
+ tbody.appendChild(tr);
+ }
+ }
+
+ async function updateDashboard() {
+ var banner = document.querySelector('zen-banner');
+ try {
+ await Promise.all([
+ fetchStats(),
+ fetchModules()
+ ]);
+
+ clearError();
+ document.getElementById('last-update').textContent = new Date().toLocaleTimeString();
+ } catch (error) {
+ console.error('Error updating dashboard:', error);
+ showError(error.message);
+ banner.setAttribute('cluster-status', 'offline');
+ }
+ }
+
+ updateDashboard();
+ setInterval(updateDashboard, REFRESH_INTERVAL);
+ </script>
+</body>
+</html>
diff --git a/src/zenserver/frontend/html/compute/index.html b/src/zenserver/frontend/html/compute/index.html
new file mode 100644
index 000000000..9597fd7f3
--- /dev/null
+++ b/src/zenserver/frontend/html/compute/index.html
@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=compute.html" /> \ No newline at end of file
diff --git a/src/zenserver/frontend/html/compute/nav.js b/src/zenserver/frontend/html/compute/nav.js
new file mode 100644
index 000000000..8ec42abd0
--- /dev/null
+++ b/src/zenserver/frontend/html/compute/nav.js
@@ -0,0 +1,79 @@
+/**
+ * zen-nav.js — Zen dashboard navigation bar Web Component
+ *
+ * Usage:
+ * <script src="nav.js" defer></script>
+ *
+ * <zen-nav>
+ * <a href="compute.html">Node</a>
+ * <a href="orchestrator.html">Orchestrator</a>
+ * </zen-nav>
+ *
+ * Each child <a> becomes a nav link. The current page is
+ * highlighted automatically based on the href.
+ */
+
+class ZenNav extends HTMLElement {
+
+ connectedCallback() {
+ if (!this.shadowRoot) this.attachShadow({ mode: 'open' });
+ this._render();
+ }
+
+ _render() {
+ const currentPath = window.location.pathname;
+ const items = Array.from(this.querySelectorAll(':scope > a'));
+
+ const links = items.map(a => {
+ const href = a.getAttribute('href') || '';
+ const label = a.textContent.trim();
+ const active = currentPath.endsWith(href);
+ return `<a class="nav-link${active ? ' active' : ''}" href="${href}">${label}</a>`;
+ }).join('');
+
+ this.shadowRoot.innerHTML = `
+ <style>
+ *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+ :host {
+ display: block;
+ margin-bottom: 16px;
+ }
+
+ .nav-bar {
+ display: flex;
+ align-items: center;
+ gap: 4px;
+ padding: 4px;
+ background: #161b22;
+ border: 1px solid #30363d;
+ border-radius: 6px;
+ }
+
+ .nav-link {
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+ font-size: 13px;
+ font-weight: 500;
+ color: #8b949e;
+ text-decoration: none;
+ padding: 6px 14px;
+ border-radius: 4px;
+ transition: color 0.15s, background 0.15s;
+ }
+
+ .nav-link:hover {
+ color: #c9d1d9;
+ background: #21262d;
+ }
+
+ .nav-link.active {
+ color: #f0f6fc;
+ background: #30363d;
+ }
+ </style>
+ <nav class="nav-bar">${links}</nav>
+ `;
+ }
+}
+
+customElements.define('zen-nav', ZenNav);
diff --git a/src/zenserver/frontend/html/compute/orchestrator.html b/src/zenserver/frontend/html/compute/orchestrator.html
new file mode 100644
index 000000000..2ee57b6b3
--- /dev/null
+++ b/src/zenserver/frontend/html/compute/orchestrator.html
@@ -0,0 +1,831 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <script src="banner.js" defer></script>
+ <script src="nav.js" defer></script>
+ <title>Zen Orchestrator Dashboard</title>
+ <style>
+ * {
+ margin: 0;
+ padding: 0;
+ box-sizing: border-box;
+ }
+
+ body {
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+ background: #0d1117;
+ color: #c9d1d9;
+ padding: 20px;
+ }
+
+ .container {
+ max-width: 1400px;
+ margin: 0 auto;
+ }
+
+ h1 {
+ font-size: 32px;
+ font-weight: 600;
+ margin-bottom: 10px;
+ color: #f0f6fc;
+ }
+
+ .header {
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ margin-bottom: 30px;
+ }
+
+ .timestamp {
+ font-size: 12px;
+ color: #6e7681;
+ }
+
+ .agent-count {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ font-size: 14px;
+ padding: 8px 16px;
+ border-radius: 6px;
+ background: #161b22;
+ border: 1px solid #30363d;
+ }
+
+ .agent-count .count {
+ font-size: 20px;
+ font-weight: 600;
+ color: #f0f6fc;
+ }
+
+ .card {
+ background: #161b22;
+ border: 1px solid #30363d;
+ border-radius: 6px;
+ padding: 20px;
+ }
+
+ .card-title {
+ font-size: 14px;
+ font-weight: 600;
+ color: #8b949e;
+ margin-bottom: 12px;
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+ }
+
+ .error {
+ color: #f85149;
+ padding: 12px;
+ background: #1c1c1c;
+ border-radius: 6px;
+ margin: 20px 0;
+ font-size: 13px;
+ }
+
+ table {
+ width: 100%;
+ border-collapse: collapse;
+ font-size: 13px;
+ }
+
+ th {
+ text-align: left;
+ color: #8b949e;
+ padding: 8px 12px;
+ border-bottom: 1px solid #30363d;
+ font-weight: 600;
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+ font-size: 11px;
+ }
+
+ td {
+ padding: 8px 12px;
+ border-bottom: 1px solid #21262d;
+ color: #c9d1d9;
+ }
+
+ tr:last-child td {
+ border-bottom: none;
+ }
+
+ .total-row td {
+ border-top: 2px solid #30363d;
+ font-weight: 600;
+ color: #f0f6fc;
+ }
+
+ .health-dot {
+ display: inline-block;
+ width: 10px;
+ height: 10px;
+ border-radius: 50%;
+ }
+
+ .health-green {
+ background: #3fb950;
+ }
+
+ .health-yellow {
+ background: #d29922;
+ }
+
+ .health-red {
+ background: #f85149;
+ }
+
+ a {
+ color: #58a6ff;
+ text-decoration: none;
+ }
+
+ a:hover {
+ text-decoration: underline;
+ }
+
+ .empty-state {
+ color: #6e7681;
+ font-size: 13px;
+ padding: 20px 0;
+ text-align: center;
+ }
+
+ .history-tabs {
+ display: flex;
+ gap: 4px;
+ background: #0d1117;
+ border-radius: 6px;
+ padding: 2px;
+ }
+
+ .history-tab {
+ background: transparent;
+ border: none;
+ color: #8b949e;
+ font-size: 12px;
+ font-weight: 600;
+ padding: 4px 12px;
+ border-radius: 4px;
+ cursor: pointer;
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+ }
+
+ .history-tab:hover {
+ color: #c9d1d9;
+ }
+
+ .history-tab.active {
+ background: #30363d;
+ color: #f0f6fc;
+ }
+ </style>
+</head>
+<body>
+ <div class="container">
+ <zen-banner cluster-status="nominal" load="0"></zen-banner>
+ <zen-nav>
+ <a href="compute.html">Node</a>
+ <a href="orchestrator.html">Orchestrator</a>
+ </zen-nav>
+ <div class="header">
+ <div>
+ <div class="timestamp">Last updated: <span id="last-update">Never</span></div>
+ </div>
+ <div class="agent-count">
+ <span>Agents:</span>
+ <span class="count" id="agent-count">-</span>
+ </div>
+ </div>
+
+ <div id="error-container"></div>
+
+ <div class="card">
+ <div class="card-title">Compute Agents</div>
+ <div id="empty-state" class="empty-state">No agents registered.</div>
+ <table id="agent-table" style="display: none;">
+ <thead>
+ <tr>
+ <th style="width: 40px; text-align: center;">Health</th>
+ <th>Hostname</th>
+ <th style="text-align: right;">CPUs</th>
+ <th style="text-align: right;">CPU Usage</th>
+ <th style="text-align: right;">Memory</th>
+ <th style="text-align: right;">Queues</th>
+ <th style="text-align: right;">Pending</th>
+ <th style="text-align: right;">Running</th>
+ <th style="text-align: right;">Completed</th>
+ <th style="text-align: right;">Traffic</th>
+ <th style="text-align: right;">Last Seen</th>
+ </tr>
+ </thead>
+ <tbody id="agent-table-body"></tbody>
+ </table>
+ </div>
+ <div class="card" style="margin-top: 20px;">
+ <div class="card-title">Connected Clients</div>
+ <div id="clients-empty" class="empty-state">No clients connected.</div>
+ <table id="clients-table" style="display: none;">
+ <thead>
+ <tr>
+ <th style="width: 40px; text-align: center;">Health</th>
+ <th>Client ID</th>
+ <th>Hostname</th>
+ <th>Address</th>
+ <th style="text-align: right;">Last Seen</th>
+ </tr>
+ </thead>
+ <tbody id="clients-table-body"></tbody>
+ </table>
+ </div>
+ <div class="card" style="margin-top: 20px;">
+ <div style="display: flex; align-items: center; gap: 12px; margin-bottom: 12px;">
+ <div class="card-title" style="margin-bottom: 0;">Event History</div>
+ <div class="history-tabs">
+ <button class="history-tab active" data-tab="workers" onclick="switchHistoryTab('workers')">Workers</button>
+ <button class="history-tab" data-tab="clients" onclick="switchHistoryTab('clients')">Clients</button>
+ </div>
+ </div>
+ <div id="history-panel-workers">
+ <div id="history-empty" class="empty-state">No provisioning events recorded.</div>
+ <table id="history-table" style="display: none;">
+ <thead>
+ <tr>
+ <th>Time</th>
+ <th>Event</th>
+ <th>Worker</th>
+ <th>Hostname</th>
+ </tr>
+ </thead>
+ <tbody id="history-table-body"></tbody>
+ </table>
+ </div>
+ <div id="history-panel-clients" style="display: none;">
+ <div id="client-history-empty" class="empty-state">No client events recorded.</div>
+ <table id="client-history-table" style="display: none;">
+ <thead>
+ <tr>
+ <th>Time</th>
+ <th>Event</th>
+ <th>Client</th>
+ <th>Hostname</th>
+ </tr>
+ </thead>
+ <tbody id="client-history-table-body"></tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+
+ <script>
+ const BASE_URL = window.location.origin;
+ const REFRESH_INTERVAL = 2000;
+
+ function escapeHtml(text) {
+ var div = document.createElement('div');
+ div.textContent = text;
+ return div.innerHTML;
+ }
+
+ function showError(message) {
+ document.getElementById('error-container').innerHTML =
+ '<div class="error">Error: ' + escapeHtml(message) + '</div>';
+ }
+
+ function clearError() {
+ document.getElementById('error-container').innerHTML = '';
+ }
+
+ function formatLastSeen(dtMs) {
+ if (dtMs == null) return '-';
+ var seconds = Math.floor(dtMs / 1000);
+ if (seconds < 60) return seconds + 's ago';
+ var minutes = Math.floor(seconds / 60);
+ if (minutes < 60) return minutes + 'm ' + (seconds % 60) + 's ago';
+ var hours = Math.floor(minutes / 60);
+ return hours + 'h ' + (minutes % 60) + 'm ago';
+ }
+
+ function healthClass(dtMs, reachable) {
+ if (reachable === false) return 'health-red';
+ if (dtMs == null) return 'health-red';
+ var seconds = dtMs / 1000;
+ if (seconds < 30 && reachable === true) return 'health-green';
+ if (seconds < 120) return 'health-yellow';
+ return 'health-red';
+ }
+
+ function healthTitle(dtMs, reachable) {
+ var seenStr = dtMs != null ? 'Last seen ' + formatLastSeen(dtMs) : 'Never seen';
+ if (reachable === true) return seenStr + ' · Reachable';
+ if (reachable === false) return seenStr + ' · Unreachable';
+ return seenStr + ' · Reachability unknown';
+ }
+
+ function formatCpuUsage(percent) {
+ if (percent == null || percent === 0) return '-';
+ return percent.toFixed(1) + '%';
+ }
+
+ function formatMemory(usedBytes, totalBytes) {
+ if (!totalBytes) return '-';
+ var usedGiB = usedBytes / (1024 * 1024 * 1024);
+ var totalGiB = totalBytes / (1024 * 1024 * 1024);
+ return usedGiB.toFixed(1) + ' / ' + totalGiB.toFixed(1) + ' GiB';
+ }
+
+ function formatBytes(bytes) {
+ if (!bytes) return '-';
+ if (bytes < 1024) return bytes + ' B';
+ if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KiB';
+ if (bytes < 1024 * 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(1) + ' MiB';
+ if (bytes < 1024 * 1024 * 1024 * 1024) return (bytes / (1024 * 1024 * 1024)).toFixed(1) + ' GiB';
+ return (bytes / (1024 * 1024 * 1024 * 1024)).toFixed(1) + ' TiB';
+ }
+
+ function formatTraffic(recv, sent) {
+ if (!recv && !sent) return '-';
+ return formatBytes(recv) + ' / ' + formatBytes(sent);
+ }
+
+ function parseIpFromUri(uri) {
+ try {
+ var url = new URL(uri);
+ var host = url.hostname;
+ // Strip IPv6 brackets
+ if (host.startsWith('[') && host.endsWith(']')) host = host.slice(1, -1);
+ // Only handle IPv4
+ var parts = host.split('.');
+ if (parts.length !== 4) return null;
+ var octets = parts.map(Number);
+ if (octets.some(function(o) { return isNaN(o) || o < 0 || o > 255; })) return null;
+ return octets;
+ } catch (e) {
+ return null;
+ }
+ }
+
+ function computeCidr(ips) {
+ if (ips.length === 0) return null;
+ if (ips.length === 1) return ips[0].join('.') + '/32';
+
+ // Convert each IP to a 32-bit integer
+ var ints = ips.map(function(o) {
+ return ((o[0] << 24) | (o[1] << 16) | (o[2] << 8) | o[3]) >>> 0;
+ });
+
+ // Find common prefix length by ANDing all identical high bits
+ var common = ~0 >>> 0;
+ for (var i = 1; i < ints.length; i++) {
+ // XOR to find differing bits, then mask away everything from the first difference down
+ var diff = (ints[0] ^ ints[i]) >>> 0;
+ if (diff !== 0) {
+ var bit = 31 - Math.floor(Math.log2(diff));
+ var mask = bit > 0 ? ((~0 << (32 - bit)) >>> 0) : 0;
+ common = (common & mask) >>> 0;
+ }
+ }
+
+ // Count leading ones in the common mask
+ var prefix = 0;
+ for (var b = 31; b >= 0; b--) {
+ if ((common >>> b) & 1) prefix++;
+ else break;
+ }
+
+ // Network address
+ var net = (ints[0] & common) >>> 0;
+ var a = (net >>> 24) & 0xff;
+ var bv = (net >>> 16) & 0xff;
+ var c = (net >>> 8) & 0xff;
+ var d = net & 0xff;
+ return a + '.' + bv + '.' + c + '.' + d + '/' + prefix;
+ }
+
+ function renderDashboard(data) {
+ var banner = document.querySelector('zen-banner');
+ if (data.hostname) {
+ banner.setAttribute('tagline', 'Orchestrator \u2014 ' + data.hostname);
+ }
+ var workers = data.workers || [];
+
+ document.getElementById('agent-count').textContent = workers.length;
+
+ if (workers.length === 0) {
+ banner.setAttribute('cluster-status', 'degraded');
+ banner.setAttribute('load', '0');
+ } else {
+ banner.setAttribute('cluster-status', 'nominal');
+ }
+
+ var emptyState = document.getElementById('empty-state');
+ var table = document.getElementById('agent-table');
+ var tbody = document.getElementById('agent-table-body');
+
+ if (workers.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ } else {
+ emptyState.style.display = 'none';
+ table.style.display = '';
+
+ tbody.innerHTML = '';
+ var totalCpus = 0;
+ var totalWeightedCpuUsage = 0;
+ var totalMemUsed = 0;
+ var totalMemTotal = 0;
+ var totalQueues = 0;
+ var totalPending = 0;
+ var totalRunning = 0;
+ var totalCompleted = 0;
+ var totalBytesRecv = 0;
+ var totalBytesSent = 0;
+ var allIps = [];
+ for (var i = 0; i < workers.length; i++) {
+ var w = workers[i];
+ var uri = w.uri || '';
+ var dt = w.dt;
+ var dashboardUrl = uri + '/dashboard/compute/';
+
+ var id = w.id || '';
+
+ var hostname = w.hostname || '';
+ var cpus = w.cpus || 0;
+ totalCpus += cpus;
+ if (cpus > 0 && typeof w.cpu_usage === 'number') {
+ totalWeightedCpuUsage += w.cpu_usage * cpus;
+ }
+
+ var memTotal = w.memory_total || 0;
+ var memUsed = w.memory_used || 0;
+ totalMemTotal += memTotal;
+ totalMemUsed += memUsed;
+
+ var activeQueues = w.active_queues || 0;
+ totalQueues += activeQueues;
+
+ var actionsPending = w.actions_pending || 0;
+ var actionsRunning = w.actions_running || 0;
+ var actionsCompleted = w.actions_completed || 0;
+ totalPending += actionsPending;
+ totalRunning += actionsRunning;
+ totalCompleted += actionsCompleted;
+
+ var bytesRecv = w.bytes_received || 0;
+ var bytesSent = w.bytes_sent || 0;
+ totalBytesRecv += bytesRecv;
+ totalBytesSent += bytesSent;
+
+ var ip = parseIpFromUri(uri);
+ if (ip) allIps.push(ip);
+
+ var reachable = w.reachable;
+ var hClass = healthClass(dt, reachable);
+ var hTitle = healthTitle(dt, reachable);
+
+ var platform = w.platform || '';
+ var badges = '';
+ if (platform) {
+ var platColors = { windows: '#0078d4', wine: '#722f37', linux: '#e95420', macos: '#a2aaad' };
+ var platColor = platColors[platform] || '#8b949e';
+ badges += ' <span style="display:inline-block;padding:1px 6px;border-radius:10px;font-size:10px;font-weight:600;color:#fff;background:' + platColor + ';vertical-align:middle;margin-left:4px;">' + escapeHtml(platform) + '</span>';
+ }
+ var provisioner = w.provisioner || '';
+ if (provisioner) {
+ var provColors = { horde: '#8957e5', nomad: '#3fb950' };
+ var provColor = provColors[provisioner] || '#8b949e';
+ badges += ' <span style="display:inline-block;padding:1px 6px;border-radius:10px;font-size:10px;font-weight:600;color:#fff;background:' + provColor + ';vertical-align:middle;margin-left:4px;">' + escapeHtml(provisioner) + '</span>';
+ }
+
+ var tr = document.createElement('tr');
+ tr.title = id;
+ tr.innerHTML =
+ '<td style="text-align: center;"><span class="health-dot ' + hClass + '" title="' + escapeHtml(hTitle) + '"></span></td>' +
+ '<td><a href="' + escapeHtml(dashboardUrl) + '" target="_blank">' + escapeHtml(hostname) + '</a>' + badges + '</td>' +
+ '<td style="text-align: right;">' + (cpus > 0 ? cpus : '-') + '</td>' +
+ '<td style="text-align: right;">' + formatCpuUsage(w.cpu_usage) + '</td>' +
+ '<td style="text-align: right;">' + formatMemory(memUsed, memTotal) + '</td>' +
+ '<td style="text-align: right;">' + (activeQueues > 0 ? activeQueues : '-') + '</td>' +
+ '<td style="text-align: right;">' + actionsPending + '</td>' +
+ '<td style="text-align: right;">' + actionsRunning + '</td>' +
+ '<td style="text-align: right;">' + actionsCompleted + '</td>' +
+ '<td style="text-align: right; color: #8b949e; font-size: 11px;">' + formatTraffic(bytesRecv, bytesSent) + '</td>' +
+ '<td style="text-align: right; color: #8b949e;">' + formatLastSeen(dt) + '</td>';
+ tbody.appendChild(tr);
+ }
+
+ var clusterLoad = totalCpus > 0 ? (totalWeightedCpuUsage / totalCpus) : 0;
+ banner.setAttribute('load', clusterLoad.toFixed(1));
+
+ // Total row
+ var cidr = computeCidr(allIps);
+ var totalTr = document.createElement('tr');
+ totalTr.className = 'total-row';
+ totalTr.innerHTML =
+ '<td></td>' +
+ '<td style="text-align: right; color: #8b949e; text-transform: uppercase; font-size: 11px;">Total' + (cidr ? ' <span style="font-family: monospace; font-weight: normal;">' + escapeHtml(cidr) + '</span>' : '') + '</td>' +
+ '<td style="text-align: right;">' + totalCpus + '</td>' +
+ '<td></td>' +
+ '<td style="text-align: right;">' + formatMemory(totalMemUsed, totalMemTotal) + '</td>' +
+ '<td style="text-align: right;">' + totalQueues + '</td>' +
+ '<td style="text-align: right;">' + totalPending + '</td>' +
+ '<td style="text-align: right;">' + totalRunning + '</td>' +
+ '<td style="text-align: right;">' + totalCompleted + '</td>' +
+ '<td style="text-align: right; font-size: 11px;">' + formatTraffic(totalBytesRecv, totalBytesSent) + '</td>' +
+ '<td></td>';
+ tbody.appendChild(totalTr);
+ }
+
+ clearError();
+ document.getElementById('last-update').textContent = new Date().toLocaleTimeString();
+
+ // Render provisioning history if present in WebSocket payload
+ if (data.events) {
+ renderProvisioningHistory(data.events);
+ }
+
+ // Render connected clients if present
+ if (data.clients) {
+ renderClients(data.clients);
+ }
+
+ // Render client history if present
+ if (data.client_events) {
+ renderClientHistory(data.client_events);
+ }
+ }
+
+ function eventBadge(type) {
+ var colors = { joined: '#3fb950', left: '#f85149', returned: '#d29922' };
+ var labels = { joined: 'Joined', left: 'Left', returned: 'Returned' };
+ var color = colors[type] || '#8b949e';
+ var label = labels[type] || type;
+ return '<span style="display:inline-block;padding:2px 8px;border-radius:4px;font-size:11px;font-weight:600;color:#0d1117;background:' + color + ';">' + escapeHtml(label) + '</span>';
+ }
+
+ function formatTimestamp(ts) {
+ if (!ts) return '-';
+ // CbObject DateTime serialized as ticks (100ns since 0001-01-01) or ISO string
+ var date;
+ if (typeof ts === 'number') {
+ // .NET-style ticks: convert to Unix ms
+ var unixMs = (ts - 621355968000000000) / 10000;
+ date = new Date(unixMs);
+ } else {
+ date = new Date(ts);
+ }
+ if (isNaN(date.getTime())) return '-';
+ return date.toLocaleTimeString();
+ }
+
+ var activeHistoryTab = 'workers';
+
+ function switchHistoryTab(tab) {
+ activeHistoryTab = tab;
+ var tabs = document.querySelectorAll('.history-tab');
+ for (var i = 0; i < tabs.length; i++) {
+ tabs[i].classList.toggle('active', tabs[i].getAttribute('data-tab') === tab);
+ }
+ document.getElementById('history-panel-workers').style.display = tab === 'workers' ? '' : 'none';
+ document.getElementById('history-panel-clients').style.display = tab === 'clients' ? '' : 'none';
+ }
+
+ function renderProvisioningHistory(events) {
+ var emptyState = document.getElementById('history-empty');
+ var table = document.getElementById('history-table');
+ var tbody = document.getElementById('history-table-body');
+
+ if (!events || events.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ return;
+ }
+
+ emptyState.style.display = 'none';
+ table.style.display = '';
+ tbody.innerHTML = '';
+
+ for (var i = 0; i < events.length; i++) {
+ var evt = events[i];
+ var tr = document.createElement('tr');
+ tr.innerHTML =
+ '<td style="color: #8b949e;">' + formatTimestamp(evt.ts) + '</td>' +
+ '<td>' + eventBadge(evt.type) + '</td>' +
+ '<td>' + escapeHtml(evt.worker_id || '') + '</td>' +
+ '<td>' + escapeHtml(evt.hostname || '') + '</td>';
+ tbody.appendChild(tr);
+ }
+ }
+
+ function clientHealthClass(dtMs) {
+ if (dtMs == null) return 'health-red';
+ var seconds = dtMs / 1000;
+ if (seconds < 30) return 'health-green';
+ if (seconds < 120) return 'health-yellow';
+ return 'health-red';
+ }
+
+ function renderClients(clients) {
+ var emptyState = document.getElementById('clients-empty');
+ var table = document.getElementById('clients-table');
+ var tbody = document.getElementById('clients-table-body');
+
+ if (!clients || clients.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ return;
+ }
+
+ emptyState.style.display = 'none';
+ table.style.display = '';
+ tbody.innerHTML = '';
+
+ for (var i = 0; i < clients.length; i++) {
+ var c = clients[i];
+ var dt = c.dt;
+ var hClass = clientHealthClass(dt);
+ var hTitle = dt != null ? 'Last seen ' + formatLastSeen(dt) : 'Never seen';
+
+ var sessionBadge = '';
+ if (c.session_id) {
+ sessionBadge = ' <span style="font-family:monospace;font-size:10px;color:#6e7681;" title="Session ' + escapeHtml(c.session_id) + '">' + escapeHtml(c.session_id.substring(0, 8)) + '</span>';
+ }
+
+ var tr = document.createElement('tr');
+ tr.innerHTML =
+ '<td style="text-align: center;"><span class="health-dot ' + hClass + '" title="' + escapeHtml(hTitle) + '"></span></td>' +
+ '<td>' + escapeHtml(c.id || '') + sessionBadge + '</td>' +
+ '<td>' + escapeHtml(c.hostname || '') + '</td>' +
+ '<td style="font-family: monospace; font-size: 12px; color: #8b949e;">' + escapeHtml(c.address || '') + '</td>' +
+ '<td style="text-align: right; color: #8b949e;">' + formatLastSeen(dt) + '</td>';
+ tbody.appendChild(tr);
+ }
+ }
+
+ function clientEventBadge(type) {
+ var colors = { connected: '#3fb950', disconnected: '#f85149', updated: '#d29922' };
+ var labels = { connected: 'Connected', disconnected: 'Disconnected', updated: 'Updated' };
+ var color = colors[type] || '#8b949e';
+ var label = labels[type] || type;
+ return '<span style="display:inline-block;padding:2px 8px;border-radius:4px;font-size:11px;font-weight:600;color:#0d1117;background:' + color + ';">' + escapeHtml(label) + '</span>';
+ }
+
+ function renderClientHistory(events) {
+ var emptyState = document.getElementById('client-history-empty');
+ var table = document.getElementById('client-history-table');
+ var tbody = document.getElementById('client-history-table-body');
+
+ if (!events || events.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ return;
+ }
+
+ emptyState.style.display = 'none';
+ table.style.display = '';
+ tbody.innerHTML = '';
+
+ for (var i = 0; i < events.length; i++) {
+ var evt = events[i];
+ var tr = document.createElement('tr');
+ tr.innerHTML =
+ '<td style="color: #8b949e;">' + formatTimestamp(evt.ts) + '</td>' +
+ '<td>' + clientEventBadge(evt.type) + '</td>' +
+ '<td>' + escapeHtml(evt.client_id || '') + '</td>' +
+ '<td>' + escapeHtml(evt.hostname || '') + '</td>';
+ tbody.appendChild(tr);
+ }
+ }
+
+ // Fetch-based polling fallback
+ var pollTimer = null;
+
+ async function fetchProvisioningHistory() {
+ try {
+ var response = await fetch(BASE_URL + '/orch/history?limit=50', {
+ headers: { 'Accept': 'application/json' }
+ });
+ if (response.ok) {
+ var data = await response.json();
+ renderProvisioningHistory(data.events || []);
+ }
+ } catch (e) {
+ console.error('Error fetching provisioning history:', e);
+ }
+ }
+
+ async function fetchClients() {
+ try {
+ var response = await fetch(BASE_URL + '/orch/clients', {
+ headers: { 'Accept': 'application/json' }
+ });
+ if (response.ok) {
+ var data = await response.json();
+ renderClients(data.clients || []);
+ }
+ } catch (e) {
+ console.error('Error fetching clients:', e);
+ }
+ }
+
+ async function fetchClientHistory() {
+ try {
+ var response = await fetch(BASE_URL + '/orch/clients/history?limit=50', {
+ headers: { 'Accept': 'application/json' }
+ });
+ if (response.ok) {
+ var data = await response.json();
+ renderClientHistory(data.client_events || []);
+ }
+ } catch (e) {
+ console.error('Error fetching client history:', e);
+ }
+ }
+
+ async function fetchDashboard() {
+ var banner = document.querySelector('zen-banner');
+ try {
+ var response = await fetch(BASE_URL + '/orch/agents', {
+ headers: { 'Accept': 'application/json' }
+ });
+
+ if (!response.ok) {
+ banner.setAttribute('cluster-status', 'degraded');
+ throw new Error('HTTP ' + response.status + ': ' + response.statusText);
+ }
+
+ renderDashboard(await response.json());
+ fetchProvisioningHistory();
+ fetchClients();
+ fetchClientHistory();
+ } catch (error) {
+ console.error('Error updating dashboard:', error);
+ showError(error.message);
+ banner.setAttribute('cluster-status', 'offline');
+ }
+ }
+
+ function startPolling() {
+ if (pollTimer) return;
+ fetchDashboard();
+ pollTimer = setInterval(fetchDashboard, REFRESH_INTERVAL);
+ }
+
+ function stopPolling() {
+ if (pollTimer) {
+ clearInterval(pollTimer);
+ pollTimer = null;
+ }
+ }
+
+ // WebSocket connection with automatic reconnect and polling fallback
+ var ws = null;
+
+ function connectWebSocket() {
+ var proto = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+ ws = new WebSocket(proto + '//' + window.location.host + '/orch/ws');
+
+ ws.onopen = function() {
+ stopPolling();
+ clearError();
+ };
+
+ ws.onmessage = function(event) {
+ try {
+ renderDashboard(JSON.parse(event.data));
+ } catch (e) {
+ console.error('WebSocket message parse error:', e);
+ }
+ };
+
+ ws.onclose = function() {
+ ws = null;
+ startPolling();
+ setTimeout(connectWebSocket, 3000);
+ };
+
+ ws.onerror = function() {
+ // onclose will fire after onerror
+ };
+ }
+
+ // Fetch orchestrator hostname for the banner
+ fetch(BASE_URL + '/orch/status', { headers: { 'Accept': 'application/json' } })
+ .then(function(r) { return r.ok ? r.json() : null; })
+ .then(function(d) {
+ if (d && d.hostname) {
+ document.querySelector('zen-banner').setAttribute('tagline', 'Orchestrator \u2014 ' + d.hostname);
+ }
+ })
+ .catch(function() {});
+
+ // Initial load via fetch, then try WebSocket
+ fetchDashboard();
+ connectWebSocket();
+ </script>
+</body>
+</html>
diff --git a/src/zenserver/frontend/html/pages/page.js b/src/zenserver/frontend/html/pages/page.js
index 3c2d3619a..592b699dc 100644
--- a/src/zenserver/frontend/html/pages/page.js
+++ b/src/zenserver/frontend/html/pages/page.js
@@ -3,6 +3,7 @@
"use strict";
import { WidgetHost } from "../util/widgets.js"
+import { Fetcher } from "../util/fetcher.js"
////////////////////////////////////////////////////////////////////////////////
export class PageBase extends WidgetHost
@@ -63,6 +64,7 @@ export class ZenPage extends PageBase
super(parent, ...args);
super.set_title("zen");
this.add_branding(parent);
+ this.add_service_nav(parent);
this.generate_crumbs();
}
@@ -78,6 +80,40 @@ export class ZenPage extends PageBase
root.tag("img").attr("src", "epicgames.ico").id("epic_logo");
}
+ add_service_nav(parent)
+ {
+ const nav = parent.tag().id("service_nav");
+
+ // Map service base URIs to dashboard links, this table is also used to detemine
+ // which links to show based on the services that are currently registered.
+
+ const service_dashboards = [
+ { base_uri: "/compute/", label: "Compute", href: "/dashboard/compute/compute.html" },
+ { base_uri: "/orch/", label: "Orchestrator", href: "/dashboard/compute/orchestrator.html" },
+ { base_uri: "/hub/", label: "Hub", href: "/dashboard/compute/hub.html" },
+ ];
+
+ new Fetcher().resource("/api/").json().then((data) => {
+ const services = data.services || [];
+ const uris = new Set(services.map(s => s.base_uri));
+
+ const links = service_dashboards.filter(d => uris.has(d.base_uri));
+
+ if (links.length === 0)
+ {
+ nav.inner().style.display = "none";
+ return;
+ }
+
+ for (const link of links)
+ {
+ nav.tag("a").text(link.label).attr("href", link.href);
+ }
+ }).catch(() => {
+ nav.inner().style.display = "none";
+ });
+ }
+
set_title(...args)
{
super.set_title(...args);
diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css
index 702bf9aa6..a80a1a4f6 100644
--- a/src/zenserver/frontend/html/zen.css
+++ b/src/zenserver/frontend/html/zen.css
@@ -80,6 +80,33 @@ input {
}
}
+/* service nav -------------------------------------------------------------- */
+
+#service_nav {
+ display: flex;
+ justify-content: center;
+ gap: 0.3em;
+ margin-bottom: 1.5em;
+ padding: 0.3em;
+ background-color: var(--theme_g3);
+ border: 1px solid var(--theme_g2);
+ border-radius: 0.4em;
+
+ a {
+ padding: 0.3em 0.9em;
+ border-radius: 0.3em;
+ font-size: 0.85em;
+ color: var(--theme_g1);
+ text-decoration: none;
+ }
+
+ a:hover {
+ background-color: var(--theme_p4);
+ color: var(--theme_g0);
+ text-decoration: none;
+ }
+}
+
/* links -------------------------------------------------------------------- */
a {
diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp
index bf0e294c5..a757cd594 100644
--- a/src/zenserver/hub/hubservice.cpp
+++ b/src/zenserver/hub/hubservice.cpp
@@ -845,7 +845,7 @@ HttpHubService::HttpHubService(std::filesystem::path HubBaseDir, std::filesystem
Obj << "currentInstanceCount" << m_Impl->GetInstanceCount();
Obj << "maxInstanceCount" << m_Impl->GetMaxInstanceCount();
Obj << "instanceLimit" << m_Impl->GetInstanceLimit();
- Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
},
HttpVerb::kGet);
}
diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
index d0a0db417..c63c618df 100644
--- a/src/zenserver/hub/zenhubserver.cpp
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -143,6 +143,8 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig)
ZEN_INFO("instantiating hub service");
m_HubService = std::make_unique<HttpHubService>(ServerConfig.DataDir / "hub", ServerConfig.DataDir / "servers");
m_HubService->SetNotificationEndpoint(ServerConfig.UpstreamNotificationEndpoint, ServerConfig.InstanceId);
+
+ m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
}
void
@@ -159,6 +161,11 @@ ZenHubServer::RegisterServices(const ZenHubServerConfig& ServerConfig)
{
m_Http->RegisterService(*m_ApiService);
}
+
+ if (m_FrontendService)
+ {
+ m_Http->RegisterService(*m_FrontendService);
+ }
}
void
diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h
index ac14362f0..4c56fdce5 100644
--- a/src/zenserver/hub/zenhubserver.h
+++ b/src/zenserver/hub/zenhubserver.h
@@ -2,6 +2,7 @@
#pragma once
+#include "frontend/frontend.h"
#include "zenserver.h"
namespace cxxopts {
@@ -81,8 +82,9 @@ private:
std::filesystem::path m_ContentRoot;
bool m_DebugOptionForcedCrash = false;
- std::unique_ptr<HttpHubService> m_HubService;
- std::unique_ptr<HttpApiService> m_ApiService;
+ std::unique_ptr<HttpHubService> m_HubService;
+ std::unique_ptr<HttpApiService> m_ApiService;
+ std::unique_ptr<HttpFrontendService> m_FrontendService;
void InitializeState(const ZenHubServerConfig& ServerConfig);
void InitializeServices(const ZenHubServerConfig& ServerConfig);
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index 3d81db656..bca26e87a 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -183,10 +183,15 @@ ZenStorageServer::RegisterServices()
m_Http->RegisterService(*m_AdminService);
+ if (m_ApiService)
+ {
+ m_Http->RegisterService(*m_ApiService);
+ }
+
#if ZEN_WITH_COMPUTE_SERVICES
- if (m_HttpFunctionService)
+ if (m_HttpComputeService)
{
- m_Http->RegisterService(*m_HttpFunctionService);
+ m_Http->RegisterService(*m_HttpComputeService);
}
#endif
}
@@ -279,8 +284,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
{
ZEN_OTEL_SPAN("InitializeComputeService");
- m_HttpFunctionService =
- std::make_unique<compute::HttpFunctionService>(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions");
+ m_HttpComputeService =
+ std::make_unique<compute::HttpComputeService>(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions");
}
#endif
@@ -316,6 +321,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
.AttachmentPassCount = ServerOptions.GcConfig.AttachmentPassCount};
m_GcScheduler.Initialize(GcConfig);
+ m_ApiService = std::make_unique<HttpApiService>(*m_Http);
+
// Create and register admin interface last to make sure all is properly initialized
m_AdminService = std::make_unique<HttpAdminService>(
m_GcScheduler,
@@ -832,7 +839,7 @@ ZenStorageServer::Cleanup()
Flush();
#if ZEN_WITH_COMPUTE_SERVICES
- m_HttpFunctionService.reset();
+ m_HttpComputeService.reset();
#endif
m_AdminService.reset();
diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h
index 456447a2a..5b163fc8e 100644
--- a/src/zenserver/storage/zenstorageserver.h
+++ b/src/zenserver/storage/zenstorageserver.h
@@ -25,7 +25,7 @@
#include "workspaces/httpworkspaces.h"
#if ZEN_WITH_COMPUTE_SERVICES
-# include <zencompute/httpfunctionservice.h>
+# include <zencompute/httpcomputeservice.h>
#endif
namespace zen {
@@ -93,7 +93,7 @@ private:
std::unique_ptr<HttpApiService> m_ApiService;
#if ZEN_WITH_COMPUTE_SERVICES
- std::unique_ptr<compute::HttpFunctionService> m_HttpFunctionService;
+ std::unique_ptr<compute::HttpComputeService> m_HttpComputeService;
#endif
};
diff --git a/src/zenserver/trace/tracerecorder.cpp b/src/zenserver/trace/tracerecorder.cpp
new file mode 100644
index 000000000..5dec20e18
--- /dev/null
+++ b/src/zenserver/trace/tracerecorder.cpp
@@ -0,0 +1,565 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "tracerecorder.h"
+
+#include <zencore/basicfile.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/uid.h>
+
+#include <asio.hpp>
+
+#include <atomic>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+namespace zen {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TraceSession : public std::enable_shared_from_this<TraceSession>
+{
+ TraceSession(asio::ip::tcp::socket&& Socket, const std::filesystem::path& OutputDir)
+ : m_Socket(std::move(Socket))
+ , m_OutputDir(OutputDir)
+ , m_SessionId(Oid::NewOid())
+ {
+ try
+ {
+ m_RemoteAddress = m_Socket.remote_endpoint().address().to_string();
+ }
+ catch (...)
+ {
+ m_RemoteAddress = "unknown";
+ }
+
+ ZEN_INFO("Trace session {} started from {}", m_SessionId, m_RemoteAddress);
+ }
+
+ ~TraceSession()
+ {
+ if (m_TraceFile.IsOpen())
+ {
+ m_TraceFile.Close();
+ }
+
+ ZEN_INFO("Trace session {} ended, {} bytes recorded to '{}'", m_SessionId, m_TotalBytesRecorded, m_TraceFilePath);
+ }
+
+ void Start() { ReadPreambleHeader(); }
+
+ bool IsActive() const { return m_Socket.is_open(); }
+
+ TraceSessionInfo GetInfo() const
+ {
+ TraceSessionInfo Info;
+ Info.SessionGuid = m_SessionGuid;
+ Info.TraceGuid = m_TraceGuid;
+ Info.ControlPort = m_ControlPort;
+ Info.TransportVersion = m_TransportVersion;
+ Info.ProtocolVersion = m_ProtocolVersion;
+ Info.RemoteAddress = m_RemoteAddress;
+ Info.BytesRecorded = m_TotalBytesRecorded;
+ Info.TraceFilePath = m_TraceFilePath;
+ return Info;
+ }
+
+private:
+ // Preamble format:
+ // [magic: 4 bytes][metadata_size: 2 bytes][metadata fields: variable][version: 2 bytes]
+ //
+ // Magic bytes: [0]=version_char ('2'-'9'), [1]='C', [2]='R', [3]='T'
+ //
+ // Metadata fields (repeated):
+ // [size: 1 byte][id: 1 byte][data: <size> bytes]
+ // Field 0: ControlPort (uint16)
+ // Field 1: SessionGuid (16 bytes)
+ // Field 2: TraceGuid (16 bytes)
+ //
+ // Version: [transport: 1 byte][protocol: 1 byte]
+
+ static constexpr size_t kMagicSize = 4;
+ static constexpr size_t kMetadataSizeFieldSize = 2;
+ static constexpr size_t kPreambleHeaderSize = kMagicSize + kMetadataSizeFieldSize;
+ static constexpr size_t kVersionSize = 2;
+ static constexpr size_t kPreambleBufferSize = 256;
+ static constexpr size_t kReadBufferSize = 64 * 1024;
+
+ void ReadPreambleHeader()
+ {
+ auto Self = shared_from_this();
+
+ // Read the first 6 bytes: 4 magic + 2 metadata size
+ asio::async_read(m_Socket,
+ asio::buffer(m_PreambleBuffer, kPreambleHeaderSize),
+ [this, Self](const asio::error_code& Ec, std::size_t /*BytesRead*/) {
+ if (Ec)
+ {
+ HandleReadError("preamble header", Ec);
+ return;
+ }
+
+ if (!ValidateMagic())
+ {
+ ZEN_WARN("Trace session {}: invalid trace magic header", m_SessionId);
+ CloseSocket();
+ return;
+ }
+
+ ReadPreambleMetadata();
+ });
+ }
+
+ bool ValidateMagic()
+ {
+ const uint8_t* Cursor = m_PreambleBuffer;
+
+ // Validate magic: bytes are version, 'C', 'R', 'T'
+ if (Cursor[3] != 'T' || Cursor[2] != 'R' || Cursor[1] != 'C')
+ {
+ return false;
+ }
+
+ if (Cursor[0] < '2' || Cursor[0] > '9')
+ {
+ return false;
+ }
+
+ // Extract the metadata fields size (does not include the trailing version bytes)
+ std::memcpy(&m_MetadataFieldsSize, Cursor + kMagicSize, sizeof(m_MetadataFieldsSize));
+
+ if (m_MetadataFieldsSize + kVersionSize > kPreambleBufferSize - kPreambleHeaderSize)
+ {
+ return false;
+ }
+
+ return true;
+ }
+
+ void ReadPreambleMetadata()
+ {
+ auto Self = shared_from_this();
+ size_t ReadSize = m_MetadataFieldsSize + kVersionSize;
+
+ // Read metadata fields + 2 version bytes
+ asio::async_read(m_Socket,
+ asio::buffer(m_PreambleBuffer + kPreambleHeaderSize, ReadSize),
+ [this, Self](const asio::error_code& Ec, std::size_t /*BytesRead*/) {
+ if (Ec)
+ {
+ HandleReadError("preamble metadata", Ec);
+ return;
+ }
+
+ if (!ParseMetadata())
+ {
+ ZEN_WARN("Trace session {}: malformed trace metadata", m_SessionId);
+ CloseSocket();
+ return;
+ }
+
+ if (!CreateTraceFile())
+ {
+ CloseSocket();
+ return;
+ }
+
+ // Write the full preamble to the trace file so it remains a valid .utrace
+ size_t PreambleSize = kPreambleHeaderSize + m_MetadataFieldsSize + kVersionSize;
+ std::error_code WriteEc;
+ m_TraceFile.Write(m_PreambleBuffer, PreambleSize, 0, WriteEc);
+
+ if (WriteEc)
+ {
+ ZEN_ERROR("Trace session {}: failed to write preamble: {}", m_SessionId, WriteEc.message());
+ CloseSocket();
+ return;
+ }
+
+ m_TotalBytesRecorded = PreambleSize;
+
+ ZEN_INFO("Trace session {}: metadata - TransportV{} ProtocolV{} ControlPort:{} SessionGuid:{} TraceGuid:{}",
+ m_SessionId,
+ m_TransportVersion,
+ m_ProtocolVersion,
+ m_ControlPort,
+ m_SessionGuid,
+ m_TraceGuid);
+
+ // Begin streaming trace data to disk
+ ReadMore();
+ });
+ }
+
+ bool ParseMetadata()
+ {
+ const uint8_t* Cursor = m_PreambleBuffer + kPreambleHeaderSize;
+ int32_t Remaining = static_cast<int32_t>(m_MetadataFieldsSize);
+
+ while (Remaining >= 2)
+ {
+ uint8_t FieldSize = Cursor[0];
+ uint8_t FieldId = Cursor[1];
+ Cursor += 2;
+ Remaining -= 2;
+
+ if (Remaining < FieldSize)
+ {
+ return false;
+ }
+
+ switch (FieldId)
+ {
+ case 0: // ControlPort
+ if (FieldSize >= sizeof(uint16_t))
+ {
+ std::memcpy(&m_ControlPort, Cursor, sizeof(uint16_t));
+ }
+ break;
+ case 1: // SessionGuid
+ if (FieldSize >= sizeof(Guid))
+ {
+ std::memcpy(&m_SessionGuid, Cursor, sizeof(Guid));
+ }
+ break;
+ case 2: // TraceGuid
+ if (FieldSize >= sizeof(Guid))
+ {
+ std::memcpy(&m_TraceGuid, Cursor, sizeof(Guid));
+ }
+ break;
+ }
+
+ Cursor += FieldSize;
+ Remaining -= FieldSize;
+ }
+
+ // Metadata should be fully consumed
+ if (Remaining != 0)
+ {
+ return false;
+ }
+
+ // Version bytes follow immediately after the metadata fields
+ const uint8_t* VersionPtr = m_PreambleBuffer + kPreambleHeaderSize + m_MetadataFieldsSize;
+ m_TransportVersion = VersionPtr[0];
+ m_ProtocolVersion = VersionPtr[1];
+
+ return true;
+ }
+
+ bool CreateTraceFile()
+ {
+ m_TraceFilePath = m_OutputDir / fmt::format("{}.utrace", m_SessionId);
+
+ try
+ {
+ m_TraceFile.Open(m_TraceFilePath, BasicFile::Mode::kTruncate);
+ ZEN_INFO("Trace session {} writing to '{}'", m_SessionId, m_TraceFilePath);
+ return true;
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("Trace session {}: failed to create trace file '{}': {}", m_SessionId, m_TraceFilePath, Ex.what());
+ return false;
+ }
+ }
+
+ void ReadMore()
+ {
+ auto Self = shared_from_this();
+
+ m_Socket.async_read_some(asio::buffer(m_ReadBuffer, kReadBufferSize),
+ [this, Self](const asio::error_code& Ec, std::size_t BytesRead) {
+ if (!Ec)
+ {
+ if (BytesRead > 0 && m_TraceFile.IsOpen())
+ {
+ std::error_code WriteEc;
+ const uint64_t FileOffset = m_TotalBytesRecorded;
+ m_TraceFile.Write(m_ReadBuffer, BytesRead, FileOffset, WriteEc);
+
+ if (WriteEc)
+ {
+ ZEN_ERROR("Trace session {}: write error: {}", m_SessionId, WriteEc.message());
+ CloseSocket();
+ return;
+ }
+
+ m_TotalBytesRecorded += BytesRead;
+ }
+
+ ReadMore();
+ }
+ else if (Ec == asio::error::eof)
+ {
+ ZEN_DEBUG("Trace session {} connection closed by peer", m_SessionId);
+ CloseSocket();
+ }
+ else if (Ec == asio::error::operation_aborted)
+ {
+ ZEN_DEBUG("Trace session {} operation aborted", m_SessionId);
+ }
+ else
+ {
+ ZEN_WARN("Trace session {} read error: {}", m_SessionId, Ec.message());
+ CloseSocket();
+ }
+ });
+ }
+
+ void HandleReadError(const char* Phase, const asio::error_code& Ec)
+ {
+ if (Ec == asio::error::eof)
+ {
+ ZEN_DEBUG("Trace session {}: connection closed during {}", m_SessionId, Phase);
+ }
+ else if (Ec == asio::error::operation_aborted)
+ {
+ ZEN_DEBUG("Trace session {}: operation aborted during {}", m_SessionId, Phase);
+ }
+ else
+ {
+ ZEN_WARN("Trace session {}: error during {}: {}", m_SessionId, Phase, Ec.message());
+ }
+
+ CloseSocket();
+ }
+
+ void CloseSocket()
+ {
+ std::error_code Ec;
+ m_Socket.close(Ec);
+
+ if (m_TraceFile.IsOpen())
+ {
+ m_TraceFile.Close();
+ }
+ }
+
+ asio::ip::tcp::socket m_Socket;
+ std::filesystem::path m_OutputDir;
+ std::filesystem::path m_TraceFilePath;
+ BasicFile m_TraceFile;
+ Oid m_SessionId;
+ std::string m_RemoteAddress;
+
+ // Preamble parsing
+ uint8_t m_PreambleBuffer[kPreambleBufferSize] = {};
+ uint16_t m_MetadataFieldsSize = 0;
+
+ // Extracted metadata
+ Guid m_SessionGuid{};
+ Guid m_TraceGuid{};
+ uint16_t m_ControlPort = 0;
+ uint8_t m_TransportVersion = 0;
+ uint8_t m_ProtocolVersion = 0;
+
+ // Streaming
+ uint8_t m_ReadBuffer[kReadBufferSize];
+ uint64_t m_TotalBytesRecorded = 0;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TraceRecorder::Impl
+{
+ Impl() : m_IoContext(), m_Acceptor(m_IoContext) {}
+
+ ~Impl() { Shutdown(); }
+
+ void Initialize(uint16_t InPort, const std::filesystem::path& OutputDir)
+ {
+ std::lock_guard<std::mutex> Lock(m_Mutex);
+
+ if (m_IsRunning)
+ {
+ ZEN_WARN("TraceRecorder already initialized");
+ return;
+ }
+
+ m_OutputDir = OutputDir;
+
+ try
+ {
+ // Create output directory if it doesn't exist
+ CreateDirectories(m_OutputDir);
+
+ // Configure acceptor
+ m_Acceptor.open(asio::ip::tcp::v4());
+ m_Acceptor.set_option(asio::socket_base::reuse_address(true));
+ m_Acceptor.bind(asio::ip::tcp::endpoint(asio::ip::tcp::v4(), InPort));
+ m_Acceptor.listen();
+
+ m_Port = m_Acceptor.local_endpoint().port();
+
+ ZEN_INFO("TraceRecorder listening on port {}, output directory: '{}'", m_Port, m_OutputDir);
+
+ m_IsRunning = true;
+
+ // Start accepting connections
+ StartAccept();
+
+ // Start IO thread
+ m_IoThread = std::thread([this]() {
+ try
+ {
+ m_IoContext.run();
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("TraceRecorder IO thread exception: {}", Ex.what());
+ }
+ });
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("Failed to initialize TraceRecorder: {}", Ex.what());
+ m_IsRunning = false;
+ throw;
+ }
+ }
+
+ void Shutdown()
+ {
+ std::lock_guard<std::mutex> Lock(m_Mutex);
+
+ if (!m_IsRunning)
+ {
+ return;
+ }
+
+ ZEN_INFO("TraceRecorder shutting down");
+
+ m_IsRunning = false;
+
+ std::error_code Ec;
+ m_Acceptor.close(Ec);
+
+ m_IoContext.stop();
+
+ if (m_IoThread.joinable())
+ {
+ m_IoThread.join();
+ }
+
+ {
+ std::lock_guard<std::mutex> SessionLock(m_SessionsMutex);
+ m_Sessions.clear();
+ }
+
+ ZEN_INFO("TraceRecorder shutdown complete");
+ }
+
+ bool IsRunning() const { return m_IsRunning; }
+
+ uint16_t GetPort() const { return m_Port; }
+
+ std::vector<TraceSessionInfo> GetActiveSessions() const
+ {
+ std::lock_guard<std::mutex> Lock(m_SessionsMutex);
+
+ std::vector<TraceSessionInfo> Result;
+ for (const auto& WeakSession : m_Sessions)
+ {
+ if (auto Session = WeakSession.lock())
+ {
+ if (Session->IsActive())
+ {
+ Result.push_back(Session->GetInfo());
+ }
+ }
+ }
+ return Result;
+ }
+
+private:
+ void StartAccept()
+ {
+ auto Socket = std::make_shared<asio::ip::tcp::socket>(m_IoContext);
+
+ m_Acceptor.async_accept(*Socket, [this, Socket](const asio::error_code& Ec) {
+ if (!Ec)
+ {
+ auto Session = std::make_shared<TraceSession>(std::move(*Socket), m_OutputDir);
+
+ {
+ std::lock_guard<std::mutex> Lock(m_SessionsMutex);
+
+ // Prune expired sessions while adding the new one
+ std::erase_if(m_Sessions, [](const std::weak_ptr<TraceSession>& Wp) { return Wp.expired(); });
+ m_Sessions.push_back(Session);
+ }
+
+ Session->Start();
+ }
+ else if (Ec != asio::error::operation_aborted)
+ {
+ ZEN_WARN("Accept error: {}", Ec.message());
+ }
+
+ // Continue accepting if still running
+ if (m_IsRunning)
+ {
+ StartAccept();
+ }
+ });
+ }
+
+ asio::io_context m_IoContext;
+ asio::ip::tcp::acceptor m_Acceptor;
+ std::thread m_IoThread;
+ std::filesystem::path m_OutputDir;
+ std::mutex m_Mutex;
+ std::atomic<bool> m_IsRunning{false};
+ uint16_t m_Port = 0;
+
+ mutable std::mutex m_SessionsMutex;
+ std::vector<std::weak_ptr<TraceSession>> m_Sessions;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TraceRecorder::TraceRecorder() : m_Impl(std::make_unique<Impl>())
+{
+}
+
+TraceRecorder::~TraceRecorder()
+{
+ Shutdown();
+}
+
+void
+TraceRecorder::Initialize(uint16_t InPort, const std::filesystem::path& OutputDir)
+{
+ m_Impl->Initialize(InPort, OutputDir);
+}
+
+void
+TraceRecorder::Shutdown()
+{
+ m_Impl->Shutdown();
+}
+
+bool
+TraceRecorder::IsRunning() const
+{
+ return m_Impl->IsRunning();
+}
+
+uint16_t
+TraceRecorder::GetPort() const
+{
+ return m_Impl->GetPort();
+}
+
+std::vector<TraceSessionInfo>
+TraceRecorder::GetActiveSessions() const
+{
+ return m_Impl->GetActiveSessions();
+}
+
+} // namespace zen
diff --git a/src/zenserver/trace/tracerecorder.h b/src/zenserver/trace/tracerecorder.h
new file mode 100644
index 000000000..48857aec8
--- /dev/null
+++ b/src/zenserver/trace/tracerecorder.h
@@ -0,0 +1,46 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/guid.h>
+#include <zencore/zencore.h>
+
+#include <filesystem>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace zen {
+
+struct TraceSessionInfo
+{
+ Guid SessionGuid{};
+ Guid TraceGuid{};
+ uint16_t ControlPort = 0;
+ uint8_t TransportVersion = 0;
+ uint8_t ProtocolVersion = 0;
+ std::string RemoteAddress;
+ uint64_t BytesRecorded = 0;
+ std::filesystem::path TraceFilePath;
+};
+
+class TraceRecorder
+{
+public:
+ TraceRecorder();
+ ~TraceRecorder();
+
+ void Initialize(uint16_t InPort, const std::filesystem::path& OutputDir);
+ void Shutdown();
+
+ bool IsRunning() const;
+ uint16_t GetPort() const;
+
+ std::vector<TraceSessionInfo> GetActiveSessions() const;
+
+private:
+ struct Impl;
+ std::unique_ptr<Impl> m_Impl;
+};
+
+} // namespace zen \ No newline at end of file
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index 9ab51beb2..915b6a3b1 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -27,6 +27,7 @@ target("zenserver")
add_packages("json11")
add_packages("lua")
add_packages("consul")
+ add_packages("nomad")
if has_config("zenmimalloc") then
add_packages("mimalloc")
@@ -36,6 +37,14 @@ target("zenserver")
add_packages("sentry-native")
end
+ if has_config("zenhorde") then
+ add_deps("zenhorde")
+ end
+
+ if has_config("zennomad") then
+ add_deps("zennomad")
+ end
+
if is_mode("release") then
set_optimize("fastest")
end
@@ -145,4 +154,14 @@ target("zenserver")
end
copy_if_newer(path.join(installdir, "bin", consul_bin), path.join(target:targetdir(), consul_bin), consul_bin)
end
+
+ local nomad_pkg = target:pkg("nomad")
+ if nomad_pkg then
+ local installdir = nomad_pkg:installdir()
+ local nomad_bin = "nomad"
+ if is_plat("windows") then
+ nomad_bin = "nomad.exe"
+ end
+ copy_if_newer(path.join(installdir, "bin", nomad_bin), path.join(target:targetdir(), nomad_bin), nomad_bin)
+ end
end)