aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver
diff options
context:
space:
mode:
authorLiam Mitchell <[email protected]>2026-03-09 19:06:36 -0700
committerLiam Mitchell <[email protected]>2026-03-09 19:06:36 -0700
commitd1abc50ee9d4fb72efc646e17decafea741caa34 (patch)
treee4288e00f2f7ca0391b83d986efcb69d3ba66a83 /src/zenserver
parentAllow requests with invalid content-types unless specified in command line or... (diff)
parentupdated chunk–block analyser (#818) (diff)
downloadzen-d1abc50ee9d4fb72efc646e17decafea741caa34.tar.xz
zen-d1abc50ee9d4fb72efc646e17decafea741caa34.zip
Merge branch 'main' into lm/restrict-content-type
Diffstat (limited to 'src/zenserver')
-rw-r--r--src/zenserver/compute/computeserver.cpp1021
-rw-r--r--src/zenserver/compute/computeserver.h188
-rw-r--r--src/zenserver/config/config.cpp93
-rw-r--r--src/zenserver/config/config.h44
-rw-r--r--src/zenserver/config/luaconfig.h2
-rw-r--r--src/zenserver/diag/diagsvcs.cpp37
-rw-r--r--src/zenserver/diag/diagsvcs.h15
-rw-r--r--src/zenserver/diag/logging.cpp61
-rw-r--r--src/zenserver/diag/otlphttp.cpp63
-rw-r--r--src/zenserver/diag/otlphttp.h28
-rw-r--r--src/zenserver/frontend/frontend.cpp56
-rw-r--r--src/zenserver/frontend/frontend.h7
-rw-r--r--src/zenserver/frontend/html.zipbin163229 -> 406051 bytes
-rw-r--r--src/zenserver/frontend/html/404.html486
-rw-r--r--src/zenserver/frontend/html/banner.js338
-rw-r--r--src/zenserver/frontend/html/compute/compute.html929
-rw-r--r--src/zenserver/frontend/html/compute/hub.html170
-rw-r--r--src/zenserver/frontend/html/compute/index.html1
-rw-r--r--src/zenserver/frontend/html/compute/orchestrator.html674
-rw-r--r--src/zenserver/frontend/html/epicgames.icobin0 -> 65288 bytes
-rw-r--r--src/zenserver/frontend/html/favicon.icobin65288 -> 12957 bytes
-rw-r--r--src/zenserver/frontend/html/index.html3
-rw-r--r--src/zenserver/frontend/html/nav.js79
-rw-r--r--src/zenserver/frontend/html/pages/cache.js690
-rw-r--r--src/zenserver/frontend/html/pages/compute.js693
-rw-r--r--src/zenserver/frontend/html/pages/cookartifacts.js397
-rw-r--r--src/zenserver/frontend/html/pages/entry.js341
-rw-r--r--src/zenserver/frontend/html/pages/hub.js122
-rw-r--r--src/zenserver/frontend/html/pages/info.js261
-rw-r--r--src/zenserver/frontend/html/pages/map.js4
-rw-r--r--src/zenserver/frontend/html/pages/metrics.js232
-rw-r--r--src/zenserver/frontend/html/pages/oplog.js4
-rw-r--r--src/zenserver/frontend/html/pages/orchestrator.js405
-rw-r--r--src/zenserver/frontend/html/pages/page.js120
-rw-r--r--src/zenserver/frontend/html/pages/project.js2
-rw-r--r--src/zenserver/frontend/html/pages/projects.js447
-rw-r--r--src/zenserver/frontend/html/pages/sessions.js61
-rw-r--r--src/zenserver/frontend/html/pages/start.js327
-rw-r--r--src/zenserver/frontend/html/pages/stat.js2
-rw-r--r--src/zenserver/frontend/html/pages/tree.js2
-rw-r--r--src/zenserver/frontend/html/pages/zcache.js8
-rw-r--r--src/zenserver/frontend/html/theme.js116
-rw-r--r--src/zenserver/frontend/html/util/compactbinary.js4
-rw-r--r--src/zenserver/frontend/html/util/friendly.js21
-rw-r--r--src/zenserver/frontend/html/util/widgets.js138
-rw-r--r--src/zenserver/frontend/html/zen.css824
-rw-r--r--src/zenserver/frontend/zipfs.cpp20
-rw-r--r--src/zenserver/frontend/zipfs.h8
-rw-r--r--src/zenserver/hub/hubservice.cpp68
-rw-r--r--src/zenserver/hub/hubservice.h7
-rw-r--r--src/zenserver/hub/zenhubserver.cpp11
-rw-r--r--src/zenserver/hub/zenhubserver.h6
-rw-r--r--src/zenserver/main.cpp69
-rw-r--r--src/zenserver/sessions/httpsessions.cpp264
-rw-r--r--src/zenserver/sessions/httpsessions.h55
-rw-r--r--src/zenserver/sessions/sessions.cpp150
-rw-r--r--src/zenserver/sessions/sessions.h83
-rw-r--r--src/zenserver/storage/admin/admin.cpp6
-rw-r--r--src/zenserver/storage/buildstore/httpbuildstore.cpp153
-rw-r--r--src/zenserver/storage/buildstore/httpbuildstore.h7
-rw-r--r--src/zenserver/storage/cache/httpstructuredcache.cpp141
-rw-r--r--src/zenserver/storage/cache/httpstructuredcache.h11
-rw-r--r--src/zenserver/storage/projectstore/httpprojectstore.cpp279
-rw-r--r--src/zenserver/storage/projectstore/httpprojectstore.h5
-rw-r--r--src/zenserver/storage/storageconfig.cpp1
-rw-r--r--src/zenserver/storage/storageconfig.h2
-rw-r--r--src/zenserver/storage/workspaces/httpworkspaces.cpp12
-rw-r--r--src/zenserver/storage/workspaces/httpworkspaces.h5
-rw-r--r--src/zenserver/storage/zenstorageserver.cpp55
-rw-r--r--src/zenserver/storage/zenstorageserver.h30
-rw-r--r--src/zenserver/trace/tracerecorder.cpp565
-rw-r--r--src/zenserver/trace/tracerecorder.h46
-rw-r--r--src/zenserver/xmake.lua40
-rw-r--r--src/zenserver/zenserver.cpp167
-rw-r--r--src/zenserver/zenserver.h38
-rw-r--r--src/zenserver/zenserver.rc2
76 files changed, 11192 insertions, 630 deletions
diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp
new file mode 100644
index 000000000..c64f081b3
--- /dev/null
+++ b/src/zenserver/compute/computeserver.cpp
@@ -0,0 +1,1021 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "computeserver.h"
+#include <zencompute/cloudmetadata.h>
+#include <zencompute/httpcomputeservice.h>
+#include <zencompute/httporchestrator.h>
+#if ZEN_WITH_COMPUTE_SERVICES
+
+# include <zencore/fmtutils.h>
+# include <zencore/memory/llm.h>
+# include <zencore/memory/memorytrace.h>
+# include <zencore/memory/tagtrace.h>
+# include <zencore/scopeguard.h>
+# include <zencore/sentryintegration.h>
+# include <zencore/system.h>
+# include <zencore/compactbinarybuilder.h>
+# include <zencore/windows.h>
+# include <zenhttp/httpclient.h>
+# include <zenhttp/httpapiservice.h>
+# include <zenstore/cidstore.h>
+# include <zenutil/service.h>
+# if ZEN_WITH_HORDE
+# include <zenhorde/hordeconfig.h>
+# include <zenhorde/hordeprovisioner.h>
+# endif
+# if ZEN_WITH_NOMAD
+# include <zennomad/nomadconfig.h>
+# include <zennomad/nomadprovisioner.h>
+# endif
+
+ZEN_THIRD_PARTY_INCLUDES_START
+# include <cxxopts.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+void
+ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options)
+{
+ Options.add_option("compute",
+ "",
+ "max-actions",
+ "Maximum number of concurrent local actions (0 = auto)",
+ cxxopts::value<int32_t>(m_ServerOptions.MaxConcurrentActions)->default_value("0"),
+ "");
+
+ Options.add_option("compute",
+ "",
+ "upstream-notification-endpoint",
+ "Endpoint URL for upstream notifications",
+ cxxopts::value<std::string>(m_ServerOptions.UpstreamNotificationEndpoint)->default_value(""),
+ "");
+
+ Options.add_option("compute",
+ "",
+ "instance-id",
+ "Instance ID for use in notifications",
+ cxxopts::value<std::string>(m_ServerOptions.InstanceId)->default_value(""),
+ "");
+
+ Options.add_option("compute",
+ "",
+ "coordinator-endpoint",
+ "Endpoint URL for coordinator service",
+ cxxopts::value<std::string>(m_ServerOptions.CoordinatorEndpoint)->default_value(""),
+ "");
+
+ Options.add_option("compute",
+ "",
+ "idms",
+ "Enable IDMS cloud detection; optionally specify a custom probe endpoint",
+ cxxopts::value<std::string>(m_ServerOptions.IdmsEndpoint)->default_value("")->implicit_value("auto"),
+ "");
+
+ Options.add_option("compute",
+ "",
+ "worker-websocket",
+ "Use WebSocket for worker-orchestrator link (instant reachability detection)",
+ cxxopts::value<bool>(m_ServerOptions.EnableWorkerWebSocket)->default_value("false"),
+ "");
+
+# if ZEN_WITH_HORDE
+ // Horde provisioning options
+ Options.add_option("horde",
+ "",
+ "horde-enabled",
+ "Enable Horde worker provisioning",
+ cxxopts::value<bool>(m_ServerOptions.HordeConfig.Enabled)->default_value("false"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-server",
+ "Horde server URL",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.ServerUrl)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-token",
+ "Horde authentication token",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.AuthToken)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-pool",
+ "Horde pool name",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.Pool)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-cluster",
+ "Horde cluster ID ('default' or '_auto' for auto-resolve)",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.Cluster)->default_value("default"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-mode",
+ "Horde connection mode (direct, tunnel, relay)",
+ cxxopts::value<std::string>(m_HordeModeStr)->default_value("direct"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-encryption",
+ "Horde transport encryption (none, aes)",
+ cxxopts::value<std::string>(m_HordeEncryptionStr)->default_value("none"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-max-cores",
+ "Maximum number of Horde cores to provision",
+ cxxopts::value<int>(m_ServerOptions.HordeConfig.MaxCores)->default_value("2048"),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-host",
+ "Host address for Horde agents to connect back to",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.HostAddress)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-condition",
+ "Additional Horde agent filter condition",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.Condition)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-binaries",
+ "Path to directory containing zenserver binary for remote upload",
+ cxxopts::value<std::string>(m_ServerOptions.HordeConfig.BinariesPath)->default_value(""),
+ "");
+
+ Options.add_option("horde",
+ "",
+ "horde-zen-service-port",
+ "Port number for Zen service communication",
+ cxxopts::value<uint16_t>(m_ServerOptions.HordeConfig.ZenServicePort)->default_value("8558"),
+ "");
+# endif
+
+# if ZEN_WITH_NOMAD
+ // Nomad provisioning options
+ Options.add_option("nomad",
+ "",
+ "nomad-enabled",
+ "Enable Nomad worker provisioning",
+ cxxopts::value<bool>(m_ServerOptions.NomadConfig.Enabled)->default_value("false"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-server",
+ "Nomad HTTP API URL",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.ServerUrl)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-token",
+ "Nomad ACL token",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.AclToken)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-datacenter",
+ "Nomad target datacenter",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.Datacenter)->default_value("dc1"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-namespace",
+ "Nomad namespace",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.Namespace)->default_value("default"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-region",
+ "Nomad region (empty for server default)",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.Region)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-driver",
+ "Nomad task driver (raw_exec, docker)",
+ cxxopts::value<std::string>(m_NomadDriverStr)->default_value("raw_exec"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-distribution",
+ "Binary distribution mode (predeployed, artifact)",
+ cxxopts::value<std::string>(m_NomadDistributionStr)->default_value("predeployed"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-binary-path",
+ "Path to zenserver on Nomad clients (predeployed mode)",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.BinaryPath)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-artifact-source",
+ "URL to download zenserver binary (artifact mode)",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.ArtifactSource)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-docker-image",
+ "Docker image for zenserver (docker driver)",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.DockerImage)->default_value(""),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-max-jobs",
+ "Maximum concurrent Nomad jobs",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.MaxJobs)->default_value("64"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-cpu-mhz",
+ "CPU MHz allocated per Nomad task",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.CpuMhz)->default_value("1000"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-memory-mb",
+ "Memory MB allocated per Nomad task",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.MemoryMb)->default_value("2048"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-cores-per-job",
+ "Estimated cores per Nomad job (for scaling)",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.CoresPerJob)->default_value("32"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-max-cores",
+ "Maximum total cores to provision via Nomad",
+ cxxopts::value<int>(m_ServerOptions.NomadConfig.MaxCores)->default_value("2048"),
+ "");
+
+ Options.add_option("nomad",
+ "",
+ "nomad-job-prefix",
+ "Prefix for generated Nomad job IDs",
+ cxxopts::value<std::string>(m_ServerOptions.NomadConfig.JobPrefix)->default_value("zenserver-worker"),
+ "");
+# endif
+}
+
+void
+ZenComputeServerConfigurator::AddConfigOptions(LuaConfig::Options& Options)
+{
+ ZEN_UNUSED(Options);
+}
+
+void
+ZenComputeServerConfigurator::ApplyOptions(cxxopts::Options& Options)
+{
+ ZEN_UNUSED(Options);
+}
+
+void
+ZenComputeServerConfigurator::OnConfigFileParsed(LuaConfig::Options& LuaOptions)
+{
+ ZEN_UNUSED(LuaOptions);
+}
+
+void
+ZenComputeServerConfigurator::ValidateOptions()
+{
+# if ZEN_WITH_HORDE
+ horde::FromString(m_ServerOptions.HordeConfig.Mode, m_HordeModeStr);
+ horde::FromString(m_ServerOptions.HordeConfig.EncryptionMode, m_HordeEncryptionStr);
+# endif
+
+# if ZEN_WITH_NOMAD
+ nomad::FromString(m_ServerOptions.NomadConfig.TaskDriver, m_NomadDriverStr);
+ nomad::FromString(m_ServerOptions.NomadConfig.BinDistribution, m_NomadDistributionStr);
+# endif
+}
+
+///////////////////////////////////////////////////////////////////////////
+
+ZenComputeServer::ZenComputeServer()
+{
+}
+
+ZenComputeServer::~ZenComputeServer()
+{
+ Cleanup();
+}
+
+int
+ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServerState::ZenServerEntry* ServerEntry)
+{
+ ZEN_TRACE_CPU("ZenComputeServer::Initialize");
+ ZEN_MEMSCOPE(GetZenserverTag());
+
+ ZEN_INFO(ZEN_APP_NAME " initializing in COMPUTE server mode");
+
+ const int EffectiveBasePort = ZenServerBase::Initialize(ServerConfig, ServerEntry);
+ if (EffectiveBasePort < 0)
+ {
+ return EffectiveBasePort;
+ }
+
+ m_CoordinatorEndpoint = ServerConfig.CoordinatorEndpoint;
+ m_InstanceId = ServerConfig.InstanceId;
+ m_EnableWorkerWebSocket = ServerConfig.EnableWorkerWebSocket;
+
+ // This is a workaround to make sure we can have automated tests. Without
+ // this the ranges for different child zen compute processes could overlap with
+ // the main test range.
+ ZenServerEnvironment::SetBaseChildId(2000);
+
+ m_DebugOptionForcedCrash = ServerConfig.ShouldCrash;
+
+ InitializeState(ServerConfig);
+ InitializeServices(ServerConfig);
+ RegisterServices(ServerConfig);
+
+ ZenServerBase::Finalize();
+
+ return EffectiveBasePort;
+}
+
+void
+ZenComputeServer::Cleanup()
+{
+ ZEN_TRACE_CPU("ZenComputeServer::Cleanup");
+ ZEN_INFO(ZEN_APP_NAME " cleaning up");
+ try
+ {
+ // Cancel the maintenance timer so it stops re-enqueuing before we
+ // tear down the provisioners it references.
+ m_ProvisionerMaintenanceTimer.cancel();
+ m_AnnounceTimer.cancel();
+
+# if ZEN_WITH_HORDE
+ // Shut down Horde provisioner first — this signals all agent threads
+ // to exit and joins them before we tear down HTTP services.
+ m_HordeProvisioner.reset();
+# endif
+
+# if ZEN_WITH_NOMAD
+ // Shut down Nomad provisioner — stops the management thread and
+ // sends stop requests for all tracked jobs.
+ m_NomadProvisioner.reset();
+# endif
+
+ // Close the orchestrator WebSocket client before stopping the io_context
+ m_WsReconnectTimer.cancel();
+ if (m_OrchestratorWsClient)
+ {
+ m_OrchestratorWsClient->Close();
+ m_OrchestratorWsClient.reset();
+ }
+ m_OrchestratorWsHandler.reset();
+
+ ResolveCloudMetadata();
+ m_CloudMetadata.reset();
+
+ // Shut down services that own threads or use the io_context before we
+ // stop the io_context and close the HTTP server.
+ if (m_OrchestratorService)
+ {
+ m_OrchestratorService->Shutdown();
+ }
+ if (m_ComputeService)
+ {
+ m_ComputeService->Shutdown();
+ }
+
+ m_IoContext.stop();
+ if (m_IoRunner.joinable())
+ {
+ m_IoRunner.join();
+ }
+
+ ShutdownServices();
+
+ if (m_Http)
+ {
+ m_Http->Close();
+ }
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("exception thrown during Cleanup() in {}: '{}'", ZEN_APP_NAME, Ex.what());
+ }
+}
+
+void
+ZenComputeServer::InitializeState(const ZenComputeServerConfig& ServerConfig)
+{
+ ZEN_UNUSED(ServerConfig);
+}
+
+void
+ZenComputeServer::InitializeServices(const ZenComputeServerConfig& ServerConfig)
+{
+ ZEN_TRACE_CPU("ZenComputeServer::InitializeServices");
+ ZEN_INFO("initializing compute services");
+
+ CidStoreConfiguration Config;
+ Config.RootDirectory = m_DataRoot / "cas";
+
+ m_CidStore = std::make_unique<CidStore>(m_GcManager);
+ m_CidStore->Initialize(Config);
+
+ if (!ServerConfig.IdmsEndpoint.empty())
+ {
+ ZEN_INFO("detecting cloud environment (async)");
+ if (ServerConfig.IdmsEndpoint == "auto")
+ {
+ m_CloudMetadataFuture = std::async(std::launch::async, [DataDir = ServerConfig.DataDir] {
+ return std::make_unique<zen::compute::CloudMetadata>(DataDir / "cloud");
+ });
+ }
+ else
+ {
+ ZEN_INFO("using custom IDMS endpoint: {}", ServerConfig.IdmsEndpoint);
+ m_CloudMetadataFuture = std::async(std::launch::async, [DataDir = ServerConfig.DataDir, Endpoint = ServerConfig.IdmsEndpoint] {
+ return std::make_unique<zen::compute::CloudMetadata>(DataDir / "cloud", Endpoint);
+ });
+ }
+ }
+
+ ZEN_INFO("instantiating API service");
+ m_ApiService = std::make_unique<zen::HttpApiService>(*m_Http);
+
+ ZEN_INFO("instantiating orchestrator service");
+ m_OrchestratorService =
+ std::make_unique<zen::compute::HttpOrchestratorService>(ServerConfig.DataDir / "orch", ServerConfig.EnableWorkerWebSocket);
+
+ ZEN_INFO("instantiating function service");
+ m_ComputeService = std::make_unique<zen::compute::HttpComputeService>(*m_CidStore,
+ m_StatsService,
+ ServerConfig.DataDir / "functions",
+ ServerConfig.MaxConcurrentActions);
+
+ m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
+
+# if ZEN_WITH_NOMAD
+ // Nomad provisioner
+ if (ServerConfig.NomadConfig.Enabled && !ServerConfig.NomadConfig.ServerUrl.empty())
+ {
+ ZEN_INFO("instantiating Nomad provisioner (server: {})", ServerConfig.NomadConfig.ServerUrl);
+
+ const auto& NomadCfg = ServerConfig.NomadConfig;
+
+ if (!NomadCfg.Validate())
+ {
+ ZEN_ERROR("invalid Nomad configuration");
+ }
+ else
+ {
+ ExtendableStringBuilder<256> OrchestratorEndpoint;
+ OrchestratorEndpoint << m_Http->GetServiceUri(m_OrchestratorService.get());
+ if (auto View = OrchestratorEndpoint.ToView(); !View.empty() && View.back() != '/')
+ {
+ OrchestratorEndpoint << '/';
+ }
+
+ m_NomadProvisioner = std::make_unique<nomad::NomadProvisioner>(NomadCfg, OrchestratorEndpoint);
+ }
+ }
+# endif
+
+# if ZEN_WITH_HORDE
+ // Horde provisioner
+ if (ServerConfig.HordeConfig.Enabled && !ServerConfig.HordeConfig.ServerUrl.empty())
+ {
+ ZEN_INFO("instantiating Horde provisioner (server: {})", ServerConfig.HordeConfig.ServerUrl);
+
+ const auto& HordeConfig = ServerConfig.HordeConfig;
+
+ if (!HordeConfig.Validate())
+ {
+ ZEN_ERROR("invalid Horde configuration");
+ }
+ else
+ {
+ ExtendableStringBuilder<256> OrchestratorEndpoint;
+ OrchestratorEndpoint << m_Http->GetServiceUri(m_OrchestratorService.get());
+ if (auto View = OrchestratorEndpoint.ToView(); !View.empty() && View.back() != '/')
+ {
+ OrchestratorEndpoint << '/';
+ }
+
+ // If no binaries path is specified, just use the running executable's directory
+ std::filesystem::path BinariesPath = HordeConfig.BinariesPath.empty() ? GetRunningExecutablePath().parent_path()
+ : std::filesystem::path(HordeConfig.BinariesPath);
+ std::filesystem::path WorkingDir = ServerConfig.DataDir / "horde";
+
+ m_HordeProvisioner = std::make_unique<horde::HordeProvisioner>(HordeConfig, BinariesPath, WorkingDir, OrchestratorEndpoint);
+ }
+ }
+# endif
+}
+
+void
+ZenComputeServer::ResolveCloudMetadata()
+{
+ if (m_CloudMetadataFuture.valid())
+ {
+ m_CloudMetadata = m_CloudMetadataFuture.get();
+ }
+}
+
+std::string
+ZenComputeServer::GetInstanceId() const
+{
+ if (!m_InstanceId.empty())
+ {
+ return m_InstanceId;
+ }
+ return fmt::format("{}-{}", GetMachineName(), GetCurrentProcessId());
+}
+
+std::string
+ZenComputeServer::GetAnnounceUrl() const
+{
+ return m_Http->GetServiceUri(nullptr);
+}
+
+void
+ZenComputeServer::RegisterServices(const ZenComputeServerConfig& ServerConfig)
+{
+ ZEN_TRACE_CPU("ZenComputeServer::RegisterServices");
+ ZEN_UNUSED(ServerConfig);
+
+ if (m_ApiService)
+ {
+ m_Http->RegisterService(*m_ApiService);
+ }
+
+ if (m_OrchestratorService)
+ {
+ m_Http->RegisterService(*m_OrchestratorService);
+ }
+
+ if (m_ComputeService)
+ {
+ m_Http->RegisterService(*m_ComputeService);
+ }
+
+ if (m_FrontendService)
+ {
+ m_Http->RegisterService(*m_FrontendService);
+ }
+}
+
+CbObject
+ZenComputeServer::BuildAnnounceBody()
+{
+ CbObjectWriter AnnounceBody;
+ AnnounceBody << "id" << GetInstanceId();
+ AnnounceBody << "uri" << GetAnnounceUrl();
+ AnnounceBody << "hostname" << GetMachineName();
+ AnnounceBody << "platform" << GetRuntimePlatformName();
+
+ ExtendedSystemMetrics Sm = ApplyReportingOverrides(m_MetricsTracker.Query());
+
+ AnnounceBody.BeginObject("metrics");
+ Describe(Sm, AnnounceBody);
+ AnnounceBody.EndObject();
+
+ AnnounceBody << "cpu_usage" << Sm.CpuUsagePercent;
+ AnnounceBody << "memory_total" << Sm.SystemMemoryMiB * 1024 * 1024;
+ AnnounceBody << "memory_used" << (Sm.SystemMemoryMiB - Sm.AvailSystemMemoryMiB) * 1024 * 1024;
+
+ AnnounceBody << "bytes_received" << m_Http->GetTotalBytesReceived();
+ AnnounceBody << "bytes_sent" << m_Http->GetTotalBytesSent();
+
+ auto Actions = m_ComputeService->GetActionCounts();
+ AnnounceBody << "actions_pending" << Actions.Pending;
+ AnnounceBody << "actions_running" << Actions.Running;
+ AnnounceBody << "actions_completed" << Actions.Completed;
+ AnnounceBody << "active_queues" << Actions.ActiveQueues;
+
+ // Derive provisioner from instance ID prefix (e.g. "horde-xxx" or "nomad-xxx")
+ if (m_InstanceId.starts_with("horde-"))
+ {
+ AnnounceBody << "provisioner"
+ << "horde";
+ }
+ else if (m_InstanceId.starts_with("nomad-"))
+ {
+ AnnounceBody << "provisioner"
+ << "nomad";
+ }
+
+ ResolveCloudMetadata();
+ if (m_CloudMetadata)
+ {
+ m_CloudMetadata->Describe(AnnounceBody);
+ }
+
+ return AnnounceBody.Save();
+}
+
+void
+ZenComputeServer::PostAnnounce()
+{
+ ZEN_TRACE_CPU("ZenComputeServer::PostAnnounce");
+
+ if (!m_ComputeService || m_CoordinatorEndpoint.empty())
+ {
+ return;
+ }
+
+ ZEN_INFO("notifying coordinator at '{}' of our availability at '{}'", m_CoordinatorEndpoint, GetAnnounceUrl());
+
+ try
+ {
+ CbObject Body = BuildAnnounceBody();
+
+ // If we have an active WebSocket connection, send via that instead of HTTP POST
+ if (m_OrchestratorWsClient && m_OrchestratorWsClient->IsOpen())
+ {
+ MemoryView View = Body.GetView();
+ m_OrchestratorWsClient->SendBinary(std::span<const uint8_t>(reinterpret_cast<const uint8_t*>(View.GetData()), View.GetSize()));
+ ZEN_INFO("announced to coordinator via WebSocket");
+ return;
+ }
+
+ HttpClient CoordinatorHttp(m_CoordinatorEndpoint);
+ HttpClient::Response Result = CoordinatorHttp.Post("announce", std::move(Body));
+
+ if (Result.Error)
+ {
+ ZEN_ERROR("failed to notify coordinator at '{}': HTTP error {} - {}",
+ m_CoordinatorEndpoint,
+ Result.Error->ErrorCode,
+ Result.Error->ErrorMessage);
+ }
+ else if (!IsHttpOk(Result.StatusCode))
+ {
+ ZEN_ERROR("failed to notify coordinator at '{}': unexpected HTTP status code {}",
+ m_CoordinatorEndpoint,
+ static_cast<int>(Result.StatusCode));
+ }
+ else
+ {
+ ZEN_INFO("successfully notified coordinator at '{}'", m_CoordinatorEndpoint);
+ }
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("failed to notify coordinator at '{}': {}", m_CoordinatorEndpoint, Ex.what());
+ }
+}
+
+void
+ZenComputeServer::EnqueueAnnounceTimer()
+{
+ if (!m_ComputeService || m_CoordinatorEndpoint.empty())
+ {
+ return;
+ }
+
+ m_AnnounceTimer.expires_after(std::chrono::seconds(15));
+ m_AnnounceTimer.async_wait([this](const asio::error_code& Ec) {
+ if (!Ec)
+ {
+ PostAnnounce();
+ EnqueueAnnounceTimer();
+ }
+ });
+ EnsureIoRunner();
+}
+
+void
+ZenComputeServer::InitializeOrchestratorWebSocket()
+{
+ if (!m_EnableWorkerWebSocket || m_CoordinatorEndpoint.empty())
+ {
+ return;
+ }
+
+ // Convert http://host:port → ws://host:port/orch/ws
+ std::string WsUrl = m_CoordinatorEndpoint;
+ if (WsUrl.starts_with("http://"))
+ {
+ WsUrl = "ws://" + WsUrl.substr(7);
+ }
+ else if (WsUrl.starts_with("https://"))
+ {
+ WsUrl = "wss://" + WsUrl.substr(8);
+ }
+ if (!WsUrl.empty() && WsUrl.back() != '/')
+ {
+ WsUrl += '/';
+ }
+ WsUrl += "orch/ws";
+
+ ZEN_INFO("establishing WebSocket link to orchestrator at {}", WsUrl);
+
+ m_OrchestratorWsHandler = std::make_unique<OrchestratorWsHandler>(*this);
+ m_OrchestratorWsClient =
+ std::make_unique<HttpWsClient>(WsUrl, *m_OrchestratorWsHandler, m_IoContext, HttpWsClientSettings{.LogCategory = "orch_ws"});
+
+ m_OrchestratorWsClient->Connect();
+ EnsureIoRunner();
+}
+
+void
+ZenComputeServer::EnqueueWsReconnect()
+{
+ m_WsReconnectTimer.expires_after(std::chrono::seconds(5));
+ m_WsReconnectTimer.async_wait([this](const asio::error_code& Ec) {
+ if (!Ec && m_OrchestratorWsClient)
+ {
+ ZEN_INFO("attempting WebSocket reconnect to orchestrator");
+ m_OrchestratorWsClient->Connect();
+ }
+ });
+ EnsureIoRunner();
+}
+
+void
+ZenComputeServer::OrchestratorWsHandler::OnWsOpen()
+{
+ ZEN_INFO("WebSocket link to orchestrator established");
+
+ // Send initial announce immediately over the WebSocket
+ Server.PostAnnounce();
+}
+
+void
+ZenComputeServer::OrchestratorWsHandler::OnWsMessage([[maybe_unused]] const WebSocketMessage& Msg)
+{
+ // Orchestrator does not push messages to workers; ignore
+}
+
+void
+ZenComputeServer::OrchestratorWsHandler::OnWsClose([[maybe_unused]] uint16_t Code, [[maybe_unused]] std::string_view Reason)
+{
+ ZEN_WARN("WebSocket link to orchestrator closed (code {}), falling back to HTTP announce", Code);
+
+ // Trigger an immediate HTTP announce so the orchestrator has fresh state,
+ // then schedule a reconnect attempt.
+ Server.PostAnnounce();
+ Server.EnqueueWsReconnect();
+}
+
+void
+ZenComputeServer::ProvisionerMaintenanceTick()
+{
+# if ZEN_WITH_HORDE
+ if (m_HordeProvisioner)
+ {
+ m_HordeProvisioner->SetTargetCoreCount(UINT32_MAX);
+ auto Stats = m_HordeProvisioner->GetStats();
+ ZEN_DEBUG("Horde maintenance: target={}, estimated={}, active={}",
+ Stats.TargetCoreCount,
+ Stats.EstimatedCoreCount,
+ Stats.ActiveCoreCount);
+ }
+# endif
+
+# if ZEN_WITH_NOMAD
+ if (m_NomadProvisioner)
+ {
+ m_NomadProvisioner->SetTargetCoreCount(UINT32_MAX);
+ auto Stats = m_NomadProvisioner->GetStats();
+ ZEN_DEBUG("Nomad maintenance: target={}, estimated={}, running jobs={}",
+ Stats.TargetCoreCount,
+ Stats.EstimatedCoreCount,
+ Stats.RunningJobCount);
+ }
+# endif
+}
+
+void
+ZenComputeServer::EnqueueProvisionerMaintenanceTimer()
+{
+ bool HasProvisioner = false;
+# if ZEN_WITH_HORDE
+ HasProvisioner = HasProvisioner || (m_HordeProvisioner != nullptr);
+# endif
+# if ZEN_WITH_NOMAD
+ HasProvisioner = HasProvisioner || (m_NomadProvisioner != nullptr);
+# endif
+
+ if (!HasProvisioner)
+ {
+ return;
+ }
+
+ m_ProvisionerMaintenanceTimer.expires_after(std::chrono::seconds(15));
+ m_ProvisionerMaintenanceTimer.async_wait([this](const asio::error_code& Ec) {
+ if (!Ec)
+ {
+ ProvisionerMaintenanceTick();
+ EnqueueProvisionerMaintenanceTimer();
+ }
+ });
+ EnsureIoRunner();
+}
+
+void
+ZenComputeServer::Run()
+{
+ ZEN_TRACE_CPU("ZenComputeServer::Run");
+
+ if (m_ProcessMonitor.IsActive())
+ {
+ CheckOwnerPid();
+ }
+
+ if (!m_TestMode)
+ {
+ // clang-format off
+ ZEN_INFO( R"(__________ _________ __ )" "\n"
+ R"(\____ /____ ____ \_ ___ \ ____ _____ ______ __ ___/ |_ ____ )" "\n"
+ R"( / // __ \ / \/ \ \/ / _ \ / \\____ \| | \ __\/ __ \ )" "\n"
+ R"( / /\ ___/| | \ \___( <_> ) Y Y \ |_> > | /| | \ ___/ )" "\n"
+ R"(/_______ \___ >___| /\______ /\____/|__|_| / __/|____/ |__| \___ >)" "\n"
+ R"( \/ \/ \/ \/ \/|__| \/ )");
+ // clang-format on
+
+ ExtendableStringBuilder<256> BuildOptions;
+ GetBuildOptions(BuildOptions, '\n');
+ ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions);
+ }
+
+ ZEN_INFO(ZEN_APP_NAME " now running as COMPUTE (pid: {})", GetCurrentProcessId());
+
+# if ZEN_PLATFORM_WINDOWS
+ if (zen::windows::IsRunningOnWine())
+ {
+ ZEN_INFO("detected Wine session - " ZEN_APP_NAME " is not formally tested on Wine and may therefore not work or perform well");
+ }
+# endif
+
+# if ZEN_USE_SENTRY
+ ZEN_INFO("sentry crash handler {}", m_UseSentry ? "ENABLED" : "DISABLED");
+ if (m_UseSentry)
+ {
+ SentryIntegration::ClearCaches();
+ }
+# endif
+
+ if (m_DebugOptionForcedCrash)
+ {
+ ZEN_DEBUG_BREAK();
+ }
+
+ const bool IsInteractiveMode = IsInteractiveSession(); // &&!m_TestMode;
+
+ SetNewState(kRunning);
+
+ OnReady();
+
+ PostAnnounce();
+ EnqueueAnnounceTimer();
+ InitializeOrchestratorWebSocket();
+
+# if ZEN_WITH_HORDE
+ // Start Horde provisioning if configured — request maximum allowed cores.
+ // SetTargetCoreCount clamps to HordeConfig::MaxCores internally.
+ if (m_HordeProvisioner)
+ {
+ ZEN_INFO("Horde provisioning starting");
+ m_HordeProvisioner->SetTargetCoreCount(UINT32_MAX);
+ auto Stats = m_HordeProvisioner->GetStats();
+ ZEN_INFO("Horde provisioning started (target cores: {})", Stats.TargetCoreCount);
+ }
+# endif
+
+# if ZEN_WITH_NOMAD
+ // Start Nomad provisioning if configured — request maximum allowed cores.
+ // SetTargetCoreCount clamps to NomadConfig::MaxCores internally.
+ if (m_NomadProvisioner)
+ {
+ m_NomadProvisioner->SetTargetCoreCount(UINT32_MAX);
+ auto Stats = m_NomadProvisioner->GetStats();
+ ZEN_INFO("Nomad provisioning started (target cores: {})", Stats.TargetCoreCount);
+ }
+# endif
+
+ EnqueueProvisionerMaintenanceTimer();
+
+ m_Http->Run(IsInteractiveMode);
+
+ SetNewState(kShuttingDown);
+
+ ZEN_INFO(ZEN_APP_NAME " exiting");
+}
+
+//////////////////////////////////////////////////////////////////////////////////
+
+ZenComputeServerMain::ZenComputeServerMain(ZenComputeServerConfig& ServerOptions)
+: ZenServerMain(ServerOptions)
+, m_ServerOptions(ServerOptions)
+{
+}
+
+void
+ZenComputeServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
+{
+ ZEN_TRACE_CPU("ZenComputeServerMain::DoRun");
+
+ ZenComputeServer Server;
+ Server.SetDataRoot(m_ServerOptions.DataDir);
+ Server.SetContentRoot(m_ServerOptions.ContentDir);
+ Server.SetTestMode(m_ServerOptions.IsTest);
+ Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+
+ const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
+ if (EffectiveBasePort == -1)
+ {
+ // Server.Initialize has already logged what the issue is - just exit with failure code here.
+ std::exit(1);
+ }
+
+ Entry->EffectiveListenPort = uint16_t(EffectiveBasePort);
+ if (EffectiveBasePort != m_ServerOptions.BasePort)
+ {
+ ZEN_INFO(ZEN_APP_NAME " - relocated to base port {}", EffectiveBasePort);
+ m_ServerOptions.BasePort = EffectiveBasePort;
+ }
+
+ std::unique_ptr<std::thread> ShutdownThread;
+ std::unique_ptr<NamedEvent> ShutdownEvent;
+
+ ExtendableStringBuilder<64> ShutdownEventName;
+ ShutdownEventName << "Zen_" << m_ServerOptions.BasePort << "_Shutdown";
+ ShutdownEvent.reset(new NamedEvent{ShutdownEventName});
+
+ // Monitor shutdown signals
+
+ ShutdownThread.reset(new std::thread{[&] {
+ SetCurrentThreadName("shutdown_mon");
+
+ ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}", ShutdownEventName, zen::GetCurrentProcessId());
+
+ if (ShutdownEvent->Wait())
+ {
+ ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId());
+ Server.RequestExit(0);
+ }
+ else
+ {
+ ZEN_INFO("shutdown signal wait() failed");
+ }
+ }});
+
+ auto CleanupShutdown = MakeGuard([&ShutdownEvent, &ShutdownThread] {
+ ReportServiceStatus(ServiceStatus::Stopping);
+
+ if (ShutdownEvent)
+ {
+ ShutdownEvent->Set();
+ }
+ if (ShutdownThread && ShutdownThread->joinable())
+ {
+ ShutdownThread->join();
+ }
+ });
+
+ // If we have a parent process, establish the mechanisms we need
+ // to be able to communicate readiness with the parent
+
+ Server.SetIsReadyFunc([&] {
+ std::error_code Ec;
+ m_LockFile.Update(MakeLockData(true), Ec);
+ ReportServiceStatus(ServiceStatus::Running);
+ NotifyReady();
+ });
+
+ Server.Run();
+}
+
+} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/src/zenserver/compute/computeserver.h b/src/zenserver/compute/computeserver.h
new file mode 100644
index 000000000..8f4edc0f0
--- /dev/null
+++ b/src/zenserver/compute/computeserver.h
@@ -0,0 +1,188 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zenserver.h"
+
+#if ZEN_WITH_COMPUTE_SERVICES
+
+# include <future>
+# include <zencore/system.h>
+# include <zenhttp/httpwsclient.h>
+# include <zenstore/gc.h>
+# include "frontend/frontend.h"
+
+namespace cxxopts {
+class Options;
+}
+namespace zen::LuaConfig {
+struct Options;
+}
+
+namespace zen::compute {
+class CloudMetadata;
+class HttpComputeService;
+class HttpOrchestratorService;
+} // namespace zen::compute
+
+# if ZEN_WITH_HORDE
+# include <zenhorde/hordeconfig.h>
+namespace zen::horde {
+class HordeProvisioner;
+} // namespace zen::horde
+# endif
+
+# if ZEN_WITH_NOMAD
+# include <zennomad/nomadconfig.h>
+namespace zen::nomad {
+class NomadProvisioner;
+} // namespace zen::nomad
+# endif
+
+namespace zen {
+
+class CidStore;
+class HttpApiService;
+
+struct ZenComputeServerConfig : public ZenServerConfig
+{
+ std::string UpstreamNotificationEndpoint;
+ std::string InstanceId; // For use in notifications
+ std::string CoordinatorEndpoint;
+ std::string IdmsEndpoint;
+ int32_t MaxConcurrentActions = 0; // 0 = auto (LogicalProcessorCount * 2)
+ bool EnableWorkerWebSocket = false; // Use WebSocket for worker↔orchestrator link
+
+# if ZEN_WITH_HORDE
+ horde::HordeConfig HordeConfig;
+# endif
+
+# if ZEN_WITH_NOMAD
+ nomad::NomadConfig NomadConfig;
+# endif
+};
+
+struct ZenComputeServerConfigurator : public ZenServerConfiguratorBase
+{
+ ZenComputeServerConfigurator(ZenComputeServerConfig& ServerOptions)
+ : ZenServerConfiguratorBase(ServerOptions)
+ , m_ServerOptions(ServerOptions)
+ {
+ }
+
+ ~ZenComputeServerConfigurator() = default;
+
+private:
+ virtual void AddCliOptions(cxxopts::Options& Options) override;
+ virtual void AddConfigOptions(LuaConfig::Options& Options) override;
+ virtual void ApplyOptions(cxxopts::Options& Options) override;
+ virtual void OnConfigFileParsed(LuaConfig::Options& LuaOptions) override;
+ virtual void ValidateOptions() override;
+
+ ZenComputeServerConfig& m_ServerOptions;
+
+# if ZEN_WITH_HORDE
+ std::string m_HordeModeStr = "direct";
+ std::string m_HordeEncryptionStr = "none";
+# endif
+
+# if ZEN_WITH_NOMAD
+ std::string m_NomadDriverStr = "raw_exec";
+ std::string m_NomadDistributionStr = "predeployed";
+# endif
+};
+
+class ZenComputeServerMain : public ZenServerMain
+{
+public:
+ ZenComputeServerMain(ZenComputeServerConfig& ServerOptions);
+ virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override;
+
+ ZenComputeServerMain(const ZenComputeServerMain&) = delete;
+ ZenComputeServerMain& operator=(const ZenComputeServerMain&) = delete;
+
+ typedef ZenComputeServerConfig Config;
+ typedef ZenComputeServerConfigurator Configurator;
+
+private:
+ ZenComputeServerConfig& m_ServerOptions;
+};
+
+/**
+ * The compute server handles DDC build function execution requests
+ * only. It's intended to be used on a pure compute resource and does
+ * not handle any storage tasks. The actual scheduling happens upstream
+ * in a storage server instance.
+ */
+
+class ZenComputeServer : public ZenServerBase
+{
+ ZenComputeServer& operator=(ZenComputeServer&&) = delete;
+ ZenComputeServer(ZenComputeServer&&) = delete;
+
+public:
+ ZenComputeServer();
+ ~ZenComputeServer();
+
+ int Initialize(const ZenComputeServerConfig& ServerConfig, ZenServerState::ZenServerEntry* ServerEntry);
+ void Run();
+ void Cleanup();
+
+private:
+ GcManager m_GcManager;
+ GcScheduler m_GcScheduler{m_GcManager};
+ std::unique_ptr<CidStore> m_CidStore;
+ std::unique_ptr<HttpApiService> m_ApiService;
+ std::unique_ptr<zen::compute::HttpComputeService> m_ComputeService;
+ std::unique_ptr<zen::compute::HttpOrchestratorService> m_OrchestratorService;
+ std::unique_ptr<zen::compute::CloudMetadata> m_CloudMetadata;
+ std::future<std::unique_ptr<zen::compute::CloudMetadata>> m_CloudMetadataFuture;
+ std::unique_ptr<HttpFrontendService> m_FrontendService;
+# if ZEN_WITH_HORDE
+ std::unique_ptr<zen::horde::HordeProvisioner> m_HordeProvisioner;
+# endif
+# if ZEN_WITH_NOMAD
+ std::unique_ptr<zen::nomad::NomadProvisioner> m_NomadProvisioner;
+# endif
+ SystemMetricsTracker m_MetricsTracker;
+ std::string m_CoordinatorEndpoint;
+ std::string m_InstanceId;
+
+ asio::steady_timer m_AnnounceTimer{m_IoContext};
+ asio::steady_timer m_ProvisionerMaintenanceTimer{m_IoContext};
+
+ void InitializeState(const ZenComputeServerConfig& ServerConfig);
+ void InitializeServices(const ZenComputeServerConfig& ServerConfig);
+ void RegisterServices(const ZenComputeServerConfig& ServerConfig);
+ void ResolveCloudMetadata();
+ void PostAnnounce();
+ void EnqueueAnnounceTimer();
+ void EnqueueProvisionerMaintenanceTimer();
+ void ProvisionerMaintenanceTick();
+ std::string GetAnnounceUrl() const;
+ std::string GetInstanceId() const;
+ CbObject BuildAnnounceBody();
+
+ // Worker→orchestrator WebSocket client
+ struct OrchestratorWsHandler : public IWsClientHandler
+ {
+ ZenComputeServer& Server;
+ explicit OrchestratorWsHandler(ZenComputeServer& S) : Server(S) {}
+
+ void OnWsOpen() override;
+ void OnWsMessage(const WebSocketMessage& Msg) override;
+ void OnWsClose(uint16_t Code, std::string_view Reason) override;
+ };
+
+ std::unique_ptr<OrchestratorWsHandler> m_OrchestratorWsHandler;
+ std::unique_ptr<HttpWsClient> m_OrchestratorWsClient;
+ asio::steady_timer m_WsReconnectTimer{m_IoContext};
+ bool m_EnableWorkerWebSocket = false;
+
+ void InitializeOrchestratorWebSocket();
+ void EnqueueWsReconnect();
+};
+
+} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp
index 07913e891..e36352dae 100644
--- a/src/zenserver/config/config.cpp
+++ b/src/zenserver/config/config.cpp
@@ -16,8 +16,8 @@
#include <zencore/iobuffer.h>
#include <zencore/logging.h>
#include <zencore/string.h>
-#include <zenutil/commandlineoptions.h>
-#include <zenutil/environmentoptions.h>
+#include <zenutil/config/commandlineoptions.h>
+#include <zenutil/config/environmentoptions.h>
ZEN_THIRD_PARTY_INCLUDES_START
#include <fmt/format.h>
@@ -119,10 +119,17 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions
ZenServerConfig& ServerOptions = m_ServerOptions;
+ // logging
+
+ LuaOptions.AddOption("server.logid"sv, ServerOptions.LoggingConfig.LogId, "log-id"sv);
+ LuaOptions.AddOption("server.abslog"sv, ServerOptions.LoggingConfig.AbsLogFile, "abslog"sv);
+ LuaOptions.AddOption("server.otlpendpoint"sv, ServerOptions.LoggingConfig.OtelEndpointUri, "otlp-endpoint"sv);
+ LuaOptions.AddOption("server.quiet"sv, ServerOptions.LoggingConfig.QuietConsole, "quiet"sv);
+ LuaOptions.AddOption("server.noconsole"sv, ServerOptions.LoggingConfig.NoConsoleOutput, "noconsole"sv);
+
// server
LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv);
- LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv);
LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv);
LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
@@ -131,12 +138,9 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions
LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv);
LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
- LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv);
- LuaOptions.AddOption("server.otlpendpoint"sv, ServerOptions.OtelEndpointUri, "otlp-endpoint"sv);
LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv);
LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv);
- LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv);
- LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv);
+ LuaOptions.AddOption("server.security.configpath"sv, ServerOptions.SecurityConfigPath, "security-config-path"sv);
////// network
@@ -182,8 +186,10 @@ struct ZenServerCmdLineOptions
std::string SystemRootDir;
std::string ContentDir;
std::string DataDir;
- std::string AbsLogFile;
std::string BaseSnapshotDir;
+ std::string SecurityConfigPath;
+
+ ZenLoggingCmdLineOptions LoggingOptions;
void AddCliOptions(cxxopts::Options& options, ZenServerConfig& ServerOptions);
void ApplyOptions(cxxopts::Options& options, ZenServerConfig& ServerOptions);
@@ -249,22 +255,7 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi
cxxopts::value<bool>(ServerOptions.ShouldCrash)->default_value("false"),
"");
- // clang-format off
- options.add_options("logging")
- ("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile))
- ("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId))
- ("quiet", "Configure console logger output to level WARN", cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false"))
- ("noconsole", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
- ("log-trace", "Change selected loggers to level TRACE", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
- ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
- ("log-info", "Change selected loggers to level INFO", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
- ("log-warn", "Change selected loggers to level WARN", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
- ("log-error", "Change selected loggers to level ERROR", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
- ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
- ("log-off", "Change selected loggers to level OFF", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
- ("otlp-endpoint", "OpenTelemetry endpoint URI (e.g http://localhost:4318)", cxxopts::value<std::string>(ServerOptions.OtelEndpointUri))
- ;
- // clang-format on
+ LoggingOptions.AddCliOptions(options, ServerOptions.LoggingConfig);
options
.add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value<int>(ServerOptions.OwnerPid), "<identifier>");
@@ -311,6 +302,13 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi
cxxopts::value<bool>(ServerOptions.HttpConfig.ForceLoopback)->default_value("false"),
"<http forceloopback>");
+ options.add_option("network",
+ "",
+ "security-config-path",
+ "Path to http security configuration file",
+ cxxopts::value<std::string>(SecurityConfigPath),
+ "<security config path>");
+
#if ZEN_WITH_HTTPSYS
options.add_option("httpsys",
"",
@@ -391,12 +389,14 @@ ZenServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenServerConfig
throw std::runtime_error(fmt::format("'--snapshot-dir' ('{}') must be a directory", ServerOptions.BaseSnapshotDir));
}
- ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir);
- ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir);
- ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir);
- ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile);
- ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile);
- ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir);
+ ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir);
+ ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir);
+ ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir);
+ ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile);
+ ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir);
+ ServerOptions.SecurityConfigPath = MakeSafeAbsolutePath(SecurityConfigPath);
+
+ LoggingOptions.ApplyOptions(ServerOptions.LoggingConfig);
}
//////////////////////////////////////////////////////////////////////////
@@ -466,34 +466,7 @@ ZenServerConfiguratorBase::Configure(int argc, char* argv[])
}
#endif
- if (m_ServerOptions.QuietConsole)
- {
- bool HasExplicitConsoleLevel = false;
- for (int i = 0; i < logging::level::LogLevelCount; ++i)
- {
- if (m_ServerOptions.Loggers[i].find("console") != std::string::npos)
- {
- HasExplicitConsoleLevel = true;
- break;
- }
- }
-
- if (!HasExplicitConsoleLevel)
- {
- std::string& WarnLoggers = m_ServerOptions.Loggers[logging::level::Warn];
- if (!WarnLoggers.empty())
- {
- WarnLoggers += ",";
- }
- WarnLoggers += "console";
- }
- }
-
- for (int i = 0; i < logging::level::LogLevelCount; ++i)
- {
- logging::ConfigureLogLevels(logging::level::LogLevel(i), m_ServerOptions.Loggers[i]);
- }
- logging::RefreshLogLevels();
+ ApplyLoggingOptions(options, m_ServerOptions.LoggingConfig);
BaseOptions.ApplyOptions(options, m_ServerOptions);
ApplyOptions(options);
@@ -532,9 +505,9 @@ ZenServerConfiguratorBase::Configure(int argc, char* argv[])
m_ServerOptions.DataDir = PickDefaultStateDirectory(m_ServerOptions.SystemRootDir);
}
- if (m_ServerOptions.AbsLogFile.empty())
+ if (m_ServerOptions.LoggingConfig.AbsLogFile.empty())
{
- m_ServerOptions.AbsLogFile = m_ServerOptions.DataDir / "logs" / "zenserver.log";
+ m_ServerOptions.LoggingConfig.AbsLogFile = m_ServerOptions.DataDir / "logs" / "zenserver.log";
}
m_ServerOptions.HttpConfig.IsDedicatedServer = m_ServerOptions.IsDedicated;
diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h
index 7c3192a1f..55aee07f9 100644
--- a/src/zenserver/config/config.h
+++ b/src/zenserver/config/config.h
@@ -6,6 +6,7 @@
#include <zencore/trace.h>
#include <zencore/zencore.h>
#include <zenhttp/httpserver.h>
+#include <zenutil/config/loggingconfig.h>
#include <filesystem>
#include <string>
#include <vector>
@@ -42,29 +43,26 @@ struct ZenServerConfig
HttpServerConfig HttpConfig;
ZenSentryConfig SentryConfig;
ZenStatsConfig StatsConfig;
- int BasePort = 8558; // Service listen port (used for both UDP and TCP)
- int OwnerPid = 0; // Parent process id (zero for standalone)
- bool IsDebug = false;
- bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not
- bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization
- bool IsTest = false;
- bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux)
- bool NoConsoleOutput = false; // Control default use of stdout for diagnostics
- bool QuietConsole = false; // Configure console logger output to level WARN
- int CoreLimit = 0; // If set, hardware concurrency queries are capped at this number
- bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements
- bool ShouldCrash = false; // Option for testing crash handling
- bool IsFirstRun = false;
- std::filesystem::path ConfigFile; // Path to Lua config file
- std::filesystem::path SystemRootDir; // System root directory (used for machine level config)
- std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental)
- std::filesystem::path DataDir; // Root directory for state (used for testing)
- std::filesystem::path AbsLogFile; // Absolute path to main log file
- std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start)
- std::string ChildId; // Id assigned by parent process (used for lifetime management)
- std::string LogId; // Id for tagging log output
- std::string Loggers[zen::logging::level::LogLevelCount];
- std::string OtelEndpointUri; // OpenTelemetry endpoint URI
+ ZenLoggingConfig LoggingConfig;
+ int BasePort = 8558; // Service listen port (used for both UDP and TCP)
+ int OwnerPid = 0; // Parent process id (zero for standalone)
+ bool IsDebug = false;
+ bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not
+ bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization
+ bool IsTest = false;
+ bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux)
+ int CoreLimit = 0; // If set, hardware concurrency queries are capped at this number
+ int LieCpu = 0;
+ bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements
+ bool ShouldCrash = false; // Option for testing crash handling
+ bool IsFirstRun = false;
+ std::filesystem::path ConfigFile; // Path to Lua config file
+ std::filesystem::path SystemRootDir; // System root directory (used for machine level config)
+ std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental)
+ std::filesystem::path DataDir; // Root directory for state (used for testing)
+ std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start)
+ std::string ChildId; // Id assigned by parent process (used for lifetime management)
+ std::filesystem::path SecurityConfigPath; // Path to a Json security configuration file
#if ZEN_WITH_TRACE
bool HasTraceCommandlineOptions = false;
diff --git a/src/zenserver/config/luaconfig.h b/src/zenserver/config/luaconfig.h
index ce7013a9a..e3ac3b343 100644
--- a/src/zenserver/config/luaconfig.h
+++ b/src/zenserver/config/luaconfig.h
@@ -4,7 +4,7 @@
#include <zenbase/concepts.h>
#include <zencore/fmtutils.h>
-#include <zenutil/commandlineoptions.h>
+#include <zenutil/config/commandlineoptions.h>
ZEN_THIRD_PARTY_INCLUDES_START
#include <fmt/format.h>
diff --git a/src/zenserver/diag/diagsvcs.cpp b/src/zenserver/diag/diagsvcs.cpp
index d8d53b0e3..dd4b8956c 100644
--- a/src/zenserver/diag/diagsvcs.cpp
+++ b/src/zenserver/diag/diagsvcs.cpp
@@ -9,12 +9,11 @@
#include <zencore/logging.h>
#include <zencore/memory/llm.h>
#include <zencore/string.h>
+#include <zencore/system.h>
#include <fstream>
#include <sstream>
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <spdlog/logger.h>
-ZEN_THIRD_PARTY_INCLUDES_END
+#include <zencore/logging/logger.h>
namespace zen {
@@ -53,6 +52,36 @@ HttpHealthService::HttpHealthService()
Writer << "AbsLogPath"sv << m_HealthInfo.AbsLogPath.string();
Writer << "BuildVersion"sv << m_HealthInfo.BuildVersion;
Writer << "HttpServerClass"sv << m_HealthInfo.HttpServerClass;
+ Writer << "Port"sv << m_HealthInfo.Port;
+ Writer << "Pid"sv << m_HealthInfo.Pid;
+ Writer << "IsDedicated"sv << m_HealthInfo.IsDedicated;
+ Writer << "StartTimeMs"sv << m_HealthInfo.StartTimeMs;
+ }
+
+ Writer.BeginObject("RuntimeConfig"sv);
+ for (const auto& Opt : m_HealthInfo.RuntimeConfig)
+ {
+ Writer << Opt.first << Opt.second;
+ }
+ Writer.EndObject();
+
+ Writer.BeginObject("BuildConfig"sv);
+ for (const auto& Opt : m_HealthInfo.BuildOptions)
+ {
+ Writer << Opt.first << Opt.second;
+ }
+ Writer.EndObject();
+
+ Writer << "Hostname"sv << GetMachineName();
+ Writer << "Platform"sv << GetRuntimePlatformName();
+ Writer << "Arch"sv << GetCpuName();
+ Writer << "OS"sv << GetOperatingSystemVersion();
+
+ {
+ auto Metrics = GetSystemMetrics();
+ Writer.BeginObject("System"sv);
+ Describe(Metrics, Writer);
+ Writer.EndObject();
}
HttpReq.WriteResponse(HttpResponseCode::OK, Writer.Save());
@@ -64,7 +93,7 @@ HttpHealthService::HttpHealthService()
[this](HttpRouterRequest& RoutedReq) {
HttpServerRequest& HttpReq = RoutedReq.ServerRequest();
- zen::Log().SpdLogger->flush();
+ zen::Log().Flush();
std::filesystem::path Path = [&] {
RwLock::SharedLockScope _(m_InfoLock);
diff --git a/src/zenserver/diag/diagsvcs.h b/src/zenserver/diag/diagsvcs.h
index 8cc869c83..87ce80b3c 100644
--- a/src/zenserver/diag/diagsvcs.h
+++ b/src/zenserver/diag/diagsvcs.h
@@ -6,6 +6,7 @@
#include <zenhttp/httpserver.h>
#include <filesystem>
+#include <vector>
//////////////////////////////////////////////////////////////////////////
@@ -89,10 +90,16 @@ private:
struct HealthServiceInfo
{
- std::filesystem::path DataRoot;
- std::filesystem::path AbsLogPath;
- std::string HttpServerClass;
- std::string BuildVersion;
+ std::filesystem::path DataRoot;
+ std::filesystem::path AbsLogPath;
+ std::string HttpServerClass;
+ std::string BuildVersion;
+ int Port = 0;
+ int Pid = 0;
+ bool IsDedicated = false;
+ int64_t StartTimeMs = 0;
+ std::vector<std::pair<std::string_view, bool>> BuildOptions;
+ std::vector<std::pair<std::string_view, std::string>> RuntimeConfig;
};
/** Health monitoring endpoint
diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp
index 4962b9006..178c3d3b5 100644
--- a/src/zenserver/diag/logging.cpp
+++ b/src/zenserver/diag/logging.cpp
@@ -6,6 +6,8 @@
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
+#include <zencore/logging/logger.h>
+#include <zencore/logging/registry.h>
#include <zencore/memory/llm.h>
#include <zencore/session.h>
#include <zencore/string.h>
@@ -14,10 +16,6 @@
#include "otlphttp.h"
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <spdlog/spdlog.h>
-ZEN_THIRD_PARTY_INCLUDES_END
-
namespace zen {
void
@@ -28,10 +26,10 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService)
const LoggingOptions LogOptions = {.IsDebug = InOptions.IsDebug,
.IsVerbose = false,
.IsTest = InOptions.IsTest,
- .NoConsoleOutput = InOptions.NoConsoleOutput,
- .QuietConsole = InOptions.QuietConsole,
- .AbsLogFile = InOptions.AbsLogFile,
- .LogId = InOptions.LogId};
+ .NoConsoleOutput = InOptions.LoggingConfig.NoConsoleOutput,
+ .QuietConsole = InOptions.LoggingConfig.QuietConsole,
+ .AbsLogFile = InOptions.LoggingConfig.AbsLogFile,
+ .LogId = InOptions.LoggingConfig.LogId};
BeginInitializeLogging(LogOptions);
@@ -43,13 +41,12 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService)
std::filesystem::path HttpLogPath = InOptions.DataDir / "logs" / "http.log";
zen::CreateDirectories(HttpLogPath.parent_path());
- auto HttpSink = std::make_shared<zen::logging::RotatingFileSink>(HttpLogPath,
- /* max size */ 128 * 1024 * 1024,
- /* max files */ 16,
- /* rotate on open */ true);
- auto HttpLogger = std::make_shared<spdlog::logger>("http_requests", HttpSink);
- spdlog::apply_logger_env_levels(HttpLogger);
- spdlog::register_logger(HttpLogger);
+ logging::SinkPtr HttpSink(new zen::logging::RotatingFileSink(HttpLogPath,
+ /* max size */ 128 * 1024 * 1024,
+ /* max files */ 16,
+ /* rotate on open */ true));
+ Ref<logging::Logger> HttpLogger(new logging::Logger("http_requests", std::vector<logging::SinkPtr>{HttpSink}));
+ logging::Registry::Instance().Register(HttpLogger);
if (WithCacheService)
{
@@ -57,33 +54,30 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService)
std::filesystem::path CacheLogPath = InOptions.DataDir / "logs" / "z$.log";
zen::CreateDirectories(CacheLogPath.parent_path());
- auto CacheSink = std::make_shared<zen::logging::RotatingFileSink>(CacheLogPath,
- /* max size */ 128 * 1024 * 1024,
- /* max files */ 16,
- /* rotate on open */ false);
- auto CacheLogger = std::make_shared<spdlog::logger>("z$", CacheSink);
- spdlog::apply_logger_env_levels(CacheLogger);
- spdlog::register_logger(CacheLogger);
+ logging::SinkPtr CacheSink(new zen::logging::RotatingFileSink(CacheLogPath,
+ /* max size */ 128 * 1024 * 1024,
+ /* max files */ 16,
+ /* rotate on open */ false));
+ Ref<logging::Logger> CacheLogger(new logging::Logger("z$", std::vector<logging::SinkPtr>{CacheSink}));
+ logging::Registry::Instance().Register(CacheLogger);
// Jupiter - only log upstream HTTP traffic to file
- auto JupiterLogger = std::make_shared<spdlog::logger>("jupiter", FileSink);
- spdlog::apply_logger_env_levels(JupiterLogger);
- spdlog::register_logger(JupiterLogger);
+ Ref<logging::Logger> JupiterLogger(new logging::Logger("jupiter", std::vector<logging::SinkPtr>{FileSink}));
+ logging::Registry::Instance().Register(JupiterLogger);
// Zen - only log upstream HTTP traffic to file
- auto ZenClientLogger = std::make_shared<spdlog::logger>("zenclient", FileSink);
- spdlog::apply_logger_env_levels(ZenClientLogger);
- spdlog::register_logger(ZenClientLogger);
+ Ref<logging::Logger> ZenClientLogger(new logging::Logger("zenclient", std::vector<logging::SinkPtr>{FileSink}));
+ logging::Registry::Instance().Register(ZenClientLogger);
}
#if ZEN_WITH_OTEL
- if (!InOptions.OtelEndpointUri.empty())
+ if (!InOptions.LoggingConfig.OtelEndpointUri.empty())
{
// TODO: Should sanity check that endpoint is reachable? Also, a valid URI?
- auto OtelSink = std::make_shared<zen::logging::OtelHttpProtobufSink>(InOptions.OtelEndpointUri);
- zen::logging::Default().SpdLogger->sinks().push_back(std::move(OtelSink));
+ logging::SinkPtr OtelSink(new zen::logging::OtelHttpProtobufSink(InOptions.LoggingConfig.OtelEndpointUri));
+ zen::logging::Default()->AddSink(std::move(OtelSink));
}
#endif
@@ -91,9 +85,10 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService)
const zen::Oid ServerSessionId = zen::GetSessionId();
- spdlog::apply_all([&](auto Logger) {
+ static constinit logging::LogPoint SessionIdPoint{{}, logging::Info, "server session id: {}"};
+ logging::Registry::Instance().ApplyAll([&](auto Logger) {
ZEN_MEMSCOPE(ELLMTag::Logging);
- Logger->info("server session id: {}", ServerSessionId);
+ Logger->Log(SessionIdPoint, fmt::make_format_args(ServerSessionId));
});
}
diff --git a/src/zenserver/diag/otlphttp.cpp b/src/zenserver/diag/otlphttp.cpp
index d62ccccb6..d6e24cbe3 100644
--- a/src/zenserver/diag/otlphttp.cpp
+++ b/src/zenserver/diag/otlphttp.cpp
@@ -10,11 +10,18 @@
#include <protozero/buffer_string.hpp>
#include <protozero/pbf_builder.hpp>
+#include <cstdio>
+
#if ZEN_WITH_OTEL
namespace zen::logging {
//////////////////////////////////////////////////////////////////////////
+//
+// Important note: in general we cannot use ZEN_WARN/ZEN_ERROR etc in this
+// file as it could cause recursive logging calls when we attempt to log
+// errors from the OTLP HTTP client itself.
+//
OtelHttpProtobufSink::OtelHttpProtobufSink(const std::string_view& Uri) : m_OtelHttp(Uri)
{
@@ -36,14 +43,44 @@ OtelHttpProtobufSink::~OtelHttpProtobufSink()
}
void
+OtelHttpProtobufSink::CheckPostResult(const HttpClient::Response& Result, const char* Endpoint) noexcept
+{
+ if (!Result.IsSuccess())
+ {
+ uint32_t PrevFailures = m_ConsecutivePostFailures.fetch_add(1);
+ if (PrevFailures < kMaxReportedFailures)
+ {
+ fprintf(stderr, "OtelHttpProtobufSink: %s\n", Result.ErrorMessage(Endpoint).c_str());
+ if (PrevFailures + 1 == kMaxReportedFailures)
+ {
+ fprintf(stderr, "OtelHttpProtobufSink: suppressing further export errors\n");
+ }
+ }
+ }
+ else
+ {
+ m_ConsecutivePostFailures.store(0);
+ }
+}
+
+void
OtelHttpProtobufSink::RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans)
{
- std::string Data = m_Encoder.FormatOtelTrace(Trace, Spans);
+ try
+ {
+ std::string Data = m_Encoder.FormatOtelTrace(Trace, Spans);
+
+ IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
+ Payload.SetContentType(ZenContentType::kProtobuf);
- IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
- Payload.SetContentType(ZenContentType::kProtobuf);
+ HttpClient::Response Result = m_OtelHttp.Post("/v1/traces", Payload);
- auto Result = m_OtelHttp.Post("/v1/traces", Payload);
+ CheckPostResult(Result, "POST /v1/traces");
+ }
+ catch (const std::exception& Ex)
+ {
+ fprintf(stderr, "OtelHttpProtobufSink: exception exporting traces: %s\n", Ex.what());
+ }
}
void
@@ -53,28 +90,26 @@ OtelHttpProtobufSink::TraceRecorder::RecordSpans(zen::otel::TraceId Trace, std::
}
void
-OtelHttpProtobufSink::log(const spdlog::details::log_msg& Msg)
+OtelHttpProtobufSink::Log(const LogMessage& Msg)
{
+ try
{
std::string Data = m_Encoder.FormatOtelProtobuf(Msg);
IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
Payload.SetContentType(ZenContentType::kProtobuf);
- auto Result = m_OtelHttp.Post("/v1/logs", Payload);
- }
+ HttpClient::Response Result = m_OtelHttp.Post("/v1/logs", Payload);
+ CheckPostResult(Result, "POST /v1/logs");
+ }
+ catch (const std::exception& Ex)
{
- std::string Data = m_Encoder.FormatOtelMetrics();
-
- IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()};
- Payload.SetContentType(ZenContentType::kProtobuf);
-
- auto Result = m_OtelHttp.Post("/v1/metrics", Payload);
+ fprintf(stderr, "OtelHttpProtobufSink: exception exporting logs: %s\n", Ex.what());
}
}
void
-OtelHttpProtobufSink::flush()
+OtelHttpProtobufSink::Flush()
{
}
diff --git a/src/zenserver/diag/otlphttp.h b/src/zenserver/diag/otlphttp.h
index 2281bdcc0..64b3dbc87 100644
--- a/src/zenserver/diag/otlphttp.h
+++ b/src/zenserver/diag/otlphttp.h
@@ -3,23 +3,25 @@
#pragma once
-#include <spdlog/sinks/sink.h>
+#include <zencore/logging/sink.h>
#include <zencore/zencore.h>
#include <zenhttp/httpclient.h>
#include <zentelemetry/otlpencoder.h>
#include <zentelemetry/otlptrace.h>
+#include <atomic>
+
#if ZEN_WITH_OTEL
namespace zen::logging {
/**
- * OTLP/HTTP sink for spdlog
+ * OTLP/HTTP sink for logging
*
* Sends log messages and traces to an OpenTelemetry collector via OTLP over HTTP
*/
-class OtelHttpProtobufSink : public spdlog::sinks::sink
+class OtelHttpProtobufSink : public Sink
{
public:
// Note that this URI should be the base URI of the OTLP HTTP endpoint, e.g.
@@ -31,12 +33,12 @@ public:
OtelHttpProtobufSink& operator=(const OtelHttpProtobufSink&) = delete;
private:
- virtual void log(const spdlog::details::log_msg& Msg) override;
- virtual void flush() override;
- virtual void set_pattern(const std::string& pattern) override { ZEN_UNUSED(pattern); }
- virtual void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) override { ZEN_UNUSED(sink_formatter); }
+ virtual void Log(const LogMessage& Msg) override;
+ virtual void Flush() override;
+ virtual void SetFormatter(std::unique_ptr<Formatter>) override {}
void RecordSpans(zen::otel::TraceId Trace, std::span<const zen::otel::Span*> Spans);
+ void CheckPostResult(const HttpClient::Response& Result, const char* Endpoint) noexcept;
// This is just a thin wrapper to call back into the sink while participating in
// reference counting from the OTEL trace back-end
@@ -54,11 +56,15 @@ private:
OtelHttpProtobufSink* m_Sink;
};
- HttpClient m_OtelHttp;
- OtlpEncoder m_Encoder;
- Ref<TraceRecorder> m_TraceRecorder;
+ static constexpr uint32_t kMaxReportedFailures = 5;
+
+ RwLock m_Lock;
+ std::atomic<uint32_t> m_ConsecutivePostFailures{0};
+ HttpClient m_OtelHttp;
+ OtlpEncoder m_Encoder;
+ Ref<TraceRecorder> m_TraceRecorder;
};
} // namespace zen::logging
-#endif \ No newline at end of file
+#endif
diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 2b157581f..579a65c5a 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -38,7 +38,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpSt
#if ZEN_EMBED_HTML_ZIP
// Load an embedded Zip archive
IoBuffer HtmlZipDataBuffer(IoBuffer::Wrap, gHtmlZipData, sizeof(gHtmlZipData) - 1);
- m_ZipFs = ZipFs(std::move(HtmlZipDataBuffer));
+ m_ZipFs = std::make_unique<ZipFs>(std::move(HtmlZipDataBuffer));
#endif
if (m_Directory.empty() && !m_ZipFs)
@@ -114,6 +114,8 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
{
using namespace std::literals;
+ ExtendableStringBuilder<256> UriBuilder;
+
std::string_view Uri = Request.RelativeUriWithExtension();
for (; Uri.length() > 0 && Uri[0] == '/'; Uri = Uri.substr(1))
;
@@ -121,6 +123,11 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
{
Uri = "index.html"sv;
}
+ else if (Uri.back() == '/')
+ {
+ UriBuilder << Uri << "index.html"sv;
+ Uri = UriBuilder;
+ }
// Dismiss if the URI contains .. anywhere to prevent arbitrary file reads
if (Uri.find("..") != Uri.npos)
@@ -145,24 +152,47 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
return Request.WriteResponse(HttpResponseCode::Forbidden);
}
- // The given content directory overrides any zip-fs discovered in the binary
- if (!m_Directory.empty())
- {
- auto FullPath = m_Directory / std::filesystem::path(Uri).make_preferred();
- FileContents File = ReadFile(FullPath);
+ auto WriteResponseForUri = [this,
+ &Request](std::string_view InUri, HttpResponseCode ResponseCode, HttpContentType ContentType) -> bool {
+ // The given content directory overrides any zip-fs discovered in the binary
+ if (!m_Directory.empty())
+ {
+ auto FullPath = m_Directory / std::filesystem::path(InUri).make_preferred();
+ FileContents File = ReadFile(FullPath);
- if (!File.ErrorCode)
+ if (!File.ErrorCode)
+ {
+ Request.WriteResponse(ResponseCode, ContentType, File.Data[0]);
+
+ return true;
+ }
+ }
+
+ if (m_ZipFs)
{
- return Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]);
+ if (IoBuffer FileBuffer = m_ZipFs->GetFile(InUri))
+ {
+ Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
+
+ return true;
+ }
}
- }
- if (IoBuffer FileBuffer = m_ZipFs.GetFile(Uri))
+ return false;
+ };
+
+ if (WriteResponseForUri(Uri, HttpResponseCode::OK, ContentType))
{
- return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
+ return;
+ }
+ else if (WriteResponseForUri("404.html"sv, HttpResponseCode::NotFound, HttpContentType::kHTML))
+ {
+ return;
+ }
+ else
+ {
+ Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
}
-
- Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
}
} // namespace zen
diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h
index 84ffaac42..6d8585b72 100644
--- a/src/zenserver/frontend/frontend.h
+++ b/src/zenserver/frontend/frontend.h
@@ -7,6 +7,7 @@
#include "zipfs.h"
#include <filesystem>
+#include <memory>
namespace zen {
@@ -20,9 +21,9 @@ public:
virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
- ZipFs m_ZipFs;
- std::filesystem::path m_Directory;
- HttpStatusService& m_StatusService;
+ std::unique_ptr<ZipFs> m_ZipFs;
+ std::filesystem::path m_Directory;
+ HttpStatusService& m_StatusService;
};
} // namespace zen
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index 5d33302dd..84472ff08 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
Binary files differ
diff --git a/src/zenserver/frontend/html/404.html b/src/zenserver/frontend/html/404.html
new file mode 100644
index 000000000..829ef2097
--- /dev/null
+++ b/src/zenserver/frontend/html/404.html
@@ -0,0 +1,486 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Ooops</title>
+<style>
+ * { margin: 0; padding: 0; box-sizing: border-box; }
+
+ :root {
+ --deep-space: #00000f;
+ --nebula-blue: #0a0a2e;
+ --star-white: #ffffff;
+ --star-blue: #c8d8ff;
+ --star-yellow: #fff3c0;
+ --star-red: #ffd0c0;
+ --nebula-glow: rgba(60, 80, 180, 0.12);
+ }
+
+ body {
+ background: var(--deep-space);
+ min-height: 100vh;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ font-family: 'Courier New', monospace;
+ overflow: hidden;
+ }
+
+ starfield-bg {
+ display: block;
+ position: fixed;
+ inset: 0;
+ z-index: 0;
+ }
+
+ canvas {
+ display: block;
+ width: 100%;
+ height: 100%;
+ }
+
+ .page-content {
+ position: relative;
+ z-index: 1;
+ text-align: center;
+ color: rgba(200, 216, 255, 0.85);
+ letter-spacing: 0.25em;
+ text-transform: uppercase;
+ pointer-events: none;
+ user-select: none;
+ }
+
+ .page-content h1 {
+ font-size: clamp(1.2rem, 4vw, 2.4rem);
+ font-weight: 300;
+ letter-spacing: 0.6em;
+ text-shadow: 0 0 40px rgba(120, 160, 255, 0.6), 0 0 80px rgba(80, 120, 255, 0.3);
+ animation: pulse 6s ease-in-out infinite;
+ }
+
+ .page-content p {
+ margin-top: 1.2rem;
+ font-size: clamp(0.55rem, 1.5vw, 0.75rem);
+ letter-spacing: 0.4em;
+ opacity: 0.45;
+ }
+
+ @keyframes pulse {
+ 0%, 100% { opacity: 0.7; }
+ 50% { opacity: 1; }
+ }
+
+ .globe-link {
+ display: block;
+ margin: 0 auto 2rem;
+ width: 160px;
+ height: 160px;
+ pointer-events: auto;
+ cursor: pointer;
+ border-radius: 50%;
+ position: relative;
+ }
+
+ .globe-link:hover .globe-glow {
+ opacity: 0.6;
+ }
+
+ .globe-glow {
+ position: absolute;
+ inset: -18px;
+ border-radius: 50%;
+ background: radial-gradient(circle, rgba(80, 140, 255, 0.35) 0%, transparent 70%);
+ opacity: 0.35;
+ transition: opacity 0.4s;
+ pointer-events: none;
+ }
+
+ .globe-link canvas {
+ display: block;
+ width: 160px;
+ height: 160px;
+ border-radius: 50%;
+ }
+</style>
+</head>
+<body>
+
+<starfield-bg
+ star-count="380"
+ speed="0.6"
+ depth="true"
+ nebula="true"
+ shooting-stars="true"
+></starfield-bg>
+
+<div class="page-content">
+ <a class="globe-link" href="/dashboard/" title="Back to Dashboard">
+ <div class="globe-glow"></div>
+ <canvas id="globe" width="320" height="320"></canvas>
+ </a>
+ <h1>404 NOT FOUND</h1>
+</div>
+
+<script>
+class StarfieldBg extends HTMLElement {
+ constructor() {
+ super();
+ this.attachShadow({ mode: 'open' });
+ }
+
+ connectedCallback() {
+ this.shadowRoot.innerHTML = `
+ <style>
+ :host { display: block; position: absolute; inset: 0; overflow: hidden; }
+ canvas { width: 100%; height: 100%; display: block; }
+ </style>
+ <canvas></canvas>
+ `;
+
+ this.canvas = this.shadowRoot.querySelector('canvas');
+ this.ctx = this.canvas.getContext('2d');
+
+ this.starCount = parseInt(this.getAttribute('star-count') || '350');
+ this.speed = parseFloat(this.getAttribute('speed') || '0.6');
+ this.useDepth = this.getAttribute('depth') !== 'false';
+ this.useNebula = this.getAttribute('nebula') !== 'false';
+ this.useShooting = this.getAttribute('shooting-stars') !== 'false';
+
+ this.stars = [];
+ this.shooters = [];
+ this.nebulaTime = 0;
+ this.frame = 0;
+
+ this.resize();
+ this.init();
+
+ this._ro = new ResizeObserver(() => { this.resize(); this.init(); });
+ this._ro.observe(this);
+
+ this.raf = requestAnimationFrame(this.tick.bind(this));
+ }
+
+ disconnectedCallback() {
+ cancelAnimationFrame(this.raf);
+ this._ro.disconnect();
+ }
+
+ resize() {
+ const dpr = window.devicePixelRatio || 1;
+ const rect = this.getBoundingClientRect();
+ this.W = rect.width || window.innerWidth;
+ this.H = rect.height || window.innerHeight;
+ this.canvas.width = this.W * dpr;
+ this.canvas.height = this.H * dpr;
+ this.ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
+ }
+
+ init() {
+ const COLORS = ['#ffffff', '#c8d8ff', '#d0e8ff', '#fff3c0', '#ffd0c0', '#e0f0ff'];
+ this.stars = Array.from({ length: this.starCount }, () => ({
+ x: Math.random() * this.W,
+ y: Math.random() * this.H,
+ z: this.useDepth ? Math.random() : 1, // depth: 0=far, 1=near
+ r: Math.random() * 1.4 + 0.2,
+ color: COLORS[Math.floor(Math.random() * COLORS.length)],
+ twinkleOffset: Math.random() * Math.PI * 2,
+ twinkleSpeed: 0.008 + Math.random() * 0.012,
+ }));
+ }
+
+ spawnShooter() {
+ const edge = Math.random() < 0.7 ? 'top' : 'left';
+ const angle = (Math.random() * 30 + 15) * (Math.PI / 180);
+ this.shooters.push({
+ x: edge === 'top' ? Math.random() * this.W : -10,
+ y: edge === 'top' ? -10 : Math.random() * this.H * 0.5,
+ vx: Math.cos(angle) * (6 + Math.random() * 6),
+ vy: Math.sin(angle) * (6 + Math.random() * 6),
+ len: 80 + Math.random() * 120,
+ life: 1,
+ decay: 0.012 + Math.random() * 0.018,
+ });
+ }
+
+ tick() {
+ this.raf = requestAnimationFrame(this.tick.bind(this));
+ this.frame++;
+ const ctx = this.ctx;
+ const W = this.W, H = this.H;
+
+ // Background
+ ctx.fillStyle = '#00000f';
+ ctx.fillRect(0, 0, W, H);
+
+ // Nebula clouds (subtle)
+ if (this.useNebula) {
+ this.nebulaTime += 0.003;
+ this.drawNebula(ctx, W, H);
+ }
+
+ // Stars
+ for (const s of this.stars) {
+ const twinkle = 0.55 + 0.45 * Math.sin(this.frame * s.twinkleSpeed + s.twinkleOffset);
+ const radius = s.r * (this.useDepth ? (0.3 + s.z * 0.7) : 1);
+ const alpha = (this.useDepth ? (0.25 + s.z * 0.75) : 1) * twinkle;
+
+ // Tiny drift
+ s.x += (s.z * this.speed * 0.08) * (this.useDepth ? 1 : 0);
+ s.y += (s.z * this.speed * 0.04) * (this.useDepth ? 1 : 0);
+ if (s.x > W + 2) s.x = -2;
+ if (s.y > H + 2) s.y = -2;
+
+ // Glow for bright stars
+ if (radius > 1.1 && alpha > 0.6) {
+ const grd = ctx.createRadialGradient(s.x, s.y, 0, s.x, s.y, radius * 3.5);
+ grd.addColorStop(0, s.color.replace(')', `, ${alpha * 0.5})`).replace('rgb', 'rgba'));
+ grd.addColorStop(1, 'transparent');
+ ctx.beginPath();
+ ctx.arc(s.x, s.y, radius * 3.5, 0, Math.PI * 2);
+ ctx.fillStyle = grd;
+ ctx.fill();
+ }
+
+ ctx.beginPath();
+ ctx.arc(s.x, s.y, radius, 0, Math.PI * 2);
+ ctx.fillStyle = hexToRgba(s.color, alpha);
+ ctx.fill();
+ }
+
+ // Shooting stars
+ if (this.useShooting) {
+ if (this.frame % 140 === 0 && Math.random() < 0.65) this.spawnShooter();
+ for (let i = this.shooters.length - 1; i >= 0; i--) {
+ const s = this.shooters[i];
+ const tailX = s.x - s.vx * (s.len / Math.hypot(s.vx, s.vy));
+ const tailY = s.y - s.vy * (s.len / Math.hypot(s.vx, s.vy));
+
+ const grd = ctx.createLinearGradient(tailX, tailY, s.x, s.y);
+ grd.addColorStop(0, `rgba(255,255,255,0)`);
+ grd.addColorStop(0.7, `rgba(200,220,255,${s.life * 0.5})`);
+ grd.addColorStop(1, `rgba(255,255,255,${s.life})`);
+
+ ctx.beginPath();
+ ctx.moveTo(tailX, tailY);
+ ctx.lineTo(s.x, s.y);
+ ctx.strokeStyle = grd;
+ ctx.lineWidth = 1.5 * s.life;
+ ctx.lineCap = 'round';
+ ctx.stroke();
+
+ // Head dot
+ ctx.beginPath();
+ ctx.arc(s.x, s.y, 1.5 * s.life, 0, Math.PI * 2);
+ ctx.fillStyle = `rgba(255,255,255,${s.life})`;
+ ctx.fill();
+
+ s.x += s.vx;
+ s.y += s.vy;
+ s.life -= s.decay;
+
+ if (s.life <= 0 || s.x > W + 200 || s.y > H + 200) {
+ this.shooters.splice(i, 1);
+ }
+ }
+ }
+ }
+
+ drawNebula(ctx, W, H) {
+ const t = this.nebulaTime;
+ const blobs = [
+ { x: W * 0.25, y: H * 0.3, rx: W * 0.35, ry: H * 0.25, color: '40,60,180', a: 0.055 },
+ { x: W * 0.75, y: H * 0.65, rx: W * 0.30, ry: H * 0.22, color: '100,40,160', a: 0.04 },
+ { x: W * 0.5, y: H * 0.5, rx: W * 0.45, ry: H * 0.35, color: '20,50,120', a: 0.035 },
+ ];
+ ctx.save();
+ for (const b of blobs) {
+ const ox = Math.sin(t * 0.7 + b.x) * 30;
+ const oy = Math.cos(t * 0.5 + b.y) * 20;
+ const grd = ctx.createRadialGradient(b.x + ox, b.y + oy, 0, b.x + ox, b.y + oy, Math.max(b.rx, b.ry));
+ grd.addColorStop(0, `rgba(${b.color}, ${b.a})`);
+ grd.addColorStop(0.5, `rgba(${b.color}, ${b.a * 0.4})`);
+ grd.addColorStop(1, `rgba(${b.color}, 0)`);
+ ctx.save();
+ ctx.scale(b.rx / Math.max(b.rx, b.ry), b.ry / Math.max(b.rx, b.ry));
+ ctx.beginPath();
+ const scale = Math.max(b.rx, b.ry);
+ ctx.arc((b.x + ox) / (b.rx / scale), (b.y + oy) / (b.ry / scale), scale, 0, Math.PI * 2);
+ ctx.fillStyle = grd;
+ ctx.fill();
+ ctx.restore();
+ }
+ ctx.restore();
+ }
+}
+
+function hexToRgba(hex, alpha) {
+ // Handle named-ish values or full hex
+ const c = hex.startsWith('#') ? hex : '#ffffff';
+ const r = parseInt(c.slice(1,3), 16);
+ const g = parseInt(c.slice(3,5), 16);
+ const b = parseInt(c.slice(5,7), 16);
+ return `rgba(${r},${g},${b},${alpha.toFixed(3)})`;
+}
+
+customElements.define('starfield-bg', StarfieldBg);
+</script>
+
+<script>
+(function() {
+ const canvas = document.getElementById('globe');
+ const ctx = canvas.getContext('2d');
+ const W = canvas.width, H = canvas.height;
+ const R = W * 0.44;
+ const cx = W / 2, cy = H / 2;
+
+ // Simplified continent outlines as lon/lat polygon chains (degrees).
+ // Each continent is an array of [lon, lat] points.
+ const continents = [
+ // North America
+ [[-130,50],[-125,55],[-120,60],[-115,65],[-100,68],[-85,70],[-75,65],[-60,52],[-65,45],[-70,42],[-75,35],[-80,30],[-85,28],[-90,28],[-95,25],[-100,20],[-105,20],[-110,25],[-115,30],[-120,35],[-125,42],[-130,50]],
+ // South America
+ [[-80,10],[-75,5],[-70,5],[-65,0],[-60,-5],[-55,-5],[-50,-10],[-45,-15],[-40,-20],[-40,-25],[-42,-30],[-48,-32],[-52,-34],[-55,-38],[-60,-42],[-65,-50],[-68,-55],[-70,-48],[-72,-40],[-75,-30],[-78,-15],[-80,-5],[-80,5],[-80,10]],
+ // Europe
+ [[-10,36],[-5,38],[0,40],[2,43],[5,44],[8,46],[10,48],[15,50],[18,54],[20,56],[25,58],[28,60],[30,62],[35,65],[40,68],[38,60],[35,55],[30,50],[28,48],[25,45],[22,40],[20,38],[15,36],[10,36],[5,36],[0,36],[-5,36],[-10,36]],
+ // Africa
+ [[-15,14],[-17,16],[-15,22],[-12,28],[-5,32],[0,35],[5,37],[10,35],[15,32],[20,30],[25,30],[30,28],[35,25],[38,18],[40,12],[42,5],[44,0],[42,-5],[40,-12],[38,-18],[35,-25],[32,-30],[30,-34],[25,-33],[20,-30],[15,-28],[12,-20],[10,-10],[8,-5],[5,0],[2,5],[0,5],[-5,5],[-10,6],[-15,10],[-15,14]],
+ // Asia (simplified)
+ [[30,35],[35,38],[40,40],[45,42],[50,45],[55,48],[60,50],[65,55],[70,60],[75,65],[80,68],[90,70],[100,68],[110,65],[120,60],[125,55],[130,50],[135,45],[140,40],[138,35],[130,30],[120,25],[110,20],[105,15],[100,10],[95,12],[90,20],[85,22],[80,25],[75,28],[70,30],[65,35],[55,35],[45,35],[40,35],[35,35],[30,35]],
+ // Australia
+ [[115,-12],[120,-14],[125,-15],[130,-14],[135,-13],[138,-16],[140,-18],[145,-20],[148,-22],[150,-25],[152,-28],[150,-33],[148,-35],[145,-37],[140,-38],[135,-36],[130,-33],[125,-30],[120,-25],[118,-22],[116,-20],[114,-18],[115,-15],[115,-12]],
+ ];
+
+ function project(lon, lat, rotation) {
+ // Convert to radians and apply rotation
+ var lonR = (lon + rotation) * Math.PI / 180;
+ var latR = lat * Math.PI / 180;
+
+ var x3 = Math.cos(latR) * Math.sin(lonR);
+ var y3 = -Math.sin(latR);
+ var z3 = Math.cos(latR) * Math.cos(lonR);
+
+ // Only visible if facing us
+ if (z3 < 0) return null;
+
+ return { x: cx + x3 * R, y: cy + y3 * R, z: z3 };
+ }
+
+ var rotation = 0;
+
+ function draw() {
+ requestAnimationFrame(draw);
+ rotation += 0.15;
+ ctx.clearRect(0, 0, W, H);
+
+ // Atmosphere glow
+ var atm = ctx.createRadialGradient(cx, cy, R * 0.85, cx, cy, R * 1.15);
+ atm.addColorStop(0, 'rgba(60,130,255,0.12)');
+ atm.addColorStop(0.5, 'rgba(60,130,255,0.06)');
+ atm.addColorStop(1, 'rgba(60,130,255,0)');
+ ctx.beginPath();
+ ctx.arc(cx, cy, R * 1.15, 0, Math.PI * 2);
+ ctx.fillStyle = atm;
+ ctx.fill();
+
+ // Ocean sphere
+ var oceanGrad = ctx.createRadialGradient(cx - R * 0.3, cy - R * 0.3, R * 0.1, cx, cy, R);
+ oceanGrad.addColorStop(0, '#1a4a8a');
+ oceanGrad.addColorStop(0.5, '#0e2d5e');
+ oceanGrad.addColorStop(1, '#071838');
+ ctx.beginPath();
+ ctx.arc(cx, cy, R, 0, Math.PI * 2);
+ ctx.fillStyle = oceanGrad;
+ ctx.fill();
+
+ // Draw continents
+ for (var c = 0; c < continents.length; c++) {
+ var pts = continents[c];
+ var projected = [];
+ var allVisible = true;
+
+ for (var i = 0; i < pts.length; i++) {
+ var p = project(pts[i][0], pts[i][1], rotation);
+ if (!p) { allVisible = false; break; }
+ projected.push(p);
+ }
+
+ if (!allVisible || projected.length < 3) continue;
+
+ ctx.beginPath();
+ ctx.moveTo(projected[0].x, projected[0].y);
+ for (var i = 1; i < projected.length; i++) {
+ ctx.lineTo(projected[i].x, projected[i].y);
+ }
+ ctx.closePath();
+
+ // Shade based on average depth
+ var avgZ = 0;
+ for (var i = 0; i < projected.length; i++) avgZ += projected[i].z;
+ avgZ /= projected.length;
+ var brightness = 0.3 + avgZ * 0.7;
+
+ var r = Math.round(30 * brightness);
+ var g = Math.round(100 * brightness);
+ var b = Math.round(50 * brightness);
+ ctx.fillStyle = 'rgb(' + r + ',' + g + ',' + b + ')';
+ ctx.fill();
+ }
+
+ // Grid lines (longitude)
+ ctx.strokeStyle = 'rgba(100,160,255,0.08)';
+ ctx.lineWidth = 0.7;
+ for (var lon = -180; lon < 180; lon += 30) {
+ ctx.beginPath();
+ var started = false;
+ for (var lat = -90; lat <= 90; lat += 3) {
+ var p = project(lon, lat, rotation);
+ if (p) {
+ if (!started) { ctx.moveTo(p.x, p.y); started = true; }
+ else ctx.lineTo(p.x, p.y);
+ } else {
+ started = false;
+ }
+ }
+ ctx.stroke();
+ }
+
+ // Grid lines (latitude)
+ for (var lat = -60; lat <= 60; lat += 30) {
+ ctx.beginPath();
+ var started = false;
+ for (var lon = -180; lon <= 180; lon += 3) {
+ var p = project(lon, lat, rotation);
+ if (p) {
+ if (!started) { ctx.moveTo(p.x, p.y); started = true; }
+ else ctx.lineTo(p.x, p.y);
+ } else {
+ started = false;
+ }
+ }
+ ctx.stroke();
+ }
+
+ // Specular highlight
+ var spec = ctx.createRadialGradient(cx - R * 0.35, cy - R * 0.35, 0, cx - R * 0.35, cy - R * 0.35, R * 0.8);
+ spec.addColorStop(0, 'rgba(180,210,255,0.18)');
+ spec.addColorStop(0.4, 'rgba(120,160,255,0.05)');
+ spec.addColorStop(1, 'rgba(0,0,0,0)');
+ ctx.beginPath();
+ ctx.arc(cx, cy, R, 0, Math.PI * 2);
+ ctx.fillStyle = spec;
+ ctx.fill();
+
+ // Rim light
+ ctx.beginPath();
+ ctx.arc(cx, cy, R, 0, Math.PI * 2);
+ ctx.strokeStyle = 'rgba(80,140,255,0.2)';
+ ctx.lineWidth = 1.5;
+ ctx.stroke();
+ }
+
+ draw();
+})();
+</script>
+</body>
+</html>
diff --git a/src/zenserver/frontend/html/banner.js b/src/zenserver/frontend/html/banner.js
new file mode 100644
index 000000000..2e878dedf
--- /dev/null
+++ b/src/zenserver/frontend/html/banner.js
@@ -0,0 +1,338 @@
+/**
+ * zen-banner.js — Zen dashboard banner Web Component
+ *
+ * Usage:
+ * <script src="banner.js" defer></script>
+ *
+ * <zen-banner></zen-banner>
+ * <zen-banner variant="compact"></zen-banner>
+ * <zen-banner cluster-status="degraded" load="78"></zen-banner>
+ *
+ * Attributes:
+ * variant "full" (default) | "compact"
+ * cluster-status "nominal" (default) | "degraded" | "offline"
+ * load 0–100 integer, shown as a percentage (default: hidden)
+ * tagline custom tagline text (default: "Orchestrator Overview" / "Orchestrator")
+ * subtitle text after "ZEN" in the wordmark (default: "COMPUTE")
+ */
+
+class ZenBanner extends HTMLElement {
+
+ static get observedAttributes() {
+ return ['variant', 'cluster-status', 'load', 'tagline', 'subtitle', 'logo-src'];
+ }
+
+ attributeChangedCallback() {
+ if (this.shadowRoot) this._render();
+ }
+
+ connectedCallback() {
+ if (!this.shadowRoot) this.attachShadow({ mode: 'open' });
+ this._render();
+ }
+
+ // ─────────────────────────────────────────────
+ // Derived values
+ // ─────────────────────────────────────────────
+
+ get _variant() { return this.getAttribute('variant') || 'full'; }
+ get _status() { return (this.getAttribute('cluster-status') || 'nominal').toLowerCase(); }
+ get _load() { return this.getAttribute('load'); } // null → hidden
+ get _tagline() { return this.getAttribute('tagline'); } // null → default
+ get _subtitle() { return this.getAttribute('subtitle'); } // null → "COMPUTE"
+ get _logoSrc() { return this.getAttribute('logo-src'); } // null → inline SVG
+
+ get _statusColor() {
+ return { nominal: '#7ecfb8', degraded: '#d4a84b', offline: '#c0504d' }[this._status] ?? '#7ecfb8';
+ }
+
+ get _statusLabel() {
+ return { nominal: 'NOMINAL', degraded: 'DEGRADED', offline: 'OFFLINE' }[this._status] ?? 'NOMINAL';
+ }
+
+ get _loadColor() {
+ const v = parseInt(this._load, 10);
+ if (isNaN(v)) return '#7ecfb8';
+ if (v >= 85) return '#c0504d';
+ if (v >= 60) return '#d4a84b';
+ return '#7ecfb8';
+ }
+
+ // ─────────────────────────────────────────────
+ // Render
+ // ─────────────────────────────────────────────
+
+ _render() {
+ const compact = this._variant === 'compact';
+ this.shadowRoot.innerHTML = `
+ <style>${this._css(compact)}</style>
+ ${this._html(compact)}
+ `;
+ }
+
+ // ─────────────────────────────────────────────
+ // CSS
+ // ─────────────────────────────────────────────
+
+ _css(compact) {
+ const height = compact ? '60px' : '100px';
+ const padding = compact ? '0 24px' : '0 32px';
+ const gap = compact ? '16px' : '24px';
+ const markSize = compact ? '34px' : '52px';
+ const divH = compact ? '32px' : '48px';
+ const nameSize = compact ? '15px' : '22px';
+ const tagSize = compact ? '9px' : '11px';
+ const sc = this._statusColor;
+ const lc = this._loadColor;
+
+ return `
+ @import url('https://fonts.googleapis.com/css2?family=Noto+Serif+JP:wght@300;400&family=Space+Mono:wght@400;700&display=swap');
+
+ *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+ :host {
+ display: block;
+ font-family: 'Space Mono', monospace;
+ }
+
+ .banner {
+ width: 100%;
+ height: ${height};
+ background: var(--theme_g3, #0b0d10);
+ border: 1px solid var(--theme_g2, #1e2330);
+ border-radius: 6px;
+ display: flex;
+ align-items: center;
+ padding: ${padding};
+ gap: ${gap};
+ position: relative;
+ overflow: hidden;
+ text-decoration: none;
+ color: inherit;
+ cursor: pointer;
+ }
+
+ /* scan-line texture */
+ .banner::before {
+ content: '';
+ position: absolute;
+ inset: 0;
+ background: repeating-linear-gradient(
+ 0deg,
+ transparent, transparent 3px,
+ rgba(255,255,255,0.012) 3px, rgba(255,255,255,0.012) 4px
+ );
+ pointer-events: none;
+ }
+
+ /* ambient glow */
+ .banner::after {
+ content: '';
+ position: absolute;
+ right: -60px;
+ top: 50%;
+ transform: translateY(-50%);
+ width: 280px;
+ height: 280px;
+ background: radial-gradient(circle, rgba(130,200,180,0.06) 0%, transparent 70%);
+ pointer-events: none;
+ }
+
+ .logo-mark {
+ flex-shrink: 0;
+ width: ${markSize};
+ height: ${markSize};
+ }
+
+ .logo-mark svg, .logo-mark img { width: 100%; height: 100%; object-fit: contain; }
+
+ .divider {
+ width: 1px;
+ height: ${divH};
+ background: linear-gradient(to bottom, transparent, var(--theme_g2, #2a3040), transparent);
+ flex-shrink: 0;
+ }
+
+ .text-block {
+ display: flex;
+ flex-direction: column;
+ gap: 4px;
+ }
+
+ .wordmark {
+ font-weight: 700;
+ font-size: ${nameSize};
+ letter-spacing: 0.12em;
+ color: var(--theme_bright, #e8e4dc);
+ text-transform: uppercase;
+ line-height: 1;
+ }
+
+ .wordmark span { color: #7ecfb8; }
+
+ .tagline {
+ font-family: 'Noto Serif JP', serif;
+ font-weight: 300;
+ font-size: ${tagSize};
+ letter-spacing: 0.3em;
+ color: var(--theme_faint, #4a5a68);
+ text-transform: uppercase;
+ }
+
+ .spacer { flex: 1; }
+
+ /* ── right-side decorative circuit ── */
+ .circuit { flex-shrink: 0; opacity: 0.22; }
+
+ /* ── status cluster ── */
+ .status-cluster {
+ display: flex;
+ flex-direction: column;
+ align-items: flex-end;
+ gap: 6px;
+ }
+
+ .status-row {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ }
+
+ .status-lbl {
+ font-size: 9px;
+ letter-spacing: 0.18em;
+ color: var(--theme_faint, #3a4555);
+ text-transform: uppercase;
+ }
+
+ .pill {
+ display: flex;
+ align-items: center;
+ gap: 5px;
+ border-radius: 20px;
+ padding: 2px 10px;
+ font-size: 10px;
+ letter-spacing: 0.1em;
+ }
+
+ .pill.cluster {
+ color: ${sc};
+ background: color-mix(in srgb, ${sc} 8%, transparent);
+ border: 1px solid color-mix(in srgb, ${sc} 28%, transparent);
+ }
+
+ .pill.load-pill {
+ color: ${lc};
+ background: color-mix(in srgb, ${lc} 8%, transparent);
+ border: 1px solid color-mix(in srgb, ${lc} 28%, transparent);
+ }
+
+ .dot {
+ width: 5px;
+ height: 5px;
+ border-radius: 50%;
+ animation: pulse 2.4s ease-in-out infinite;
+ }
+
+ .dot.cluster { background: ${sc}; }
+ .dot.load-dot { background: ${lc}; animation-delay: 0.5s; }
+
+ @keyframes pulse {
+ 0%, 100% { opacity: 1; }
+ 50% { opacity: 0.25; }
+ }
+ `;
+ }
+
+ // ─────────────────────────────────────────────
+ // HTML template
+ // ─────────────────────────────────────────────
+
+ _html(compact) {
+ const loadAttr = this._load;
+ const hasCluster = !compact && this.hasAttribute('cluster-status');
+ const hasLoad = !compact && loadAttr !== null;
+ const showRight = hasCluster || hasLoad;
+
+ const circuit = showRight ? `
+ <svg class="circuit" width="60" height="60" viewBox="0 0 60 60" fill="none">
+ <path d="M5 30 H22 L28 18 H60" stroke="#7ecfb8" stroke-width="0.8"/>
+ <path d="M5 38 H18 L24 46 H60" stroke="#7ecfb8" stroke-width="0.8"/>
+ <circle cx="22" cy="30" r="2" fill="none" stroke="#7ecfb8" stroke-width="0.8"/>
+ <circle cx="18" cy="38" r="2" fill="none" stroke="#7ecfb8" stroke-width="0.8"/>
+ <circle cx="10" cy="30" r="1.2" fill="#7ecfb8"/>
+ <circle cx="10" cy="38" r="1.2" fill="#7ecfb8"/>
+ </svg>` : '';
+
+ const clusterRow = hasCluster ? `
+ <div class="status-row">
+ <span class="status-lbl">Cluster</span>
+ <div class="pill cluster">
+ <div class="dot cluster"></div>
+ ${this._statusLabel}
+ </div>
+ </div>` : '';
+
+ const loadRow = hasLoad ? `
+ <div class="status-row">
+ <span class="status-lbl">Load</span>
+ <div class="pill load-pill">
+ <div class="dot load-dot"></div>
+ ${parseInt(loadAttr, 10)} %
+ </div>
+ </div>` : '';
+
+ const rightSide = showRight ? `
+ ${circuit}
+ <div class="status-cluster">
+ ${clusterRow}
+ ${loadRow}
+ </div>
+ ` : '';
+
+ return `
+ <a class="banner" href="/dashboard/">
+ <div class="logo-mark">${this._logoMark()}</div>
+ <div class="divider"></div>
+ <div class="text-block">
+ <div class="wordmark">ZEN<span> ${this._subtitle ?? 'COMPUTE'}</span></div>
+ <div class="tagline">${this._tagline ?? (compact ? 'Orchestrator' : 'Orchestrator Overview')}</div>
+ </div>
+ <div class="spacer"></div>
+ ${rightSide}
+ </a>
+ `;
+ }
+
+ // ─────────────────────────────────────────────
+ // SVG logo mark
+ // ─────────────────────────────────────────────
+
+ _logoMark() {
+ const src = this._logoSrc;
+ if (src) {
+ return `<img src="${src}" alt="zen">`;
+ }
+ return `
+ <svg viewBox="0 0 52 52" fill="none" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="26" cy="26" r="22" stroke="#2a3a48" stroke-width="1.5"/>
+ <path d="M26 4 A22 22 0 1 1 12 43.1" stroke="#7ecfb8" stroke-width="2" stroke-linecap="round" fill="none"/>
+ <circle cx="17" cy="17" r="1.6" fill="#7ecfb8" />
+ <circle cx="26" cy="17" r="1.6" fill="#7ecfb8" />
+ <circle cx="35" cy="17" r="1.6" fill="#7ecfb8" />
+ <circle cx="17" cy="26" r="1.6" fill="#7ecfb8" opacity="0.6"/>
+ <circle cx="26" cy="26" r="2.2" fill="#7ecfb8"/>
+ <circle cx="35" cy="26" r="1.6" fill="#7ecfb8" opacity="0.6"/>
+ <circle cx="17" cy="35" r="1.6" fill="#7ecfb8"/>
+ <circle cx="26" cy="35" r="1.6" fill="#7ecfb8"/>
+ <circle cx="35" cy="35" r="1.6" fill="#7ecfb8"/>
+ <line x1="17" y1="17" x2="35" y2="17" stroke="#7ecfb8" stroke-width="0.7" stroke-opacity="0.25"/>
+ <line x1="35" y1="17" x2="17" y2="35" stroke="#7ecfb8" stroke-width="0.7" stroke-opacity="0.25"/>
+ <line x1="17" y1="35" x2="35" y2="35" stroke="#7ecfb8" stroke-width="0.7" stroke-opacity="0.2"/>
+ <line x1="26" y1="17" x2="26" y2="35" stroke="#7ecfb8" stroke-width="0.7" stroke-opacity="0.2"/>
+ </svg>
+ `;
+ }
+}
+
+customElements.define('zen-banner', ZenBanner);
diff --git a/src/zenserver/frontend/html/compute/compute.html b/src/zenserver/frontend/html/compute/compute.html
new file mode 100644
index 000000000..66c20175f
--- /dev/null
+++ b/src/zenserver/frontend/html/compute/compute.html
@@ -0,0 +1,929 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>Zen Compute Dashboard</title>
+ <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.umd.min.js"></script>
+ <link rel="stylesheet" type="text/css" href="../zen.css" />
+ <script src="../theme.js"></script>
+ <script src="../banner.js" defer></script>
+ <script src="../nav.js" defer></script>
+ <style>
+ .grid {
+ grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
+ }
+
+ .chart-container {
+ position: relative;
+ height: 300px;
+ margin-top: 20px;
+ }
+
+ .stats-row {
+ display: flex;
+ justify-content: space-between;
+ margin-bottom: 12px;
+ padding: 8px 0;
+ border-bottom: 1px solid var(--theme_border_subtle);
+ }
+
+ .stats-row:last-child {
+ border-bottom: none;
+ margin-bottom: 0;
+ }
+
+ .stats-label {
+ color: var(--theme_g1);
+ font-size: 13px;
+ }
+
+ .stats-value {
+ color: var(--theme_bright);
+ font-weight: 600;
+ font-size: 13px;
+ }
+
+ .rate-stats {
+ display: grid;
+ grid-template-columns: repeat(3, 1fr);
+ gap: 16px;
+ margin-top: 16px;
+ }
+
+ .rate-item {
+ text-align: center;
+ }
+
+ .rate-value {
+ font-size: 20px;
+ font-weight: 600;
+ color: var(--theme_p0);
+ }
+
+ .rate-label {
+ font-size: 11px;
+ color: var(--theme_g1);
+ margin-top: 4px;
+ text-transform: uppercase;
+ }
+
+ .worker-row {
+ cursor: pointer;
+ transition: background 0.15s;
+ }
+
+ .worker-row:hover {
+ background: var(--theme_p4);
+ }
+
+ .worker-row.selected {
+ background: var(--theme_p3);
+ }
+
+ .worker-detail {
+ margin-top: 20px;
+ border-top: 1px solid var(--theme_g2);
+ padding-top: 16px;
+ }
+
+ .worker-detail-title {
+ font-size: 15px;
+ font-weight: 600;
+ color: var(--theme_bright);
+ margin-bottom: 12px;
+ }
+
+ .detail-section {
+ margin-bottom: 16px;
+ }
+
+ .detail-section-label {
+ font-size: 11px;
+ font-weight: 600;
+ color: var(--theme_g1);
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+ margin-bottom: 6px;
+ }
+
+ .detail-table {
+ width: 100%;
+ border-collapse: collapse;
+ font-size: 12px;
+ }
+
+ .detail-table td {
+ padding: 4px 8px;
+ color: var(--theme_g0);
+ border-bottom: 1px solid var(--theme_border_subtle);
+ vertical-align: top;
+ }
+
+ .detail-table td:first-child {
+ color: var(--theme_g1);
+ width: 40%;
+ font-family: monospace;
+ }
+
+ .detail-table tr:last-child td {
+ border-bottom: none;
+ }
+
+ .detail-mono {
+ font-family: monospace;
+ font-size: 11px;
+ color: var(--theme_g1);
+ }
+
+ .detail-tag {
+ display: inline-block;
+ padding: 2px 8px;
+ border-radius: 4px;
+ background: var(--theme_border_subtle);
+ color: var(--theme_g0);
+ font-size: 11px;
+ margin: 2px 4px 2px 0;
+ }
+ </style>
+</head>
+<body>
+ <div class="container" style="max-width: 1400px; margin: 0 auto;">
+ <zen-banner cluster-status="nominal" load="0" tagline="Node Overview" logo-src="../favicon.ico"></zen-banner>
+ <zen-nav>
+ <a href="/dashboard/">Home</a>
+ <a href="compute.html">Node</a>
+ <a href="orchestrator.html">Orchestrator</a>
+ </zen-nav>
+ <div class="timestamp">Last updated: <span id="last-update">Never</span></div>
+
+ <div id="error-container"></div>
+
+ <!-- Action Queue Stats -->
+ <div class="section-title">Action Queue</div>
+ <div class="grid">
+ <div class="card">
+ <div class="card-title">Pending Actions</div>
+ <div class="metric-value" id="actions-pending">-</div>
+ <div class="metric-label">Waiting to be scheduled</div>
+ </div>
+ <div class="card">
+ <div class="card-title">Running Actions</div>
+ <div class="metric-value" id="actions-running">-</div>
+ <div class="metric-label">Currently executing</div>
+ </div>
+ <div class="card">
+ <div class="card-title">Completed Actions</div>
+ <div class="metric-value" id="actions-complete">-</div>
+ <div class="metric-label">Results available</div>
+ </div>
+ </div>
+
+ <!-- Action Queue Chart -->
+ <div class="card" style="margin-bottom: 30px;">
+ <div class="card-title">Action Queue History</div>
+ <div class="chart-container">
+ <canvas id="queue-chart"></canvas>
+ </div>
+ </div>
+
+ <!-- Performance Metrics -->
+ <div class="section-title">Performance Metrics</div>
+ <div class="card" style="margin-bottom: 30px;">
+ <div class="card-title">Completion Rate</div>
+ <div class="rate-stats">
+ <div class="rate-item">
+ <div class="rate-value" id="rate-1">-</div>
+ <div class="rate-label">1 min rate</div>
+ </div>
+ <div class="rate-item">
+ <div class="rate-value" id="rate-5">-</div>
+ <div class="rate-label">5 min rate</div>
+ </div>
+ <div class="rate-item">
+ <div class="rate-value" id="rate-15">-</div>
+ <div class="rate-label">15 min rate</div>
+ </div>
+ </div>
+ <div style="margin-top: 20px;">
+ <div class="stats-row">
+ <span class="stats-label">Total Retired</span>
+ <span class="stats-value" id="retired-count">-</span>
+ </div>
+ <div class="stats-row">
+ <span class="stats-label">Mean Rate</span>
+ <span class="stats-value" id="rate-mean">-</span>
+ </div>
+ </div>
+ </div>
+
+ <!-- Workers -->
+ <div class="section-title">Workers</div>
+ <div class="card" style="margin-bottom: 30px;">
+ <div class="card-title">Worker Status</div>
+ <div class="stats-row">
+ <span class="stats-label">Registered Workers</span>
+ <span class="stats-value" id="worker-count">-</span>
+ </div>
+ <div id="worker-table-container" style="margin-top: 16px; display: none;">
+ <table id="worker-table">
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Platform</th>
+ <th style="text-align: right;">Cores</th>
+ <th style="text-align: right;">Timeout</th>
+ <th style="text-align: right;">Functions</th>
+ <th>Worker ID</th>
+ </tr>
+ </thead>
+ <tbody id="worker-table-body"></tbody>
+ </table>
+ <div id="worker-detail" class="worker-detail" style="display: none;"></div>
+ </div>
+ </div>
+
+ <!-- Queues -->
+ <div class="section-title">Queues</div>
+ <div class="card" style="margin-bottom: 30px;">
+ <div class="card-title">Queue Status</div>
+ <div id="queue-list-empty" class="empty-state" style="text-align: left;">No queues.</div>
+ <div id="queue-list-container" style="display: none;">
+ <table id="queue-list-table">
+ <thead>
+ <tr>
+ <th style="text-align: right; width: 60px;">ID</th>
+ <th style="text-align: center; width: 80px;">Status</th>
+ <th style="text-align: right;">Active</th>
+ <th style="text-align: right;">Completed</th>
+ <th style="text-align: right;">Failed</th>
+ <th style="text-align: right;">Abandoned</th>
+ <th style="text-align: right;">Cancelled</th>
+ <th>Token</th>
+ </tr>
+ </thead>
+ <tbody id="queue-list-body"></tbody>
+ </table>
+ </div>
+ </div>
+
+ <!-- Action History -->
+ <div class="section-title">Recent Actions</div>
+ <div class="card" style="margin-bottom: 30px;">
+ <div class="card-title">Action History</div>
+ <div id="action-history-empty" class="empty-state" style="text-align: left;">No actions recorded yet.</div>
+ <div id="action-history-container" style="display: none;">
+ <table id="action-history-table">
+ <thead>
+ <tr>
+ <th style="text-align: right; width: 60px;">LSN</th>
+ <th style="text-align: right; width: 60px;">Queue</th>
+ <th style="text-align: center; width: 70px;">Status</th>
+ <th>Function</th>
+ <th style="text-align: right; width: 80px;">Started</th>
+ <th style="text-align: right; width: 80px;">Finished</th>
+ <th style="text-align: right; width: 80px;">Duration</th>
+ <th>Worker ID</th>
+ <th>Action ID</th>
+ </tr>
+ </thead>
+ <tbody id="action-history-body"></tbody>
+ </table>
+ </div>
+ </div>
+
+ <!-- System Resources -->
+ <div class="section-title">System Resources</div>
+ <div class="grid">
+ <div class="card">
+ <div class="card-title">CPU Usage</div>
+ <div class="metric-value" id="cpu-usage">-</div>
+ <div class="metric-label">Percent</div>
+ <div class="progress-bar">
+ <div class="progress-fill" id="cpu-progress" style="width: 0%"></div>
+ </div>
+ <div style="position: relative; height: 60px; margin-top: 12px;">
+ <canvas id="cpu-chart"></canvas>
+ </div>
+ <div style="margin-top: 12px;">
+ <div class="stats-row">
+ <span class="stats-label">Packages</span>
+ <span class="stats-value" id="cpu-packages">-</span>
+ </div>
+ <div class="stats-row">
+ <span class="stats-label">Physical Cores</span>
+ <span class="stats-value" id="cpu-cores">-</span>
+ </div>
+ <div class="stats-row">
+ <span class="stats-label">Logical Processors</span>
+ <span class="stats-value" id="cpu-lp">-</span>
+ </div>
+ </div>
+ </div>
+ <div class="card">
+ <div class="card-title">Memory</div>
+ <div class="stats-row">
+ <span class="stats-label">Used</span>
+ <span class="stats-value" id="memory-used">-</span>
+ </div>
+ <div class="stats-row">
+ <span class="stats-label">Total</span>
+ <span class="stats-value" id="memory-total">-</span>
+ </div>
+ <div class="progress-bar">
+ <div class="progress-fill" id="memory-progress" style="width: 0%"></div>
+ </div>
+ </div>
+ <div class="card">
+ <div class="card-title">Disk</div>
+ <div class="stats-row">
+ <span class="stats-label">Used</span>
+ <span class="stats-value" id="disk-used">-</span>
+ </div>
+ <div class="stats-row">
+ <span class="stats-label">Total</span>
+ <span class="stats-value" id="disk-total">-</span>
+ </div>
+ <div class="progress-bar">
+ <div class="progress-fill" id="disk-progress" style="width: 0%"></div>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <script>
+ // Configuration
+ const BASE_URL = window.location.origin;
+ const REFRESH_INTERVAL = 2000; // 2 seconds
+ const MAX_HISTORY_POINTS = 60; // Show last 2 minutes
+
+ // Data storage
+ const history = {
+ timestamps: [],
+ pending: [],
+ running: [],
+ completed: [],
+ cpu: []
+ };
+
+ // CPU sparkline chart
+ const cpuCtx = document.getElementById('cpu-chart').getContext('2d');
+ const cpuChart = new Chart(cpuCtx, {
+ type: 'line',
+ data: {
+ labels: [],
+ datasets: [{
+ data: [],
+ borderColor: '#58a6ff',
+ backgroundColor: 'rgba(88, 166, 255, 0.15)',
+ borderWidth: 1.5,
+ tension: 0.4,
+ fill: true,
+ pointRadius: 0
+ }]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ animation: false,
+ plugins: { legend: { display: false }, tooltip: { enabled: false } },
+ scales: {
+ x: { display: false },
+ y: { display: false, min: 0, max: 100 }
+ }
+ }
+ });
+
+ // Queue chart setup
+ const ctx = document.getElementById('queue-chart').getContext('2d');
+ const chart = new Chart(ctx, {
+ type: 'line',
+ data: {
+ labels: [],
+ datasets: [
+ {
+ label: 'Pending',
+ data: [],
+ borderColor: '#f0883e',
+ backgroundColor: 'rgba(240, 136, 62, 0.1)',
+ tension: 0.4,
+ fill: true
+ },
+ {
+ label: 'Running',
+ data: [],
+ borderColor: '#58a6ff',
+ backgroundColor: 'rgba(88, 166, 255, 0.1)',
+ tension: 0.4,
+ fill: true
+ },
+ {
+ label: 'Completed',
+ data: [],
+ borderColor: '#3fb950',
+ backgroundColor: 'rgba(63, 185, 80, 0.1)',
+ tension: 0.4,
+ fill: true
+ }
+ ]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: {
+ legend: {
+ display: true,
+ labels: {
+ color: '#8b949e'
+ }
+ }
+ },
+ scales: {
+ x: {
+ display: false
+ },
+ y: {
+ beginAtZero: true,
+ ticks: {
+ color: '#8b949e'
+ },
+ grid: {
+ color: '#21262d'
+ }
+ }
+ }
+ }
+ });
+
+ // Helper functions
+ function escapeHtml(text) {
+ var div = document.createElement('div');
+ div.textContent = text;
+ return div.innerHTML;
+ }
+
+ function formatBytes(bytes) {
+ if (bytes === 0) return '0 B';
+ const k = 1024;
+ const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
+ const i = Math.floor(Math.log(bytes) / Math.log(k));
+ return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+ }
+
+ function formatRate(rate) {
+ return rate.toFixed(2) + '/s';
+ }
+
+ function showError(message) {
+ const container = document.getElementById('error-container');
+ container.innerHTML = `<div class="error">Error: ${escapeHtml(message)}</div>`;
+ }
+
+ function clearError() {
+ document.getElementById('error-container').innerHTML = '';
+ }
+
+ function updateTimestamp() {
+ const now = new Date();
+ document.getElementById('last-update').textContent = now.toLocaleTimeString();
+ }
+
+ // Fetch functions
+ async function fetchJSON(endpoint) {
+ const response = await fetch(`${BASE_URL}${endpoint}`, {
+ headers: {
+ 'Accept': 'application/json'
+ }
+ });
+ if (!response.ok) {
+ throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+ }
+ return await response.json();
+ }
+
+ async function fetchHealth() {
+ try {
+ const response = await fetch(`${BASE_URL}/compute/ready`);
+ const isHealthy = response.status === 200;
+
+ const banner = document.querySelector('zen-banner');
+
+ if (isHealthy) {
+ banner.setAttribute('cluster-status', 'nominal');
+ banner.setAttribute('load', '0');
+ } else {
+ banner.setAttribute('cluster-status', 'degraded');
+ banner.setAttribute('load', '0');
+ }
+
+ return isHealthy;
+ } catch (error) {
+ const banner = document.querySelector('zen-banner');
+ banner.setAttribute('cluster-status', 'degraded');
+ banner.setAttribute('load', '0');
+ throw error;
+ }
+ }
+
+ async function fetchStats() {
+ const data = await fetchJSON('/stats/compute');
+
+ // Update action counts
+ document.getElementById('actions-pending').textContent = data.actions_pending || 0;
+ document.getElementById('actions-running').textContent = data.actions_submitted || 0;
+ document.getElementById('actions-complete').textContent = data.actions_complete || 0;
+
+ // Update completion rates
+ if (data.actions_retired) {
+ document.getElementById('rate-1').textContent = formatRate(data.actions_retired.rate_1 || 0);
+ document.getElementById('rate-5').textContent = formatRate(data.actions_retired.rate_5 || 0);
+ document.getElementById('rate-15').textContent = formatRate(data.actions_retired.rate_15 || 0);
+ document.getElementById('retired-count').textContent = data.actions_retired.count || 0;
+ document.getElementById('rate-mean').textContent = formatRate(data.actions_retired.rate_mean || 0);
+ }
+
+ // Update chart
+ const now = new Date().toLocaleTimeString();
+ history.timestamps.push(now);
+ history.pending.push(data.actions_pending || 0);
+ history.running.push(data.actions_submitted || 0);
+ history.completed.push(data.actions_complete || 0);
+
+ // Keep only last N points
+ if (history.timestamps.length > MAX_HISTORY_POINTS) {
+ history.timestamps.shift();
+ history.pending.shift();
+ history.running.shift();
+ history.completed.shift();
+ }
+
+ chart.data.labels = history.timestamps;
+ chart.data.datasets[0].data = history.pending;
+ chart.data.datasets[1].data = history.running;
+ chart.data.datasets[2].data = history.completed;
+ chart.update('none');
+ }
+
+ async function fetchSysInfo() {
+ const data = await fetchJSON('/compute/sysinfo');
+
+ // Update CPU
+ const cpuUsage = data.cpu_usage || 0;
+ document.getElementById('cpu-usage').textContent = cpuUsage.toFixed(1) + '%';
+ document.getElementById('cpu-progress').style.width = cpuUsage + '%';
+
+ const banner = document.querySelector('zen-banner');
+ banner.setAttribute('load', cpuUsage.toFixed(1));
+
+ history.cpu.push(cpuUsage);
+ if (history.cpu.length > MAX_HISTORY_POINTS) history.cpu.shift();
+ cpuChart.data.labels = history.cpu.map(() => '');
+ cpuChart.data.datasets[0].data = history.cpu;
+ cpuChart.update('none');
+
+ document.getElementById('cpu-packages').textContent = data.cpu_count ?? '-';
+ document.getElementById('cpu-cores').textContent = data.core_count ?? '-';
+ document.getElementById('cpu-lp').textContent = data.lp_count ?? '-';
+
+ // Update Memory
+ const memUsed = data.memory_used || 0;
+ const memTotal = data.memory_total || 1;
+ const memPercent = (memUsed / memTotal) * 100;
+ document.getElementById('memory-used').textContent = formatBytes(memUsed);
+ document.getElementById('memory-total').textContent = formatBytes(memTotal);
+ document.getElementById('memory-progress').style.width = memPercent + '%';
+
+ // Update Disk
+ const diskUsed = data.disk_used || 0;
+ const diskTotal = data.disk_total || 1;
+ const diskPercent = (diskUsed / diskTotal) * 100;
+ document.getElementById('disk-used').textContent = formatBytes(diskUsed);
+ document.getElementById('disk-total').textContent = formatBytes(diskTotal);
+ document.getElementById('disk-progress').style.width = diskPercent + '%';
+ }
+
+ // Persists the selected worker ID across refreshes
+ let selectedWorkerId = null;
+
+ function renderWorkerDetail(id, desc) {
+ const panel = document.getElementById('worker-detail');
+
+ if (!desc) {
+ panel.style.display = 'none';
+ return;
+ }
+
+ function field(label, value) {
+ return `<tr><td>${label}</td><td>${value ?? '-'}</td></tr>`;
+ }
+
+ function monoField(label, value) {
+ return `<tr><td>${label}</td><td class="detail-mono">${value ?? '-'}</td></tr>`;
+ }
+
+ // Functions
+ const functions = desc.functions || [];
+ const functionsHtml = functions.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
+ `<table class="detail-table">${functions.map(f =>
+ `<tr><td>${escapeHtml(f.name || '-')}</td><td class="detail-mono">${escapeHtml(f.version || '-')}</td></tr>`
+ ).join('')}</table>`;
+
+ // Executables
+ const executables = desc.executables || [];
+ const totalExecSize = executables.reduce((sum, e) => sum + (e.size || 0), 0);
+ const execHtml = executables.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
+ `<table class="detail-table">
+ <tr style="font-size:11px;">
+ <td style="color:var(--theme_faint);padding-bottom:4px;">Path</td>
+ <td style="color:var(--theme_faint);padding-bottom:4px;">Hash</td>
+ <td style="color:var(--theme_faint);padding-bottom:4px;text-align:right;">Size</td>
+ </tr>
+ ${executables.map(e =>
+ `<tr>
+ <td>${escapeHtml(e.name || '-')}</td>
+ <td class="detail-mono">${escapeHtml(e.hash || '-')}</td>
+ <td style="text-align:right;white-space:nowrap;">${e.size != null ? formatBytes(e.size) : '-'}</td>
+ </tr>`
+ ).join('')}
+ <tr style="border-top:1px solid var(--theme_g2);">
+ <td style="color:var(--theme_g1);padding-top:6px;">Total</td>
+ <td></td>
+ <td style="text-align:right;white-space:nowrap;padding-top:6px;color:var(--theme_bright);font-weight:600;">${formatBytes(totalExecSize)}</td>
+ </tr>
+ </table>`;
+
+ // Files
+ const files = desc.files || [];
+ const filesHtml = files.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
+ `<table class="detail-table">${files.map(f =>
+ `<tr><td>${escapeHtml(f.name || f)}</td><td class="detail-mono">${escapeHtml(f.hash || '')}</td></tr>`
+ ).join('')}</table>`;
+
+ // Dirs
+ const dirs = desc.dirs || [];
+ const dirsHtml = dirs.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
+ dirs.map(d => `<span class="detail-tag">${escapeHtml(d)}</span>`).join('');
+
+ // Environment
+ const env = desc.environment || [];
+ const envHtml = env.length === 0 ? '<span style="color:var(--theme_faint);font-size:12px;">none</span>' :
+ env.map(e => `<span class="detail-tag">${escapeHtml(e)}</span>`).join('');
+
+ panel.innerHTML = `
+ <div class="worker-detail-title">${escapeHtml(desc.name || id)}</div>
+ <div class="detail-section">
+ <table class="detail-table">
+ ${field('Worker ID', `<span class="detail-mono">${escapeHtml(id)}</span>`)}
+ ${field('Path', escapeHtml(desc.path || '-'))}
+ ${field('Platform', escapeHtml(desc.host || '-'))}
+ ${monoField('Build System', desc.buildsystem_version)}
+ ${field('Cores', desc.cores)}
+ ${field('Timeout', desc.timeout != null ? desc.timeout + 's' : null)}
+ </table>
+ </div>
+ <div class="detail-section">
+ <div class="detail-section-label">Functions</div>
+ ${functionsHtml}
+ </div>
+ <div class="detail-section">
+ <div class="detail-section-label">Executables</div>
+ ${execHtml}
+ </div>
+ <div class="detail-section">
+ <div class="detail-section-label">Files</div>
+ ${filesHtml}
+ </div>
+ <div class="detail-section">
+ <div class="detail-section-label">Directories</div>
+ ${dirsHtml}
+ </div>
+ <div class="detail-section">
+ <div class="detail-section-label">Environment</div>
+ ${envHtml}
+ </div>
+ `;
+ panel.style.display = 'block';
+ }
+
+ async function fetchWorkers() {
+ const data = await fetchJSON('/compute/workers');
+ const workerIds = data.workers || [];
+
+ document.getElementById('worker-count').textContent = workerIds.length;
+
+ const container = document.getElementById('worker-table-container');
+ const tbody = document.getElementById('worker-table-body');
+
+ if (workerIds.length === 0) {
+ container.style.display = 'none';
+ selectedWorkerId = null;
+ return;
+ }
+
+ const descriptors = await Promise.all(
+ workerIds.map(id => fetchJSON(`/compute/workers/${id}`).catch(() => null))
+ );
+
+ // Build a map for quick lookup by ID
+ const descriptorMap = {};
+ workerIds.forEach((id, i) => { descriptorMap[id] = descriptors[i]; });
+
+ tbody.innerHTML = '';
+ descriptors.forEach((desc, i) => {
+ const id = workerIds[i];
+ const name = desc ? (desc.name || '-') : '-';
+ const host = desc ? (desc.host || '-') : '-';
+ const cores = desc ? (desc.cores != null ? desc.cores : '-') : '-';
+ const timeout = desc ? (desc.timeout != null ? desc.timeout + 's' : '-') : '-';
+ const functions = desc ? (desc.functions ? desc.functions.length : 0) : '-';
+
+ const tr = document.createElement('tr');
+ tr.className = 'worker-row' + (id === selectedWorkerId ? ' selected' : '');
+ tr.dataset.workerId = id;
+ tr.innerHTML = `
+ <td style="color: var(--theme_bright);">${escapeHtml(name)}</td>
+ <td>${escapeHtml(host)}</td>
+ <td style="text-align: right;">${escapeHtml(String(cores))}</td>
+ <td style="text-align: right;">${escapeHtml(String(timeout))}</td>
+ <td style="text-align: right;">${escapeHtml(String(functions))}</td>
+ <td style="color: var(--theme_g1); font-family: monospace; font-size: 11px;">${escapeHtml(id)}</td>
+ `;
+ tr.addEventListener('click', () => {
+ document.querySelectorAll('.worker-row').forEach(r => r.classList.remove('selected'));
+ if (selectedWorkerId === id) {
+ // Toggle off
+ selectedWorkerId = null;
+ document.getElementById('worker-detail').style.display = 'none';
+ } else {
+ selectedWorkerId = id;
+ tr.classList.add('selected');
+ renderWorkerDetail(id, descriptorMap[id]);
+ }
+ });
+ tbody.appendChild(tr);
+ });
+
+ // Re-render detail if selected worker is still present
+ if (selectedWorkerId && descriptorMap[selectedWorkerId]) {
+ renderWorkerDetail(selectedWorkerId, descriptorMap[selectedWorkerId]);
+ } else if (selectedWorkerId && !descriptorMap[selectedWorkerId]) {
+ selectedWorkerId = null;
+ document.getElementById('worker-detail').style.display = 'none';
+ }
+
+ container.style.display = 'block';
+ }
+
+ // Windows FILETIME: 100ns ticks since 1601-01-01. Convert to JS Date.
+ const FILETIME_EPOCH_OFFSET_MS = 11644473600000n;
+ function filetimeToDate(ticks) {
+ if (!ticks) return null;
+ const ms = BigInt(ticks) / 10000n - FILETIME_EPOCH_OFFSET_MS;
+ return new Date(Number(ms));
+ }
+
+ function formatTime(date) {
+ if (!date) return '-';
+ return date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' });
+ }
+
+ function formatDuration(startDate, endDate) {
+ if (!startDate || !endDate) return '-';
+ const ms = endDate - startDate;
+ if (ms < 0) return '-';
+ if (ms < 1000) return ms + ' ms';
+ if (ms < 60000) return (ms / 1000).toFixed(2) + ' s';
+ const m = Math.floor(ms / 60000);
+ const s = ((ms % 60000) / 1000).toFixed(0).padStart(2, '0');
+ return `${m}m ${s}s`;
+ }
+
+ async function fetchQueues() {
+ const data = await fetchJSON('/compute/queues');
+ const queues = data.queues || [];
+
+ const empty = document.getElementById('queue-list-empty');
+ const container = document.getElementById('queue-list-container');
+ const tbody = document.getElementById('queue-list-body');
+
+ if (queues.length === 0) {
+ empty.style.display = '';
+ container.style.display = 'none';
+ return;
+ }
+
+ empty.style.display = 'none';
+ tbody.innerHTML = '';
+
+ for (const q of queues) {
+ const id = q.queue_id ?? '-';
+ const badge = q.state === 'cancelled'
+ ? '<span class="status-badge failure">cancelled</span>'
+ : q.state === 'draining'
+ ? '<span class="status-badge" style="background:color-mix(in srgb, var(--theme_warn) 15%, transparent);color:var(--theme_warn);">draining</span>'
+ : q.is_complete
+ ? '<span class="status-badge success">complete</span>'
+ : '<span class="status-badge" style="background:color-mix(in srgb, var(--theme_p0) 15%, transparent);color:var(--theme_p0);">active</span>';
+ const token = q.queue_token
+ ? `<span class="detail-mono">${escapeHtml(q.queue_token)}</span>`
+ : '<span style="color:var(--theme_faint);">-</span>';
+
+ const tr = document.createElement('tr');
+ tr.innerHTML = `
+ <td style="text-align: right; font-family: monospace; color: var(--theme_bright);">${escapeHtml(String(id))}</td>
+ <td style="text-align: center;">${badge}</td>
+ <td style="text-align: right;">${q.active_count ?? 0}</td>
+ <td style="text-align: right; color: var(--theme_ok);">${q.completed_count ?? 0}</td>
+ <td style="text-align: right; color: var(--theme_fail);">${q.failed_count ?? 0}</td>
+ <td style="text-align: right; color: var(--theme_warn);">${q.abandoned_count ?? 0}</td>
+ <td style="text-align: right; color: var(--theme_warn);">${q.cancelled_count ?? 0}</td>
+ <td>${token}</td>
+ `;
+ tbody.appendChild(tr);
+ }
+
+ container.style.display = 'block';
+ }
+
+ async function fetchActionHistory() {
+ const data = await fetchJSON('/compute/jobs/history?limit=50');
+ const entries = data.history || [];
+
+ const empty = document.getElementById('action-history-empty');
+ const container = document.getElementById('action-history-container');
+ const tbody = document.getElementById('action-history-body');
+
+ if (entries.length === 0) {
+ empty.style.display = '';
+ container.style.display = 'none';
+ return;
+ }
+
+ empty.style.display = 'none';
+ tbody.innerHTML = '';
+
+ // Entries arrive oldest-first; reverse to show newest at top
+ for (const entry of [...entries].reverse()) {
+ const lsn = entry.lsn ?? '-';
+ const succeeded = entry.succeeded;
+ const badge = succeeded == null
+ ? '<span class="status-badge" style="background:var(--theme_border_subtle);color:var(--theme_g1);">unknown</span>'
+ : succeeded
+ ? '<span class="status-badge success">ok</span>'
+ : '<span class="status-badge failure">failed</span>';
+ const desc = entry.actionDescriptor || {};
+ const fn = desc.Function || '-';
+ const workerId = entry.workerId || '-';
+ const actionId = entry.actionId || '-';
+
+ const startDate = filetimeToDate(entry.time_Running);
+ const endDate = filetimeToDate(entry.time_Completed ?? entry.time_Failed);
+
+ const queueId = entry.queueId || 0;
+ const queueCell = queueId
+ ? `<a href="/compute/queues/${queueId}" style="color: var(--theme_ln); text-decoration: none; font-family: monospace;">${escapeHtml(String(queueId))}</a>`
+ : '<span style="color: var(--theme_faint);">-</span>';
+
+ const tr = document.createElement('tr');
+ tr.innerHTML = `
+ <td style="text-align: right; font-family: monospace; color: var(--theme_g1);">${escapeHtml(String(lsn))}</td>
+ <td style="text-align: right;">${queueCell}</td>
+ <td style="text-align: center;">${badge}</td>
+ <td style="color: var(--theme_bright);">${escapeHtml(fn)}</td>
+ <td style="text-align: right; font-size: 12px; white-space: nowrap; color: var(--theme_g1);">${formatTime(startDate)}</td>
+ <td style="text-align: right; font-size: 12px; white-space: nowrap; color: var(--theme_g1);">${formatTime(endDate)}</td>
+ <td style="text-align: right; font-size: 12px; white-space: nowrap;">${formatDuration(startDate, endDate)}</td>
+ <td style="font-family: monospace; font-size: 11px; color: var(--theme_g1);">${escapeHtml(workerId)}</td>
+ <td style="font-family: monospace; font-size: 11px; color: var(--theme_g1);">${escapeHtml(actionId)}</td>
+ `;
+ tbody.appendChild(tr);
+ }
+
+ container.style.display = 'block';
+ }
+
+ async function updateDashboard() {
+ try {
+ await Promise.all([
+ fetchHealth(),
+ fetchStats(),
+ fetchSysInfo(),
+ fetchWorkers(),
+ fetchQueues(),
+ fetchActionHistory()
+ ]);
+
+ clearError();
+ updateTimestamp();
+ } catch (error) {
+ console.error('Error updating dashboard:', error);
+ showError(error.message);
+ }
+ }
+
+ // Start updating
+ updateDashboard();
+ setInterval(updateDashboard, REFRESH_INTERVAL);
+ </script>
+</body>
+</html>
diff --git a/src/zenserver/frontend/html/compute/hub.html b/src/zenserver/frontend/html/compute/hub.html
new file mode 100644
index 000000000..32e1b05db
--- /dev/null
+++ b/src/zenserver/frontend/html/compute/hub.html
@@ -0,0 +1,170 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <link rel="stylesheet" type="text/css" href="../zen.css" />
+ <script src="../theme.js"></script>
+ <script src="../banner.js" defer></script>
+ <script src="../nav.js" defer></script>
+ <title>Zen Hub Dashboard</title>
+</head>
+<body>
+ <div class="container" style="max-width: 1400px; margin: 0 auto;">
+ <zen-banner cluster-status="nominal" subtitle="HUB" tagline="Overview" logo-src="../favicon.ico"></zen-banner>
+ <zen-nav>
+ <a href="/dashboard/">Home</a>
+ <a href="hub.html">Hub</a>
+ </zen-nav>
+ <div class="timestamp">Last updated: <span id="last-update">Never</span></div>
+
+ <div id="error-container"></div>
+
+ <div class="section-title">Capacity</div>
+ <div class="grid">
+ <div class="card">
+ <div class="card-title">Active Modules</div>
+ <div class="metric-value" id="instance-count">-</div>
+ <div class="metric-label">Currently provisioned</div>
+ </div>
+ <div class="card">
+ <div class="card-title">Peak Modules</div>
+ <div class="metric-value" id="max-instance-count">-</div>
+ <div class="metric-label">High watermark</div>
+ </div>
+ <div class="card">
+ <div class="card-title">Instance Limit</div>
+ <div class="metric-value" id="instance-limit">-</div>
+ <div class="metric-label">Maximum allowed</div>
+ <div class="progress-bar">
+ <div class="progress-fill" id="capacity-progress" style="width: 0%"></div>
+ </div>
+ </div>
+ </div>
+
+ <div class="section-title">Modules</div>
+ <div class="card">
+ <div class="card-title">Storage Server Instances</div>
+ <div id="empty-state" class="empty-state">No modules provisioned.</div>
+ <table id="module-table" style="display: none;">
+ <thead>
+ <tr>
+ <th>Module ID</th>
+ <th style="text-align: center;">Status</th>
+ </tr>
+ </thead>
+ <tbody id="module-table-body"></tbody>
+ </table>
+ </div>
+ </div>
+
+ <script>
+ const BASE_URL = window.location.origin;
+ const REFRESH_INTERVAL = 2000;
+
+ function escapeHtml(text) {
+ var div = document.createElement('div');
+ div.textContent = text;
+ return div.innerHTML;
+ }
+
+ function showError(message) {
+ document.getElementById('error-container').innerHTML =
+ '<div class="error">Error: ' + escapeHtml(message) + '</div>';
+ }
+
+ function clearError() {
+ document.getElementById('error-container').innerHTML = '';
+ }
+
+ async function fetchJSON(endpoint) {
+ var response = await fetch(BASE_URL + endpoint, {
+ headers: { 'Accept': 'application/json' }
+ });
+ if (!response.ok) {
+ throw new Error('HTTP ' + response.status + ': ' + response.statusText);
+ }
+ return await response.json();
+ }
+
+ async function fetchStats() {
+ var data = await fetchJSON('/hub/stats');
+
+ var current = data.currentInstanceCount || 0;
+ var max = data.maxInstanceCount || 0;
+ var limit = data.instanceLimit || 0;
+
+ document.getElementById('instance-count').textContent = current;
+ document.getElementById('max-instance-count').textContent = max;
+ document.getElementById('instance-limit').textContent = limit;
+
+ var pct = limit > 0 ? (current / limit) * 100 : 0;
+ document.getElementById('capacity-progress').style.width = pct + '%';
+
+ var banner = document.querySelector('zen-banner');
+ if (current === 0) {
+ banner.setAttribute('cluster-status', 'nominal');
+ } else if (limit > 0 && current >= limit * 0.9) {
+ banner.setAttribute('cluster-status', 'degraded');
+ } else {
+ banner.setAttribute('cluster-status', 'nominal');
+ }
+ }
+
+ async function fetchModules() {
+ var data = await fetchJSON('/hub/status');
+ var modules = data.modules || [];
+
+ var emptyState = document.getElementById('empty-state');
+ var table = document.getElementById('module-table');
+ var tbody = document.getElementById('module-table-body');
+
+ if (modules.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ return;
+ }
+
+ emptyState.style.display = 'none';
+ table.style.display = '';
+
+ tbody.innerHTML = '';
+ for (var i = 0; i < modules.length; i++) {
+ var m = modules[i];
+ var moduleId = m.moduleId || '';
+ var provisioned = m.provisioned;
+
+ var badge = provisioned
+ ? '<span class="status-badge active">Provisioned</span>'
+ : '<span class="status-badge inactive">Inactive</span>';
+
+ var tr = document.createElement('tr');
+ tr.innerHTML =
+ '<td style="font-family: monospace; font-size: 12px;">' + escapeHtml(moduleId) + '</td>' +
+ '<td style="text-align: center;">' + badge + '</td>';
+ tbody.appendChild(tr);
+ }
+ }
+
+ async function updateDashboard() {
+ var banner = document.querySelector('zen-banner');
+ try {
+ await Promise.all([
+ fetchStats(),
+ fetchModules()
+ ]);
+
+ clearError();
+ document.getElementById('last-update').textContent = new Date().toLocaleTimeString();
+ } catch (error) {
+ console.error('Error updating dashboard:', error);
+ showError(error.message);
+ banner.setAttribute('cluster-status', 'offline');
+ }
+ }
+
+ updateDashboard();
+ setInterval(updateDashboard, REFRESH_INTERVAL);
+ </script>
+</body>
+</html>
diff --git a/src/zenserver/frontend/html/compute/index.html b/src/zenserver/frontend/html/compute/index.html
new file mode 100644
index 000000000..9597fd7f3
--- /dev/null
+++ b/src/zenserver/frontend/html/compute/index.html
@@ -0,0 +1 @@
+<meta http-equiv="refresh" content="0; url=compute.html" /> \ No newline at end of file
diff --git a/src/zenserver/frontend/html/compute/orchestrator.html b/src/zenserver/frontend/html/compute/orchestrator.html
new file mode 100644
index 000000000..a519dee18
--- /dev/null
+++ b/src/zenserver/frontend/html/compute/orchestrator.html
@@ -0,0 +1,674 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <link rel="stylesheet" type="text/css" href="../zen.css" />
+ <script src="../theme.js"></script>
+ <script src="../banner.js" defer></script>
+ <script src="../nav.js" defer></script>
+ <title>Zen Orchestrator Dashboard</title>
+ <style>
+ .agent-count {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ font-size: 14px;
+ padding: 8px 16px;
+ border-radius: 6px;
+ background: var(--theme_g3);
+ border: 1px solid var(--theme_g2);
+ }
+
+ .agent-count .count {
+ font-size: 20px;
+ font-weight: 600;
+ color: var(--theme_bright);
+ }
+ </style>
+</head>
+<body>
+ <div class="container" style="max-width: 1400px; margin: 0 auto;">
+ <zen-banner cluster-status="nominal" load="0" logo-src="../favicon.ico"></zen-banner>
+ <zen-nav>
+ <a href="/dashboard/">Home</a>
+ <a href="compute.html">Node</a>
+ <a href="orchestrator.html">Orchestrator</a>
+ </zen-nav>
+ <div class="header">
+ <div>
+ <div class="timestamp">Last updated: <span id="last-update">Never</span></div>
+ </div>
+ <div class="agent-count">
+ <span>Agents:</span>
+ <span class="count" id="agent-count">-</span>
+ </div>
+ </div>
+
+ <div id="error-container"></div>
+
+ <div class="card">
+ <div class="card-title">Compute Agents</div>
+ <div id="empty-state" class="empty-state">No agents registered.</div>
+ <table id="agent-table" style="display: none;">
+ <thead>
+ <tr>
+ <th style="width: 40px; text-align: center;">Health</th>
+ <th>Hostname</th>
+ <th style="text-align: right;">CPUs</th>
+ <th style="text-align: right;">CPU Usage</th>
+ <th style="text-align: right;">Memory</th>
+ <th style="text-align: right;">Queues</th>
+ <th style="text-align: right;">Pending</th>
+ <th style="text-align: right;">Running</th>
+ <th style="text-align: right;">Completed</th>
+ <th style="text-align: right;">Traffic</th>
+ <th style="text-align: right;">Last Seen</th>
+ </tr>
+ </thead>
+ <tbody id="agent-table-body"></tbody>
+ </table>
+ </div>
+ <div class="card" style="margin-top: 20px;">
+ <div class="card-title">Connected Clients</div>
+ <div id="clients-empty" class="empty-state">No clients connected.</div>
+ <table id="clients-table" style="display: none;">
+ <thead>
+ <tr>
+ <th style="width: 40px; text-align: center;">Health</th>
+ <th>Client ID</th>
+ <th>Hostname</th>
+ <th>Address</th>
+ <th style="text-align: right;">Last Seen</th>
+ </tr>
+ </thead>
+ <tbody id="clients-table-body"></tbody>
+ </table>
+ </div>
+ <div class="card" style="margin-top: 20px;">
+ <div style="display: flex; align-items: center; gap: 12px; margin-bottom: 12px;">
+ <div class="card-title" style="margin-bottom: 0;">Event History</div>
+ <div class="history-tabs">
+ <button class="history-tab active" data-tab="workers" onclick="switchHistoryTab('workers')">Workers</button>
+ <button class="history-tab" data-tab="clients" onclick="switchHistoryTab('clients')">Clients</button>
+ </div>
+ </div>
+ <div id="history-panel-workers">
+ <div id="history-empty" class="empty-state">No provisioning events recorded.</div>
+ <table id="history-table" style="display: none;">
+ <thead>
+ <tr>
+ <th>Time</th>
+ <th>Event</th>
+ <th>Worker</th>
+ <th>Hostname</th>
+ </tr>
+ </thead>
+ <tbody id="history-table-body"></tbody>
+ </table>
+ </div>
+ <div id="history-panel-clients" style="display: none;">
+ <div id="client-history-empty" class="empty-state">No client events recorded.</div>
+ <table id="client-history-table" style="display: none;">
+ <thead>
+ <tr>
+ <th>Time</th>
+ <th>Event</th>
+ <th>Client</th>
+ <th>Hostname</th>
+ </tr>
+ </thead>
+ <tbody id="client-history-table-body"></tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+
+ <script>
+ const BASE_URL = window.location.origin;
+ const REFRESH_INTERVAL = 2000;
+
+ function escapeHtml(text) {
+ var div = document.createElement('div');
+ div.textContent = text;
+ return div.innerHTML;
+ }
+
+ function showError(message) {
+ document.getElementById('error-container').innerHTML =
+ '<div class="error">Error: ' + escapeHtml(message) + '</div>';
+ }
+
+ function clearError() {
+ document.getElementById('error-container').innerHTML = '';
+ }
+
+ function formatLastSeen(dtMs) {
+ if (dtMs == null) return '-';
+ var seconds = Math.floor(dtMs / 1000);
+ if (seconds < 60) return seconds + 's ago';
+ var minutes = Math.floor(seconds / 60);
+ if (minutes < 60) return minutes + 'm ' + (seconds % 60) + 's ago';
+ var hours = Math.floor(minutes / 60);
+ return hours + 'h ' + (minutes % 60) + 'm ago';
+ }
+
+ function healthClass(dtMs, reachable) {
+ if (reachable === false) return 'health-red';
+ if (dtMs == null) return 'health-red';
+ var seconds = dtMs / 1000;
+ if (seconds < 30 && reachable === true) return 'health-green';
+ if (seconds < 120) return 'health-yellow';
+ return 'health-red';
+ }
+
+ function healthTitle(dtMs, reachable) {
+ var seenStr = dtMs != null ? 'Last seen ' + formatLastSeen(dtMs) : 'Never seen';
+ if (reachable === true) return seenStr + ' · Reachable';
+ if (reachable === false) return seenStr + ' · Unreachable';
+ return seenStr + ' · Reachability unknown';
+ }
+
+ function formatCpuUsage(percent) {
+ if (percent == null || percent === 0) return '-';
+ return percent.toFixed(1) + '%';
+ }
+
+ function formatMemory(usedBytes, totalBytes) {
+ if (!totalBytes) return '-';
+ var usedGiB = usedBytes / (1024 * 1024 * 1024);
+ var totalGiB = totalBytes / (1024 * 1024 * 1024);
+ return usedGiB.toFixed(1) + ' / ' + totalGiB.toFixed(1) + ' GiB';
+ }
+
+ function formatBytes(bytes) {
+ if (!bytes) return '-';
+ if (bytes < 1024) return bytes + ' B';
+ if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KiB';
+ if (bytes < 1024 * 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(1) + ' MiB';
+ if (bytes < 1024 * 1024 * 1024 * 1024) return (bytes / (1024 * 1024 * 1024)).toFixed(1) + ' GiB';
+ return (bytes / (1024 * 1024 * 1024 * 1024)).toFixed(1) + ' TiB';
+ }
+
+ function formatTraffic(recv, sent) {
+ if (!recv && !sent) return '-';
+ return formatBytes(recv) + ' / ' + formatBytes(sent);
+ }
+
+ function parseIpFromUri(uri) {
+ try {
+ var url = new URL(uri);
+ var host = url.hostname;
+ // Strip IPv6 brackets
+ if (host.startsWith('[') && host.endsWith(']')) host = host.slice(1, -1);
+ // Only handle IPv4
+ var parts = host.split('.');
+ if (parts.length !== 4) return null;
+ var octets = parts.map(Number);
+ if (octets.some(function(o) { return isNaN(o) || o < 0 || o > 255; })) return null;
+ return octets;
+ } catch (e) {
+ return null;
+ }
+ }
+
+ function computeCidr(ips) {
+ if (ips.length === 0) return null;
+ if (ips.length === 1) return ips[0].join('.') + '/32';
+
+ // Convert each IP to a 32-bit integer
+ var ints = ips.map(function(o) {
+ return ((o[0] << 24) | (o[1] << 16) | (o[2] << 8) | o[3]) >>> 0;
+ });
+
+ // Find common prefix length by ANDing all identical high bits
+ var common = ~0 >>> 0;
+ for (var i = 1; i < ints.length; i++) {
+ // XOR to find differing bits, then mask away everything from the first difference down
+ var diff = (ints[0] ^ ints[i]) >>> 0;
+ if (diff !== 0) {
+ var bit = 31 - Math.floor(Math.log2(diff));
+ var mask = bit > 0 ? ((~0 << (32 - bit)) >>> 0) : 0;
+ common = (common & mask) >>> 0;
+ }
+ }
+
+ // Count leading ones in the common mask
+ var prefix = 0;
+ for (var b = 31; b >= 0; b--) {
+ if ((common >>> b) & 1) prefix++;
+ else break;
+ }
+
+ // Network address
+ var net = (ints[0] & common) >>> 0;
+ var a = (net >>> 24) & 0xff;
+ var bv = (net >>> 16) & 0xff;
+ var c = (net >>> 8) & 0xff;
+ var d = net & 0xff;
+ return a + '.' + bv + '.' + c + '.' + d + '/' + prefix;
+ }
+
+ function renderDashboard(data) {
+ var banner = document.querySelector('zen-banner');
+ if (data.hostname) {
+ banner.setAttribute('tagline', 'Orchestrator \u2014 ' + data.hostname);
+ }
+ var workers = data.workers || [];
+
+ document.getElementById('agent-count').textContent = workers.length;
+
+ if (workers.length === 0) {
+ banner.setAttribute('cluster-status', 'degraded');
+ banner.setAttribute('load', '0');
+ } else {
+ banner.setAttribute('cluster-status', 'nominal');
+ }
+
+ var emptyState = document.getElementById('empty-state');
+ var table = document.getElementById('agent-table');
+ var tbody = document.getElementById('agent-table-body');
+
+ if (workers.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ } else {
+ emptyState.style.display = 'none';
+ table.style.display = '';
+
+ tbody.innerHTML = '';
+ var totalCpus = 0;
+ var totalWeightedCpuUsage = 0;
+ var totalMemUsed = 0;
+ var totalMemTotal = 0;
+ var totalQueues = 0;
+ var totalPending = 0;
+ var totalRunning = 0;
+ var totalCompleted = 0;
+ var totalBytesRecv = 0;
+ var totalBytesSent = 0;
+ var allIps = [];
+ for (var i = 0; i < workers.length; i++) {
+ var w = workers[i];
+ var uri = w.uri || '';
+ var dt = w.dt;
+ var dashboardUrl = uri + '/dashboard/compute/';
+
+ var id = w.id || '';
+
+ var hostname = w.hostname || '';
+ var cpus = w.cpus || 0;
+ totalCpus += cpus;
+ if (cpus > 0 && typeof w.cpu_usage === 'number') {
+ totalWeightedCpuUsage += w.cpu_usage * cpus;
+ }
+
+ var memTotal = w.memory_total || 0;
+ var memUsed = w.memory_used || 0;
+ totalMemTotal += memTotal;
+ totalMemUsed += memUsed;
+
+ var activeQueues = w.active_queues || 0;
+ totalQueues += activeQueues;
+
+ var actionsPending = w.actions_pending || 0;
+ var actionsRunning = w.actions_running || 0;
+ var actionsCompleted = w.actions_completed || 0;
+ totalPending += actionsPending;
+ totalRunning += actionsRunning;
+ totalCompleted += actionsCompleted;
+
+ var bytesRecv = w.bytes_received || 0;
+ var bytesSent = w.bytes_sent || 0;
+ totalBytesRecv += bytesRecv;
+ totalBytesSent += bytesSent;
+
+ var ip = parseIpFromUri(uri);
+ if (ip) allIps.push(ip);
+
+ var reachable = w.reachable;
+ var hClass = healthClass(dt, reachable);
+ var hTitle = healthTitle(dt, reachable);
+
+ var platform = w.platform || '';
+ var badges = '';
+ if (platform) {
+ var platColors = { windows: '#0078d4', wine: '#722f37', linux: '#e95420', macos: '#a2aaad' };
+ var platColor = platColors[platform] || '#8b949e';
+ badges += ' <span style="display:inline-block;padding:1px 6px;border-radius:10px;font-size:10px;font-weight:600;color:#fff;background:' + platColor + ';vertical-align:middle;margin-left:4px;">' + escapeHtml(platform) + '</span>';
+ }
+ var provisioner = w.provisioner || '';
+ if (provisioner) {
+ var provColors = { horde: '#8957e5', nomad: '#3fb950' };
+ var provColor = provColors[provisioner] || '#8b949e';
+ badges += ' <span style="display:inline-block;padding:1px 6px;border-radius:10px;font-size:10px;font-weight:600;color:#fff;background:' + provColor + ';vertical-align:middle;margin-left:4px;">' + escapeHtml(provisioner) + '</span>';
+ }
+
+ var tr = document.createElement('tr');
+ tr.title = id;
+ tr.innerHTML =
+ '<td style="text-align: center;"><span class="health-dot ' + hClass + '" title="' + escapeHtml(hTitle) + '"></span></td>' +
+ '<td><a href="' + escapeHtml(dashboardUrl) + '" target="_blank">' + escapeHtml(hostname) + '</a>' + badges + '</td>' +
+ '<td style="text-align: right;">' + (cpus > 0 ? cpus : '-') + '</td>' +
+ '<td style="text-align: right;">' + formatCpuUsage(w.cpu_usage) + '</td>' +
+ '<td style="text-align: right;">' + formatMemory(memUsed, memTotal) + '</td>' +
+ '<td style="text-align: right;">' + (activeQueues > 0 ? activeQueues : '-') + '</td>' +
+ '<td style="text-align: right;">' + actionsPending + '</td>' +
+ '<td style="text-align: right;">' + actionsRunning + '</td>' +
+ '<td style="text-align: right;">' + actionsCompleted + '</td>' +
+ '<td style="text-align: right; font-size: 11px; color: var(--theme_g1);">' + formatTraffic(bytesRecv, bytesSent) + '</td>' +
+ '<td style="text-align: right; color: var(--theme_g1);">' + formatLastSeen(dt) + '</td>';
+ tbody.appendChild(tr);
+ }
+
+ var clusterLoad = totalCpus > 0 ? (totalWeightedCpuUsage / totalCpus) : 0;
+ banner.setAttribute('load', clusterLoad.toFixed(1));
+
+ // Total row
+ var cidr = computeCidr(allIps);
+ var totalTr = document.createElement('tr');
+ totalTr.className = 'total-row';
+ totalTr.innerHTML =
+ '<td></td>' +
+ '<td style="text-align: right; color: var(--theme_g1); text-transform: uppercase; font-size: 11px;">Total' + (cidr ? ' <span style="font-family: monospace; font-weight: normal;">' + escapeHtml(cidr) + '</span>' : '') + '</td>' +
+ '<td style="text-align: right;">' + totalCpus + '</td>' +
+ '<td></td>' +
+ '<td style="text-align: right;">' + formatMemory(totalMemUsed, totalMemTotal) + '</td>' +
+ '<td style="text-align: right;">' + totalQueues + '</td>' +
+ '<td style="text-align: right;">' + totalPending + '</td>' +
+ '<td style="text-align: right;">' + totalRunning + '</td>' +
+ '<td style="text-align: right;">' + totalCompleted + '</td>' +
+ '<td style="text-align: right; font-size: 11px;">' + formatTraffic(totalBytesRecv, totalBytesSent) + '</td>' +
+ '<td></td>';
+ tbody.appendChild(totalTr);
+ }
+
+ clearError();
+ document.getElementById('last-update').textContent = new Date().toLocaleTimeString();
+
+ // Render provisioning history if present in WebSocket payload
+ if (data.events) {
+ renderProvisioningHistory(data.events);
+ }
+
+ // Render connected clients if present
+ if (data.clients) {
+ renderClients(data.clients);
+ }
+
+ // Render client history if present
+ if (data.client_events) {
+ renderClientHistory(data.client_events);
+ }
+ }
+
+ function eventBadge(type) {
+ var colors = { joined: 'var(--theme_ok)', left: 'var(--theme_fail)', returned: 'var(--theme_warn)' };
+ var labels = { joined: 'Joined', left: 'Left', returned: 'Returned' };
+ var color = colors[type] || 'var(--theme_g1)';
+ var label = labels[type] || type;
+ return '<span style="display:inline-block;padding:2px 8px;border-radius:4px;font-size:11px;font-weight:600;color:var(--theme_g4);background:' + color + ';">' + escapeHtml(label) + '</span>';
+ }
+
+ function formatTimestamp(ts) {
+ if (!ts) return '-';
+ // CbObject DateTime serialized as ticks (100ns since 0001-01-01) or ISO string
+ var date;
+ if (typeof ts === 'number') {
+ // .NET-style ticks: convert to Unix ms
+ var unixMs = (ts - 621355968000000000) / 10000;
+ date = new Date(unixMs);
+ } else {
+ date = new Date(ts);
+ }
+ if (isNaN(date.getTime())) return '-';
+ return date.toLocaleTimeString();
+ }
+
+ var activeHistoryTab = 'workers';
+
+ function switchHistoryTab(tab) {
+ activeHistoryTab = tab;
+ var tabs = document.querySelectorAll('.history-tab');
+ for (var i = 0; i < tabs.length; i++) {
+ tabs[i].classList.toggle('active', tabs[i].getAttribute('data-tab') === tab);
+ }
+ document.getElementById('history-panel-workers').style.display = tab === 'workers' ? '' : 'none';
+ document.getElementById('history-panel-clients').style.display = tab === 'clients' ? '' : 'none';
+ }
+
+ function renderProvisioningHistory(events) {
+ var emptyState = document.getElementById('history-empty');
+ var table = document.getElementById('history-table');
+ var tbody = document.getElementById('history-table-body');
+
+ if (!events || events.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ return;
+ }
+
+ emptyState.style.display = 'none';
+ table.style.display = '';
+ tbody.innerHTML = '';
+
+ for (var i = 0; i < events.length; i++) {
+ var evt = events[i];
+ var tr = document.createElement('tr');
+ tr.innerHTML =
+ '<td style="color: var(--theme_g1);">' + formatTimestamp(evt.ts) + '</td>' +
+ '<td>' + eventBadge(evt.type) + '</td>' +
+ '<td>' + escapeHtml(evt.worker_id || '') + '</td>' +
+ '<td>' + escapeHtml(evt.hostname || '') + '</td>';
+ tbody.appendChild(tr);
+ }
+ }
+
+ function clientHealthClass(dtMs) {
+ if (dtMs == null) return 'health-red';
+ var seconds = dtMs / 1000;
+ if (seconds < 30) return 'health-green';
+ if (seconds < 120) return 'health-yellow';
+ return 'health-red';
+ }
+
+ function renderClients(clients) {
+ var emptyState = document.getElementById('clients-empty');
+ var table = document.getElementById('clients-table');
+ var tbody = document.getElementById('clients-table-body');
+
+ if (!clients || clients.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ return;
+ }
+
+ emptyState.style.display = 'none';
+ table.style.display = '';
+ tbody.innerHTML = '';
+
+ for (var i = 0; i < clients.length; i++) {
+ var c = clients[i];
+ var dt = c.dt;
+ var hClass = clientHealthClass(dt);
+ var hTitle = dt != null ? 'Last seen ' + formatLastSeen(dt) : 'Never seen';
+
+ var sessionBadge = '';
+ if (c.session_id) {
+ sessionBadge = ' <span style="font-family:monospace;font-size:10px;color:var(--theme_faint);" title="Session ' + escapeHtml(c.session_id) + '">' + escapeHtml(c.session_id.substring(0, 8)) + '</span>';
+ }
+
+ var tr = document.createElement('tr');
+ tr.innerHTML =
+ '<td style="text-align: center;"><span class="health-dot ' + hClass + '" title="' + escapeHtml(hTitle) + '"></span></td>' +
+ '<td>' + escapeHtml(c.id || '') + sessionBadge + '</td>' +
+ '<td>' + escapeHtml(c.hostname || '') + '</td>' +
+ '<td style="font-family: monospace; font-size: 12px; color: var(--theme_g1);">' + escapeHtml(c.address || '') + '</td>' +
+ '<td style="text-align: right; color: var(--theme_g1);">' + formatLastSeen(dt) + '</td>';
+ tbody.appendChild(tr);
+ }
+ }
+
+ function clientEventBadge(type) {
+ var colors = { connected: 'var(--theme_ok)', disconnected: 'var(--theme_fail)', updated: 'var(--theme_warn)' };
+ var labels = { connected: 'Connected', disconnected: 'Disconnected', updated: 'Updated' };
+ var color = colors[type] || 'var(--theme_g1)';
+ var label = labels[type] || type;
+ return '<span style="display:inline-block;padding:2px 8px;border-radius:4px;font-size:11px;font-weight:600;color:var(--theme_g4);background:' + color + ';">' + escapeHtml(label) + '</span>';
+ }
+
+ function renderClientHistory(events) {
+ var emptyState = document.getElementById('client-history-empty');
+ var table = document.getElementById('client-history-table');
+ var tbody = document.getElementById('client-history-table-body');
+
+ if (!events || events.length === 0) {
+ emptyState.style.display = '';
+ table.style.display = 'none';
+ return;
+ }
+
+ emptyState.style.display = 'none';
+ table.style.display = '';
+ tbody.innerHTML = '';
+
+ for (var i = 0; i < events.length; i++) {
+ var evt = events[i];
+ var tr = document.createElement('tr');
+ tr.innerHTML =
+ '<td style="color: var(--theme_g1);">' + formatTimestamp(evt.ts) + '</td>' +
+ '<td>' + clientEventBadge(evt.type) + '</td>' +
+ '<td>' + escapeHtml(evt.client_id || '') + '</td>' +
+ '<td>' + escapeHtml(evt.hostname || '') + '</td>';
+ tbody.appendChild(tr);
+ }
+ }
+
+ // Fetch-based polling fallback
+ var pollTimer = null;
+
+ async function fetchProvisioningHistory() {
+ try {
+ var response = await fetch(BASE_URL + '/orch/history?limit=50', {
+ headers: { 'Accept': 'application/json' }
+ });
+ if (response.ok) {
+ var data = await response.json();
+ renderProvisioningHistory(data.events || []);
+ }
+ } catch (e) {
+ console.error('Error fetching provisioning history:', e);
+ }
+ }
+
+ async function fetchClients() {
+ try {
+ var response = await fetch(BASE_URL + '/orch/clients', {
+ headers: { 'Accept': 'application/json' }
+ });
+ if (response.ok) {
+ var data = await response.json();
+ renderClients(data.clients || []);
+ }
+ } catch (e) {
+ console.error('Error fetching clients:', e);
+ }
+ }
+
+ async function fetchClientHistory() {
+ try {
+ var response = await fetch(BASE_URL + '/orch/clients/history?limit=50', {
+ headers: { 'Accept': 'application/json' }
+ });
+ if (response.ok) {
+ var data = await response.json();
+ renderClientHistory(data.client_events || []);
+ }
+ } catch (e) {
+ console.error('Error fetching client history:', e);
+ }
+ }
+
+ async function fetchDashboard() {
+ var banner = document.querySelector('zen-banner');
+ try {
+ var response = await fetch(BASE_URL + '/orch/agents', {
+ headers: { 'Accept': 'application/json' }
+ });
+
+ if (!response.ok) {
+ banner.setAttribute('cluster-status', 'degraded');
+ throw new Error('HTTP ' + response.status + ': ' + response.statusText);
+ }
+
+ renderDashboard(await response.json());
+ fetchProvisioningHistory();
+ fetchClients();
+ fetchClientHistory();
+ } catch (error) {
+ console.error('Error updating dashboard:', error);
+ showError(error.message);
+ banner.setAttribute('cluster-status', 'offline');
+ }
+ }
+
+ function startPolling() {
+ if (pollTimer) return;
+ fetchDashboard();
+ pollTimer = setInterval(fetchDashboard, REFRESH_INTERVAL);
+ }
+
+ function stopPolling() {
+ if (pollTimer) {
+ clearInterval(pollTimer);
+ pollTimer = null;
+ }
+ }
+
+ // WebSocket connection with automatic reconnect and polling fallback
+ var ws = null;
+
+ function connectWebSocket() {
+ var proto = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+ ws = new WebSocket(proto + '//' + window.location.host + '/orch/ws');
+
+ ws.onopen = function() {
+ stopPolling();
+ clearError();
+ };
+
+ ws.onmessage = function(event) {
+ try {
+ renderDashboard(JSON.parse(event.data));
+ } catch (e) {
+ console.error('WebSocket message parse error:', e);
+ }
+ };
+
+ ws.onclose = function() {
+ ws = null;
+ startPolling();
+ setTimeout(connectWebSocket, 3000);
+ };
+
+ ws.onerror = function() {
+ // onclose will fire after onerror
+ };
+ }
+
+ // Fetch orchestrator hostname for the banner
+ fetch(BASE_URL + '/orch/status', { headers: { 'Accept': 'application/json' } })
+ .then(function(r) { return r.ok ? r.json() : null; })
+ .then(function(d) {
+ if (d && d.hostname) {
+ document.querySelector('zen-banner').setAttribute('tagline', 'Orchestrator \u2014 ' + d.hostname);
+ }
+ })
+ .catch(function() {});
+
+ // Initial load via fetch, then try WebSocket
+ fetchDashboard();
+ connectWebSocket();
+ </script>
+</body>
+</html>
diff --git a/src/zenserver/frontend/html/epicgames.ico b/src/zenserver/frontend/html/epicgames.ico
new file mode 100644
index 000000000..1cfa301a2
--- /dev/null
+++ b/src/zenserver/frontend/html/epicgames.ico
Binary files differ
diff --git a/src/zenserver/frontend/html/favicon.ico b/src/zenserver/frontend/html/favicon.ico
index 1cfa301a2..f7fb251b5 100644
--- a/src/zenserver/frontend/html/favicon.ico
+++ b/src/zenserver/frontend/html/favicon.ico
Binary files differ
diff --git a/src/zenserver/frontend/html/index.html b/src/zenserver/frontend/html/index.html
index 6a736e914..24a136a30 100644
--- a/src/zenserver/frontend/html/index.html
+++ b/src/zenserver/frontend/html/index.html
@@ -10,6 +10,9 @@
</script>
<link rel="shortcut icon" href="favicon.ico">
<link rel="stylesheet" type="text/css" href="zen.css" />
+ <script src="theme.js"></script>
+ <script src="banner.js" defer></script>
+ <script src="nav.js" defer></script>
<script type="module" src="zen.js"></script>
</head>
</html>
diff --git a/src/zenserver/frontend/html/nav.js b/src/zenserver/frontend/html/nav.js
new file mode 100644
index 000000000..a5de203f2
--- /dev/null
+++ b/src/zenserver/frontend/html/nav.js
@@ -0,0 +1,79 @@
+/**
+ * zen-nav.js — Zen dashboard navigation bar Web Component
+ *
+ * Usage:
+ * <script src="nav.js" defer></script>
+ *
+ * <zen-nav>
+ * <a href="compute.html">Node</a>
+ * <a href="orchestrator.html">Orchestrator</a>
+ * </zen-nav>
+ *
+ * Each child <a> becomes a nav link. The current page is
+ * highlighted automatically based on the href.
+ */
+
+class ZenNav extends HTMLElement {
+
+ connectedCallback() {
+ if (!this.shadowRoot) this.attachShadow({ mode: 'open' });
+ this._render();
+ }
+
+ _render() {
+ const currentPath = window.location.pathname;
+ const items = Array.from(this.querySelectorAll(':scope > a'));
+
+ const links = items.map(a => {
+ const href = a.getAttribute('href') || '';
+ const label = a.textContent.trim();
+ const active = currentPath.endsWith(href);
+ return `<a class="nav-link${active ? ' active' : ''}" href="${href}">${label}</a>`;
+ }).join('');
+
+ this.shadowRoot.innerHTML = `
+ <style>
+ *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+ :host {
+ display: block;
+ margin-bottom: 16px;
+ }
+
+ .nav-bar {
+ display: flex;
+ align-items: center;
+ gap: 4px;
+ padding: 4px;
+ background: var(--theme_g3);
+ border: 1px solid var(--theme_g2);
+ border-radius: 6px;
+ }
+
+ .nav-link {
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+ font-size: 13px;
+ font-weight: 500;
+ color: var(--theme_g1);
+ text-decoration: none;
+ padding: 6px 14px;
+ border-radius: 4px;
+ transition: color 0.15s, background 0.15s;
+ }
+
+ .nav-link:hover {
+ color: var(--theme_g0);
+ background: var(--theme_p4);
+ }
+
+ .nav-link.active {
+ color: var(--theme_bright);
+ background: var(--theme_g2);
+ }
+ </style>
+ <nav class="nav-bar">${links}</nav>
+ `;
+ }
+}
+
+customElements.define('zen-nav', ZenNav);
diff --git a/src/zenserver/frontend/html/pages/cache.js b/src/zenserver/frontend/html/pages/cache.js
new file mode 100644
index 000000000..3b838958a
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/cache.js
@@ -0,0 +1,690 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
+import { Modal } from "../util/modal.js"
+import { Table, Toolbar } from "../util/widgets.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+ async main()
+ {
+ this.set_title("cache");
+
+ // Cache Service Stats
+ const stats_section = this._collapsible_section("Cache Service Stats");
+ stats_section.tag().classify("dropall").text("raw yaml \u2192").on_click(() => {
+ window.open("/stats/z$.yaml?cidstorestats=true&cachestorestats=true", "_blank");
+ });
+ this._stats_grid = stats_section.tag().classify("grid").classify("stats-tiles");
+ this._details_host = stats_section;
+ this._details_container = null;
+ this._selected_category = null;
+
+ const stats = await new Fetcher().resource("stats", "z$").json();
+ if (stats)
+ {
+ this._render_stats(stats);
+ }
+
+ this._connect_stats_ws();
+
+ // Cache Namespaces
+ var section = this._collapsible_section("Cache Namespaces");
+
+ section.tag().classify("dropall").text("drop-all").on_click(() => this.drop_all());
+
+ var columns = [
+ "namespace",
+ "dir",
+ "buckets",
+ "entries",
+ "size disk",
+ "size mem",
+ "actions",
+ ];
+
+ var zcache_info = await new Fetcher().resource("/z$/").json();
+ this._cache_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_AlignNumeric);
+
+ for (const namespace of zcache_info["Namespaces"] || [])
+ {
+ new Fetcher().resource(`/z$/${namespace}/`).json().then((data) => {
+ const row = this._cache_table.add_row(
+ "",
+ data["Configuration"]["RootDir"],
+ data["Buckets"].length,
+ data["EntryCount"],
+ Friendly.bytes(data["StorageSize"].DiskSize),
+ Friendly.bytes(data["StorageSize"].MemorySize)
+ );
+ var cell = row.get_cell(0);
+ cell.tag().text(namespace).on_click(() => this.view_namespace(namespace));
+
+ cell = row.get_cell(-1);
+ const action_tb = new Toolbar(cell, true);
+ action_tb.left().add("view").on_click(() => this.view_namespace(namespace));
+ action_tb.left().add("drop").on_click(() => this.drop_namespace(namespace));
+
+ row.attr("zs_name", namespace);
+ });
+ }
+
+ // Namespace detail area (inside namespaces section so it collapses together)
+ this._namespace_host = section;
+ this._namespace_container = null;
+ this._selected_namespace = null;
+
+ // Restore namespace from URL if present
+ const ns_param = this.get_param("namespace");
+ if (ns_param)
+ {
+ this.view_namespace(ns_param);
+ }
+ }
+
+ _collapsible_section(name)
+ {
+ const section = this.add_section(name);
+ const container = section._parent.inner();
+ const heading = container.firstElementChild;
+
+ heading.style.cursor = "pointer";
+ heading.style.userSelect = "none";
+
+ const indicator = document.createElement("span");
+ indicator.textContent = " \u25BC";
+ indicator.style.fontSize = "0.7em";
+ heading.appendChild(indicator);
+
+ let collapsed = false;
+ heading.addEventListener("click", (e) => {
+ if (e.target !== heading && e.target !== indicator)
+ {
+ return;
+ }
+ collapsed = !collapsed;
+ indicator.textContent = collapsed ? " \u25B6" : " \u25BC";
+ let sibling = heading.nextElementSibling;
+ while (sibling)
+ {
+ sibling.style.display = collapsed ? "none" : "";
+ sibling = sibling.nextElementSibling;
+ }
+ });
+
+ return section;
+ }
+
+ _connect_stats_ws()
+ {
+ try
+ {
+ const proto = location.protocol === "https:" ? "wss:" : "ws:";
+ const ws = new WebSocket(`${proto}//${location.host}/stats`);
+
+ try { this._ws_paused = localStorage.getItem("zen-ws-paused") === "true"; } catch (e) { this._ws_paused = false; }
+ document.addEventListener("zen-ws-toggle", (e) => {
+ this._ws_paused = e.detail.paused;
+ });
+
+ ws.onmessage = (ev) => {
+ if (this._ws_paused)
+ {
+ return;
+ }
+ try
+ {
+ const all_stats = JSON.parse(ev.data);
+ const stats = all_stats["z$"];
+ if (stats)
+ {
+ this._render_stats(stats);
+ }
+ }
+ catch (e) { /* ignore parse errors */ }
+ };
+
+ ws.onclose = () => { this._stats_ws = null; };
+ ws.onerror = () => { ws.close(); };
+
+ this._stats_ws = ws;
+ }
+ catch (e) { /* WebSocket not available */ }
+ }
+
+ _render_stats(stats)
+ {
+ const safe = (obj, path) => path.split(".").reduce((a, b) => a && a[b], obj);
+ const grid = this._stats_grid;
+
+ this._last_stats = stats;
+ grid.inner().innerHTML = "";
+
+ // Store I/O tile
+ {
+ const store = safe(stats, "cache.store");
+ if (store)
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile").classify("stats-tile-detailed");
+ if (this._selected_category === "store") tile.classify("stats-tile-selected");
+ tile.on_click(() => this._select_category("store"));
+ tile.tag().classify("card-title").text("Store I/O");
+ const columns = tile.tag().classify("tile-columns");
+
+ const left = columns.tag().classify("tile-metrics");
+ const storeHits = store.hits || 0;
+ const storeMisses = store.misses || 0;
+ const storeTotal = storeHits + storeMisses;
+ const storeRatio = storeTotal > 0 ? ((storeHits / storeTotal) * 100).toFixed(1) + "%" : "-";
+ this._metric(left, storeRatio, "store hit ratio", true);
+ this._metric(left, Friendly.sep(storeHits), "hits");
+ this._metric(left, Friendly.sep(storeMisses), "misses");
+ this._metric(left, Friendly.sep(store.writes || 0), "writes");
+ this._metric(left, Friendly.sep(store.rejected_reads || 0), "rejected reads");
+ this._metric(left, Friendly.sep(store.rejected_writes || 0), "rejected writes");
+
+ const right = columns.tag().classify("tile-metrics");
+ const readRateMean = safe(store, "read.bytes.rate_mean") || 0;
+ const readRate1 = safe(store, "read.bytes.rate_1") || 0;
+ const readRate5 = safe(store, "read.bytes.rate_5") || 0;
+ const writeRateMean = safe(store, "write.bytes.rate_mean") || 0;
+ const writeRate1 = safe(store, "write.bytes.rate_1") || 0;
+ const writeRate5 = safe(store, "write.bytes.rate_5") || 0;
+ this._metric(right, Friendly.bytes(readRateMean) + "/s", "read rate (mean)", true);
+ this._metric(right, Friendly.bytes(readRate1) + "/s", "read rate (1m)");
+ this._metric(right, Friendly.bytes(readRate5) + "/s", "read rate (5m)");
+ this._metric(right, Friendly.bytes(writeRateMean) + "/s", "write rate (mean)");
+ this._metric(right, Friendly.bytes(writeRate1) + "/s", "write rate (1m)");
+ this._metric(right, Friendly.bytes(writeRate5) + "/s", "write rate (5m)");
+ }
+ }
+
+ // Hit/Miss tile
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Hit Ratio");
+ const columns = tile.tag().classify("tile-columns");
+
+ const left = columns.tag().classify("tile-metrics");
+ const hits = safe(stats, "cache.hits") || 0;
+ const misses = safe(stats, "cache.misses") || 0;
+ const writes = safe(stats, "cache.writes") || 0;
+ const total = hits + misses;
+ const ratio = total > 0 ? ((hits / total) * 100).toFixed(1) + "%" : "-";
+
+ this._metric(left, ratio, "hit ratio", true);
+ this._metric(left, Friendly.sep(hits), "hits");
+ this._metric(left, Friendly.sep(misses), "misses");
+ this._metric(left, Friendly.sep(writes), "writes");
+
+ const right = columns.tag().classify("tile-metrics");
+ const cidHits = safe(stats, "cache.cidhits") || 0;
+ const cidMisses = safe(stats, "cache.cidmisses") || 0;
+ const cidWrites = safe(stats, "cache.cidwrites") || 0;
+ const cidTotal = cidHits + cidMisses;
+ const cidRatio = cidTotal > 0 ? ((cidHits / cidTotal) * 100).toFixed(1) + "%" : "-";
+
+ this._metric(right, cidRatio, "cid hit ratio", true);
+ this._metric(right, Friendly.sep(cidHits), "cid hits");
+ this._metric(right, Friendly.sep(cidMisses), "cid misses");
+ this._metric(right, Friendly.sep(cidWrites), "cid writes");
+ }
+
+ // HTTP Requests tile
+ {
+ const req = safe(stats, "requests");
+ if (req)
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("HTTP Requests");
+ const columns = tile.tag().classify("tile-columns");
+
+ const left = columns.tag().classify("tile-metrics");
+ const reqData = req.requests || req;
+ this._metric(left, Friendly.sep(reqData.count || 0), "total requests", true);
+ if (reqData.rate_mean > 0)
+ {
+ this._metric(left, Friendly.sep(reqData.rate_mean, 1) + "/s", "req/sec (mean)");
+ }
+ if (reqData.rate_1 > 0)
+ {
+ this._metric(left, Friendly.sep(reqData.rate_1, 1) + "/s", "req/sec (1m)");
+ }
+ if (reqData.rate_5 > 0)
+ {
+ this._metric(left, Friendly.sep(reqData.rate_5, 1) + "/s", "req/sec (5m)");
+ }
+ if (reqData.rate_15 > 0)
+ {
+ this._metric(left, Friendly.sep(reqData.rate_15, 1) + "/s", "req/sec (15m)");
+ }
+ const badRequests = safe(stats, "cache.badrequestcount") || 0;
+ this._metric(left, Friendly.sep(badRequests), "bad requests");
+
+ const right = columns.tag().classify("tile-metrics");
+ this._metric(right, Friendly.duration(reqData.t_avg || 0), "avg latency", true);
+ if (reqData.t_p75)
+ {
+ this._metric(right, Friendly.duration(reqData.t_p75), "p75");
+ }
+ if (reqData.t_p95)
+ {
+ this._metric(right, Friendly.duration(reqData.t_p95), "p95");
+ }
+ if (reqData.t_p99)
+ {
+ this._metric(right, Friendly.duration(reqData.t_p99), "p99");
+ }
+ if (reqData.t_p999)
+ {
+ this._metric(right, Friendly.duration(reqData.t_p999), "p999");
+ }
+ if (reqData.t_max)
+ {
+ this._metric(right, Friendly.duration(reqData.t_max), "max");
+ }
+ }
+ }
+
+ // RPC tile
+ {
+ const rpc = safe(stats, "cache.rpc");
+ if (rpc)
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("RPC");
+ const columns = tile.tag().classify("tile-columns");
+
+ const left = columns.tag().classify("tile-metrics");
+ this._metric(left, Friendly.sep(rpc.count || 0), "rpc calls", true);
+ this._metric(left, Friendly.sep(rpc.ops || 0), "batch ops");
+
+ const right = columns.tag().classify("tile-metrics");
+ if (rpc.records)
+ {
+ this._metric(right, Friendly.sep(rpc.records.count || 0), "record calls");
+ this._metric(right, Friendly.sep(rpc.records.ops || 0), "record ops");
+ }
+ if (rpc.values)
+ {
+ this._metric(right, Friendly.sep(rpc.values.count || 0), "value calls");
+ this._metric(right, Friendly.sep(rpc.values.ops || 0), "value ops");
+ }
+ if (rpc.chunks)
+ {
+ this._metric(right, Friendly.sep(rpc.chunks.count || 0), "chunk calls");
+ this._metric(right, Friendly.sep(rpc.chunks.ops || 0), "chunk ops");
+ }
+ }
+ }
+
+ // Storage tile
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile").classify("stats-tile-detailed");
+ if (this._selected_category === "storage") tile.classify("stats-tile-selected");
+ tile.on_click(() => this._select_category("storage"));
+ tile.tag().classify("card-title").text("Storage");
+ const columns = tile.tag().classify("tile-columns");
+
+ const left = columns.tag().classify("tile-metrics");
+ this._metric(left, safe(stats, "cache.size.disk") != null ? Friendly.bytes(safe(stats, "cache.size.disk")) : "-", "cache disk", true);
+ this._metric(left, safe(stats, "cache.size.memory") != null ? Friendly.bytes(safe(stats, "cache.size.memory")) : "-", "cache memory");
+
+ const right = columns.tag().classify("tile-metrics");
+ this._metric(right, safe(stats, "cid.size.total") != null ? Friendly.bytes(safe(stats, "cid.size.total")) : "-", "cid total", true);
+ this._metric(right, safe(stats, "cid.size.tiny") != null ? Friendly.bytes(safe(stats, "cid.size.tiny")) : "-", "cid tiny");
+ this._metric(right, safe(stats, "cid.size.small") != null ? Friendly.bytes(safe(stats, "cid.size.small")) : "-", "cid small");
+ this._metric(right, safe(stats, "cid.size.large") != null ? Friendly.bytes(safe(stats, "cid.size.large")) : "-", "cid large");
+ }
+
+ // Upstream tile (only if upstream is active)
+ {
+ const upstream = safe(stats, "upstream");
+ if (upstream)
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Upstream");
+ const body = tile.tag().classify("tile-metrics");
+
+ const upstreamHits = safe(stats, "cache.upstream_hits") || 0;
+ this._metric(body, Friendly.sep(upstreamHits), "upstream hits", true);
+
+ if (upstream.url)
+ {
+ this._metric(body, upstream.url, "endpoint");
+ }
+ }
+ }
+ }
+
+ _metric(parent, value, label, hero = false)
+ {
+ const m = parent.tag().classify("tile-metric");
+ if (hero)
+ {
+ m.classify("tile-metric-hero");
+ }
+ m.tag().classify("metric-value").text(value);
+ m.tag().classify("metric-label").text(label);
+ }
+
+ async _select_category(category)
+ {
+ // Toggle off if already selected
+ if (this._selected_category === category)
+ {
+ this._selected_category = null;
+ this._clear_details();
+ this._render_stats(this._last_stats);
+ return;
+ }
+
+ this._selected_category = category;
+ this._render_stats(this._last_stats);
+
+ // Fetch detailed stats
+ const detailed = await new Fetcher()
+ .resource("stats", "z$")
+ .param("cachestorestats", "true")
+ .param("cidstorestats", "true")
+ .json();
+
+ if (!detailed || this._selected_category !== category)
+ {
+ return;
+ }
+
+ this._clear_details();
+
+ const safe = (obj, path) => path.split(".").reduce((a, b) => a && a[b], obj);
+
+ if (category === "store")
+ {
+ this._render_store_details(detailed, safe);
+ }
+ else if (category === "storage")
+ {
+ this._render_storage_details(detailed, safe);
+ }
+ }
+
+ _clear_details()
+ {
+ if (this._details_container)
+ {
+ this._details_container.inner().remove();
+ this._details_container = null;
+ }
+ }
+
+ _render_store_details(stats, safe)
+ {
+ const namespaces = safe(stats, "cache.store.namespaces") || [];
+ if (namespaces.length === 0)
+ {
+ return;
+ }
+
+ const container = this._details_host.tag();
+ this._details_container = container;
+
+ const columns = [
+ "namespace",
+ "bucket",
+ "hits",
+ "misses",
+ "writes",
+ "hit ratio",
+ "read count",
+ "read bandwidth",
+ "write count",
+ "write bandwidth",
+ ];
+ const table = new Table(container, columns, Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric);
+
+ for (const ns of namespaces)
+ {
+ const nsHits = ns.hits || 0;
+ const nsMisses = ns.misses || 0;
+ const nsTotal = nsHits + nsMisses;
+ const nsRatio = nsTotal > 0 ? ((nsHits / nsTotal) * 100).toFixed(1) + "%" : "-";
+
+ const readCount = safe(ns, "read.request.count") || 0;
+ const readBytes = safe(ns, "read.bytes.count") || 0;
+ const writeCount = safe(ns, "write.request.count") || 0;
+ const writeBytes = safe(ns, "write.bytes.count") || 0;
+
+ table.add_row(
+ ns.namespace,
+ "",
+ Friendly.sep(nsHits),
+ Friendly.sep(nsMisses),
+ Friendly.sep(ns.writes || 0),
+ nsRatio,
+ Friendly.sep(readCount),
+ Friendly.bytes(readBytes),
+ Friendly.sep(writeCount),
+ Friendly.bytes(writeBytes),
+ );
+
+ if (ns.buckets && ns.buckets.length > 0)
+ {
+ for (const bucket of ns.buckets)
+ {
+ const bHits = bucket.hits || 0;
+ const bMisses = bucket.misses || 0;
+ const bTotal = bHits + bMisses;
+ const bRatio = bTotal > 0 ? ((bHits / bTotal) * 100).toFixed(1) + "%" : "-";
+
+ const bReadCount = safe(bucket, "read.request.count") || 0;
+ const bReadBytes = safe(bucket, "read.bytes.count") || 0;
+ const bWriteCount = safe(bucket, "write.request.count") || 0;
+ const bWriteBytes = safe(bucket, "write.bytes.count") || 0;
+
+ table.add_row(
+ ns.namespace,
+ bucket.bucket,
+ Friendly.sep(bHits),
+ Friendly.sep(bMisses),
+ Friendly.sep(bucket.writes || 0),
+ bRatio,
+ Friendly.sep(bReadCount),
+ Friendly.bytes(bReadBytes),
+ Friendly.sep(bWriteCount),
+ Friendly.bytes(bWriteBytes),
+ );
+ }
+ }
+ }
+ }
+
+ _render_storage_details(stats, safe)
+ {
+ const namespaces = safe(stats, "cache.store.namespaces") || [];
+ if (namespaces.length === 0)
+ {
+ return;
+ }
+
+ const container = this._details_host.tag();
+ this._details_container = container;
+
+ const columns = [
+ "namespace",
+ "bucket",
+ "disk",
+ "memory",
+ ];
+ const table = new Table(container, columns, Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric);
+
+ for (const ns of namespaces)
+ {
+ const diskSize = safe(ns, "size.disk") || 0;
+ const memSize = safe(ns, "size.memory") || 0;
+
+ table.add_row(
+ ns.namespace,
+ "",
+ Friendly.bytes(diskSize),
+ Friendly.bytes(memSize),
+ );
+
+ if (ns.buckets && ns.buckets.length > 0)
+ {
+ for (const bucket of ns.buckets)
+ {
+ const bDisk = safe(bucket, "size.disk") || 0;
+ const bMem = safe(bucket, "size.memory") || 0;
+
+ table.add_row(
+ ns.namespace,
+ bucket.bucket,
+ Friendly.bytes(bDisk),
+ Friendly.bytes(bMem),
+ );
+ }
+ }
+ }
+ }
+
+ async view_namespace(namespace)
+ {
+ // Toggle off if already selected
+ if (this._selected_namespace === namespace)
+ {
+ this._selected_namespace = null;
+ this._clear_namespace();
+ this._clear_param("namespace");
+ return;
+ }
+
+ this._selected_namespace = namespace;
+ this._clear_namespace();
+ this.set_param("namespace", namespace);
+
+ const info = await new Fetcher().resource(`/z$/${namespace}/`).json();
+ if (this._selected_namespace !== namespace)
+ {
+ return;
+ }
+
+ const section = this._namespace_host.add_section(namespace);
+ this._namespace_container = section;
+
+ // Buckets table
+ const bucket_section = section.add_section("Buckets");
+ const bucket_columns = ["name", "disk", "memory", "entries", "actions"];
+ const bucket_table = bucket_section.add_widget(
+ Table,
+ bucket_columns,
+ Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric
+ );
+
+ // Right-align header for numeric columns (skip # and name)
+ const header = bucket_table._element.firstElementChild;
+ for (let i = 2; i < header.children.length - 1; i++)
+ {
+ header.children[i].style.textAlign = "right";
+ }
+
+ let totalDisk = 0, totalMem = 0, totalEntries = 0;
+ const total_row = bucket_table.add_row("TOTAL");
+ total_row.get_cell(0).style("fontWeight", "bold");
+ total_row.get_cell(1).style("textAlign", "right").style("fontWeight", "bold");
+ total_row.get_cell(2).style("textAlign", "right").style("fontWeight", "bold");
+ total_row.get_cell(3).style("textAlign", "right").style("fontWeight", "bold");
+
+ for (const bucket of info["Buckets"])
+ {
+ const row = bucket_table.add_row(bucket);
+ new Fetcher().resource(`/z$/${namespace}/${bucket}`).json().then((data) => {
+ row.get_cell(1).text(Friendly.bytes(data["StorageSize"]["DiskSize"])).style("textAlign", "right");
+ row.get_cell(2).text(Friendly.bytes(data["StorageSize"]["MemorySize"])).style("textAlign", "right");
+ row.get_cell(3).text(Friendly.sep(data["DiskEntryCount"])).style("textAlign", "right");
+
+ const cell = row.get_cell(-1);
+ const action_tb = new Toolbar(cell, true);
+ action_tb.left().add("drop").on_click(() => this.drop_bucket(namespace, bucket));
+
+ totalDisk += data["StorageSize"]["DiskSize"];
+ totalMem += data["StorageSize"]["MemorySize"];
+ totalEntries += data["DiskEntryCount"];
+ total_row.get_cell(1).text(Friendly.bytes(totalDisk)).style("textAlign", "right").style("fontWeight", "bold");
+ total_row.get_cell(2).text(Friendly.bytes(totalMem)).style("textAlign", "right").style("fontWeight", "bold");
+ total_row.get_cell(3).text(Friendly.sep(totalEntries)).style("textAlign", "right").style("fontWeight", "bold");
+ });
+ }
+
+ }
+
+ _clear_param(name)
+ {
+ this._params.delete(name);
+ const url = new URL(window.location);
+ url.searchParams.delete(name);
+ history.replaceState(null, "", url);
+ }
+
+ _clear_namespace()
+ {
+ if (this._namespace_container)
+ {
+ this._namespace_container._parent.inner().remove();
+ this._namespace_container = null;
+ }
+ }
+
+ drop_bucket(namespace, bucket)
+ {
+ const drop = async () => {
+ await new Fetcher().resource("z$", namespace, bucket).delete();
+ // Refresh the namespace view
+ this._selected_namespace = null;
+ this._clear_namespace();
+ this.view_namespace(namespace);
+ };
+
+ new Modal()
+ .title("Confirmation")
+ .message(`Drop bucket '${bucket}'?`)
+ .option("Yes", () => drop())
+ .option("No");
+ }
+
+ drop_namespace(namespace)
+ {
+ const drop = async () => {
+ await new Fetcher().resource("z$", namespace).delete();
+ this.reload();
+ };
+
+ new Modal()
+ .title("Confirmation")
+ .message(`Drop cache namespace '${namespace}'?`)
+ .option("Yes", () => drop())
+ .option("No");
+ }
+
+ async drop_all()
+ {
+ const drop = async () => {
+ for (const row of this._cache_table)
+ {
+ const namespace = row.attr("zs_name");
+ await new Fetcher().resource("z$", namespace).delete();
+ }
+ this.reload();
+ };
+
+ new Modal()
+ .title("Confirmation")
+ .message("Drop every cache namespace?")
+ .option("Yes", () => drop())
+ .option("No");
+ }
+}
diff --git a/src/zenserver/frontend/html/pages/compute.js b/src/zenserver/frontend/html/pages/compute.js
new file mode 100644
index 000000000..ab3d49c27
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/compute.js
@@ -0,0 +1,693 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
+import { Table } from "../util/widgets.js"
+
+const MAX_HISTORY_POINTS = 60;
+
+// Windows FILETIME: 100ns ticks since 1601-01-01
+const FILETIME_EPOCH_OFFSET_MS = 11644473600000n;
+function filetimeToDate(ticks)
+{
+ if (!ticks) return null;
+ const ms = BigInt(ticks) / 10000n - FILETIME_EPOCH_OFFSET_MS;
+ return new Date(Number(ms));
+}
+
+function formatTime(date)
+{
+ if (!date) return "-";
+ return date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" });
+}
+
+function formatDuration(startDate, endDate)
+{
+ if (!startDate || !endDate) return "-";
+ const ms = endDate - startDate;
+ if (ms < 0) return "-";
+ if (ms < 1000) return ms + " ms";
+ if (ms < 60000) return (ms / 1000).toFixed(2) + " s";
+ const m = Math.floor(ms / 60000);
+ const s = ((ms % 60000) / 1000).toFixed(0).padStart(2, "0");
+ return `${m}m ${s}s`;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+ async main()
+ {
+ this.set_title("compute");
+
+ this._history = { timestamps: [], pending: [], running: [], completed: [], cpu: [] };
+ this._selected_worker = null;
+ this._chart_js = null;
+ this._queue_chart = null;
+ this._cpu_chart = null;
+
+ this._ws_paused = false;
+ try { this._ws_paused = localStorage.getItem("zen-ws-paused") === "true"; } catch (e) {}
+ document.addEventListener("zen-ws-toggle", (e) => {
+ this._ws_paused = e.detail.paused;
+ });
+
+ // Action Queue section
+ const queue_section = this._collapsible_section("Action Queue");
+ this._queue_grid = queue_section.tag().classify("grid").classify("stats-tiles");
+ this._chart_host = queue_section;
+
+ // Performance Metrics section
+ const perf_section = this._collapsible_section("Performance Metrics");
+ this._perf_host = perf_section;
+ this._perf_grid = null;
+
+ // Workers section
+ const workers_section = this._collapsible_section("Workers");
+ this._workers_host = workers_section;
+ this._workers_table = null;
+ this._worker_detail_container = null;
+
+ // Queues section
+ const queues_section = this._collapsible_section("Queues");
+ this._queues_host = queues_section;
+ this._queues_table = null;
+
+ // Action History section
+ const history_section = this._collapsible_section("Recent Actions");
+ this._history_host = history_section;
+ this._history_table = null;
+
+ // System Resources section
+ const sys_section = this._collapsible_section("System Resources");
+ this._sys_grid = sys_section.tag().classify("grid").classify("stats-tiles");
+
+ // Load Chart.js dynamically
+ this._load_chartjs();
+
+ // Initial fetch
+ await this._fetch_all();
+
+ // Poll
+ this._poll_timer = setInterval(() => {
+ if (!this._ws_paused)
+ {
+ this._fetch_all();
+ }
+ }, 2000);
+ }
+
+ _collapsible_section(name)
+ {
+ const section = this.add_section(name);
+ const container = section._parent.inner();
+ const heading = container.firstElementChild;
+
+ heading.style.cursor = "pointer";
+ heading.style.userSelect = "none";
+
+ const indicator = document.createElement("span");
+ indicator.textContent = " \u25BC";
+ indicator.style.fontSize = "0.7em";
+ heading.appendChild(indicator);
+
+ let collapsed = false;
+ heading.addEventListener("click", (e) => {
+ if (e.target !== heading && e.target !== indicator)
+ {
+ return;
+ }
+ collapsed = !collapsed;
+ indicator.textContent = collapsed ? " \u25B6" : " \u25BC";
+ let sibling = heading.nextElementSibling;
+ while (sibling)
+ {
+ sibling.style.display = collapsed ? "none" : "";
+ sibling = sibling.nextElementSibling;
+ }
+ });
+
+ return section;
+ }
+
+ async _load_chartjs()
+ {
+ if (window.Chart)
+ {
+ this._chart_js = window.Chart;
+ this._init_charts();
+ return;
+ }
+
+ try
+ {
+ const script = document.createElement("script");
+ script.src = "https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.umd.min.js";
+ script.onload = () => {
+ this._chart_js = window.Chart;
+ this._init_charts();
+ };
+ document.head.appendChild(script);
+ }
+ catch (e) { /* Chart.js not available */ }
+ }
+
+ _init_charts()
+ {
+ if (!this._chart_js)
+ {
+ return;
+ }
+
+ // Queue history chart
+ {
+ const card = this._chart_host.tag().classify("card");
+ card.tag().classify("card-title").text("Action Queue History");
+ const container = card.tag();
+ container.style("position", "relative").style("height", "300px").style("marginTop", "20px");
+ const canvas = document.createElement("canvas");
+ container.inner().appendChild(canvas);
+
+ this._queue_chart = new this._chart_js(canvas.getContext("2d"), {
+ type: "line",
+ data: {
+ labels: [],
+ datasets: [
+ { label: "Pending", data: [], borderColor: "#f0883e", backgroundColor: "rgba(240, 136, 62, 0.1)", tension: 0.4, fill: true },
+ { label: "Running", data: [], borderColor: "#58a6ff", backgroundColor: "rgba(88, 166, 255, 0.1)", tension: 0.4, fill: true },
+ { label: "Completed", data: [], borderColor: "#3fb950", backgroundColor: "rgba(63, 185, 80, 0.1)", tension: 0.4, fill: true },
+ ]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ plugins: { legend: { display: true, labels: { color: "#8b949e" } } },
+ scales: { x: { display: false }, y: { beginAtZero: true, ticks: { color: "#8b949e" }, grid: { color: "#21262d" } } }
+ }
+ });
+ }
+
+ // CPU sparkline (will be appended to CPU card later)
+ this._cpu_canvas = document.createElement("canvas");
+ this._cpu_chart = new this._chart_js(this._cpu_canvas.getContext("2d"), {
+ type: "line",
+ data: {
+ labels: [],
+ datasets: [{
+ data: [],
+ borderColor: "#58a6ff",
+ backgroundColor: "rgba(88, 166, 255, 0.15)",
+ borderWidth: 1.5,
+ tension: 0.4,
+ fill: true,
+ pointRadius: 0
+ }]
+ },
+ options: {
+ responsive: true,
+ maintainAspectRatio: false,
+ animation: false,
+ plugins: { legend: { display: false }, tooltip: { enabled: false } },
+ scales: { x: { display: false }, y: { display: false, min: 0, max: 100 } }
+ }
+ });
+ }
+
+ async _fetch_all()
+ {
+ try
+ {
+ const [stats, sysinfo, workers_data, queues_data, history_data] = await Promise.all([
+ new Fetcher().resource("/stats/compute").json().catch(() => null),
+ new Fetcher().resource("/compute/sysinfo").json().catch(() => null),
+ new Fetcher().resource("/compute/workers").json().catch(() => null),
+ new Fetcher().resource("/compute/queues").json().catch(() => null),
+ new Fetcher().resource("/compute/jobs/history").param("limit", "50").json().catch(() => null),
+ ]);
+
+ if (stats)
+ {
+ this._render_queue_stats(stats);
+ this._update_queue_chart(stats);
+ this._render_perf(stats);
+ }
+ if (sysinfo)
+ {
+ this._render_sysinfo(sysinfo);
+ }
+ if (workers_data)
+ {
+ this._render_workers(workers_data);
+ }
+ if (queues_data)
+ {
+ this._render_queues(queues_data);
+ }
+ if (history_data)
+ {
+ this._render_action_history(history_data);
+ }
+ }
+ catch (e) { /* service unavailable */ }
+ }
+
+ _render_queue_stats(data)
+ {
+ const grid = this._queue_grid;
+ grid.inner().innerHTML = "";
+
+ const tiles = [
+ { title: "Pending Actions", value: data.actions_pending || 0, label: "waiting to be scheduled" },
+ { title: "Running Actions", value: data.actions_submitted || 0, label: "currently executing" },
+ { title: "Completed Actions", value: data.actions_complete || 0, label: "results available" },
+ ];
+
+ for (const t of tiles)
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text(t.title);
+ const body = tile.tag().classify("tile-metrics");
+ this._metric(body, Friendly.sep(t.value), t.label, true);
+ }
+ }
+
+ _update_queue_chart(data)
+ {
+ const h = this._history;
+ h.timestamps.push(new Date().toLocaleTimeString());
+ h.pending.push(data.actions_pending || 0);
+ h.running.push(data.actions_submitted || 0);
+ h.completed.push(data.actions_complete || 0);
+
+ while (h.timestamps.length > MAX_HISTORY_POINTS)
+ {
+ h.timestamps.shift();
+ h.pending.shift();
+ h.running.shift();
+ h.completed.shift();
+ }
+
+ if (this._queue_chart)
+ {
+ this._queue_chart.data.labels = h.timestamps;
+ this._queue_chart.data.datasets[0].data = h.pending;
+ this._queue_chart.data.datasets[1].data = h.running;
+ this._queue_chart.data.datasets[2].data = h.completed;
+ this._queue_chart.update("none");
+ }
+ }
+
+ _render_perf(data)
+ {
+ if (!this._perf_grid)
+ {
+ this._perf_grid = this._perf_host.tag().classify("grid").classify("stats-tiles");
+ }
+ const grid = this._perf_grid;
+ grid.inner().innerHTML = "";
+
+ const retired = data.actions_retired || {};
+
+ // Completion rate card
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Completion Rate");
+ const body = tile.tag().classify("tile-columns");
+
+ const left = body.tag().classify("tile-metrics");
+ this._metric(left, this._fmt_rate(retired.rate_1), "1 min rate", true);
+ this._metric(left, this._fmt_rate(retired.rate_5), "5 min rate");
+ this._metric(left, this._fmt_rate(retired.rate_15), "15 min rate");
+
+ const right = body.tag().classify("tile-metrics");
+ this._metric(right, Friendly.sep(retired.count || 0), "total retired", true);
+ this._metric(right, this._fmt_rate(retired.rate_mean), "mean rate");
+ }
+ }
+
+ _fmt_rate(rate)
+ {
+ if (rate == null) return "-";
+ return rate.toFixed(2) + "/s";
+ }
+
+ _render_workers(data)
+ {
+ const workerIds = data.workers || [];
+
+ if (this._workers_table)
+ {
+ this._workers_table.clear();
+ }
+ else
+ {
+ this._workers_table = this._workers_host.add_widget(
+ Table,
+ ["name", "platform", "cores", "timeout", "functions", "worker ID"],
+ Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric, -1
+ );
+ }
+
+ if (workerIds.length === 0)
+ {
+ return;
+ }
+
+ // Fetch each worker's descriptor
+ Promise.all(
+ workerIds.map(id =>
+ new Fetcher().resource("/compute/workers", id).json()
+ .then(desc => ({ id, desc }))
+ .catch(() => ({ id, desc: null }))
+ )
+ ).then(results => {
+ this._workers_table.clear();
+ for (const { id, desc } of results)
+ {
+ const name = desc ? (desc.name || "-") : "-";
+ const host = desc ? (desc.host || "-") : "-";
+ const cores = desc ? (desc.cores != null ? desc.cores : "-") : "-";
+ const timeout = desc ? (desc.timeout != null ? desc.timeout + "s" : "-") : "-";
+ const functions = desc ? (desc.functions ? desc.functions.length : 0) : "-";
+
+ const row = this._workers_table.add_row(
+ "",
+ host,
+ String(cores),
+ String(timeout),
+ String(functions),
+ id,
+ );
+
+ // Make name clickable to expand detail
+ const cell = row.get_cell(0);
+ cell.tag().text(name).on_click(() => this._toggle_worker_detail(id, desc));
+
+ // Highlight selected
+ if (id === this._selected_worker)
+ {
+ row.style("background", "var(--theme_p3)");
+ }
+ }
+
+ this._worker_descriptors = Object.fromEntries(results.map(r => [r.id, r.desc]));
+
+ // Re-render detail if still selected
+ if (this._selected_worker && this._worker_descriptors[this._selected_worker])
+ {
+ this._show_worker_detail(this._selected_worker, this._worker_descriptors[this._selected_worker]);
+ }
+ else if (this._selected_worker)
+ {
+ this._selected_worker = null;
+ this._clear_worker_detail();
+ }
+ });
+ }
+
+ _toggle_worker_detail(id, desc)
+ {
+ if (this._selected_worker === id)
+ {
+ this._selected_worker = null;
+ this._clear_worker_detail();
+ return;
+ }
+ this._selected_worker = id;
+ this._show_worker_detail(id, desc);
+ }
+
+ _clear_worker_detail()
+ {
+ if (this._worker_detail_container)
+ {
+ this._worker_detail_container._parent.inner().remove();
+ this._worker_detail_container = null;
+ }
+ }
+
+ _show_worker_detail(id, desc)
+ {
+ this._clear_worker_detail();
+ if (!desc)
+ {
+ return;
+ }
+
+ const section = this._workers_host.add_section(desc.name || id);
+ this._worker_detail_container = section;
+
+ // Basic info table
+ const info_table = section.add_widget(
+ Table, ["property", "value"], Table.Flag_FitLeft|Table.Flag_PackRight
+ );
+ const fields = [
+ ["Worker ID", id],
+ ["Path", desc.path || "-"],
+ ["Platform", desc.host || "-"],
+ ["Build System", desc.buildsystem_version || "-"],
+ ["Cores", desc.cores != null ? String(desc.cores) : "-"],
+ ["Timeout", desc.timeout != null ? desc.timeout + "s" : "-"],
+ ];
+ for (const [label, value] of fields)
+ {
+ info_table.add_row(label, value);
+ }
+
+ // Functions
+ const functions = desc.functions || [];
+ if (functions.length > 0)
+ {
+ const fn_section = section.add_section("Functions");
+ const fn_table = fn_section.add_widget(
+ Table, ["name", "version"], Table.Flag_FitLeft|Table.Flag_PackRight
+ );
+ for (const f of functions)
+ {
+ fn_table.add_row(f.name || "-", f.version || "-");
+ }
+ }
+
+ // Executables
+ const executables = desc.executables || [];
+ if (executables.length > 0)
+ {
+ const exec_section = section.add_section("Executables");
+ const exec_table = exec_section.add_widget(
+ Table, ["path", "hash", "size"], Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_AlignNumeric
+ );
+ let totalSize = 0;
+ for (const e of executables)
+ {
+ exec_table.add_row(e.name || "-", e.hash || "-", e.size != null ? Friendly.bytes(e.size) : "-");
+ totalSize += e.size || 0;
+ }
+ const total_row = exec_table.add_row("TOTAL", "", Friendly.bytes(totalSize));
+ total_row.get_cell(0).style("fontWeight", "bold");
+ total_row.get_cell(2).style("fontWeight", "bold");
+ }
+
+ // Files
+ const files = desc.files || [];
+ if (files.length > 0)
+ {
+ const files_section = section.add_section("Files");
+ const files_table = files_section.add_widget(
+ Table, ["name", "hash"], Table.Flag_FitLeft|Table.Flag_PackRight
+ );
+ for (const f of files)
+ {
+ files_table.add_row(typeof f === "string" ? f : (f.name || "-"), typeof f === "string" ? "" : (f.hash || ""));
+ }
+ }
+
+ // Directories
+ const dirs = desc.dirs || [];
+ if (dirs.length > 0)
+ {
+ const dirs_section = section.add_section("Directories");
+ for (const d of dirs)
+ {
+ dirs_section.tag().classify("detail-tag").text(d);
+ }
+ }
+
+ // Environment
+ const env = desc.environment || [];
+ if (env.length > 0)
+ {
+ const env_section = section.add_section("Environment");
+ for (const e of env)
+ {
+ env_section.tag().classify("detail-tag").text(e);
+ }
+ }
+ }
+
+ _render_queues(data)
+ {
+ const queues = data.queues || [];
+
+ if (this._queues_table)
+ {
+ this._queues_table.clear();
+ }
+ else
+ {
+ this._queues_table = this._queues_host.add_widget(
+ Table,
+ ["ID", "status", "active", "completed", "failed", "abandoned", "cancelled", "token"],
+ Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric, -1
+ );
+ }
+
+ for (const q of queues)
+ {
+ const id = q.queue_id != null ? String(q.queue_id) : "-";
+ const status = q.state === "cancelled" ? "cancelled"
+ : q.state === "draining" ? "draining"
+ : q.is_complete ? "complete" : "active";
+
+ this._queues_table.add_row(
+ id,
+ status,
+ String(q.active_count ?? 0),
+ String(q.completed_count ?? 0),
+ String(q.failed_count ?? 0),
+ String(q.abandoned_count ?? 0),
+ String(q.cancelled_count ?? 0),
+ q.queue_token || "-",
+ );
+ }
+ }
+
+ _render_action_history(data)
+ {
+ const entries = data.history || [];
+
+ if (this._history_table)
+ {
+ this._history_table.clear();
+ }
+ else
+ {
+ this._history_table = this._history_host.add_widget(
+ Table,
+ ["LSN", "queue", "status", "function", "started", "finished", "duration", "worker ID", "action ID"],
+ Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric, -1
+ );
+ }
+
+ // Entries arrive oldest-first; reverse to show newest at top
+ for (const entry of [...entries].reverse())
+ {
+ const lsn = entry.lsn != null ? String(entry.lsn) : "-";
+ const queueId = entry.queueId ? String(entry.queueId) : "-";
+ const status = entry.succeeded == null ? "unknown"
+ : entry.succeeded ? "ok" : "failed";
+ const desc = entry.actionDescriptor || {};
+ const fn = desc.Function || "-";
+ const startDate = filetimeToDate(entry.time_Running);
+ const endDate = filetimeToDate(entry.time_Completed ?? entry.time_Failed);
+
+ this._history_table.add_row(
+ lsn,
+ queueId,
+ status,
+ fn,
+ formatTime(startDate),
+ formatTime(endDate),
+ formatDuration(startDate, endDate),
+ entry.workerId || "-",
+ entry.actionId || "-",
+ );
+ }
+ }
+
+ _render_sysinfo(data)
+ {
+ const grid = this._sys_grid;
+ grid.inner().innerHTML = "";
+
+ // CPU card
+ {
+ const cpuUsage = data.cpu_usage || 0;
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("CPU Usage");
+ const body = tile.tag().classify("tile-metrics");
+ this._metric(body, cpuUsage.toFixed(1) + "%", "percent", true);
+
+ // Progress bar
+ const bar = body.tag().classify("progress-bar");
+ bar.tag().classify("progress-fill").style("width", cpuUsage + "%");
+
+ // CPU sparkline
+ this._history.cpu.push(cpuUsage);
+ while (this._history.cpu.length > MAX_HISTORY_POINTS) this._history.cpu.shift();
+ if (this._cpu_chart)
+ {
+ const sparkContainer = body.tag();
+ sparkContainer.style("position", "relative").style("height", "60px").style("marginTop", "12px");
+ sparkContainer.inner().appendChild(this._cpu_canvas);
+
+ this._cpu_chart.data.labels = this._history.cpu.map(() => "");
+ this._cpu_chart.data.datasets[0].data = this._history.cpu;
+ this._cpu_chart.update("none");
+ }
+
+ // CPU details
+ this._stat_row(body, "Packages", data.cpu_count != null ? String(data.cpu_count) : "-");
+ this._stat_row(body, "Physical Cores", data.core_count != null ? String(data.core_count) : "-");
+ this._stat_row(body, "Logical Processors", data.lp_count != null ? String(data.lp_count) : "-");
+ }
+
+ // Memory card
+ {
+ const memUsed = data.memory_used || 0;
+ const memTotal = data.memory_total || 1;
+ const memPercent = (memUsed / memTotal) * 100;
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Memory");
+ const body = tile.tag().classify("tile-metrics");
+ this._stat_row(body, "Used", Friendly.bytes(memUsed));
+ this._stat_row(body, "Total", Friendly.bytes(memTotal));
+ const bar = body.tag().classify("progress-bar");
+ bar.tag().classify("progress-fill").style("width", memPercent + "%");
+ }
+
+ // Disk card
+ {
+ const diskUsed = data.disk_used || 0;
+ const diskTotal = data.disk_total || 1;
+ const diskPercent = (diskUsed / diskTotal) * 100;
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Disk");
+ const body = tile.tag().classify("tile-metrics");
+ this._stat_row(body, "Used", Friendly.bytes(diskUsed));
+ this._stat_row(body, "Total", Friendly.bytes(diskTotal));
+ const bar = body.tag().classify("progress-bar");
+ bar.tag().classify("progress-fill").style("width", diskPercent + "%");
+ }
+ }
+
+ _stat_row(parent, label, value)
+ {
+ const row = parent.tag().classify("stats-row");
+ row.tag().classify("stats-label").text(label);
+ row.tag().classify("stats-value").text(value);
+ }
+
+ _metric(parent, value, label, hero = false)
+ {
+ const m = parent.tag().classify("tile-metric");
+ if (hero)
+ {
+ m.classify("tile-metric-hero");
+ }
+ m.tag().classify("metric-value").text(value);
+ m.tag().classify("metric-label").text(label);
+ }
+}
diff --git a/src/zenserver/frontend/html/pages/cookartifacts.js b/src/zenserver/frontend/html/pages/cookartifacts.js
new file mode 100644
index 000000000..f2ae094b9
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/cookartifacts.js
@@ -0,0 +1,397 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Table, Toolbar, PropTable } from "../util/widgets.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+ main()
+ {
+ this.set_title("cook artifacts");
+
+ const project = this.get_param("project");
+ const oplog = this.get_param("oplog");
+ const opkey = this.get_param("opkey");
+ const artifact_hash = this.get_param("hash");
+
+ // Fetch the artifact content as JSON
+ this._artifact = new Fetcher()
+ .resource("prj", project, "oplog", oplog, artifact_hash + ".json")
+ .json();
+
+ // Optionally fetch entry info for display context
+ if (opkey)
+ {
+ this._entry = new Fetcher()
+ .resource("prj", project, "oplog", oplog, "entries")
+ .param("opkey", opkey)
+ .cbo();
+ }
+
+ this._build_page();
+ }
+
+ // Map CookDependency enum values to display names
+ _get_dependency_type_name(type_value)
+ {
+ const type_names = {
+ 0: "None",
+ 1: "File",
+ 2: "Function",
+ 3: "TransitiveBuild",
+ 4: "Package",
+ 5: "ConsoleVariable",
+ 6: "Config",
+ 7: "SettingsObject",
+ 8: "NativeClass",
+ 9: "AssetRegistryQuery",
+ 10: "RedirectionTarget"
+ };
+ return type_names[type_value] || `Unknown (${type_value})`;
+ }
+
+ // Check if Data content should be expandable
+ _should_make_expandable(data_string)
+ {
+ if (!data_string || data_string.length < 40)
+ return false;
+
+ // Check if it's JSON array or object
+ if (!data_string.startsWith('[') && !data_string.startsWith('{'))
+ return false;
+
+ // Check if formatting would add newlines
+ try {
+ const parsed = JSON.parse(data_string);
+ const formatted = JSON.stringify(parsed, null, 2);
+ return formatted.includes('\n');
+ } catch (e) {
+ return false;
+ }
+ }
+
+ // Get first line of content for collapsed state
+ _get_first_line(data_string)
+ {
+ if (!data_string)
+ return "";
+
+ const newline_index = data_string.indexOf('\n');
+ if (newline_index === -1)
+ {
+ // No newline, truncate if too long
+ return data_string.length > 80 ? data_string.substring(0, 77) + "..." : data_string;
+ }
+ return data_string.substring(0, newline_index) + "...";
+ }
+
+ // Format JSON with indentation
+ _format_json(data_string)
+ {
+ try {
+ const parsed = JSON.parse(data_string);
+ return JSON.stringify(parsed, null, 2);
+ } catch (e) {
+ return data_string;
+ }
+ }
+
+ // Toggle expand/collapse state
+ _toggle_data_cell(cell)
+ {
+ const is_expanded = cell.attr("expanded") !== null;
+ const full_data = cell.attr("data-full");
+
+ // Find the text wrapper span
+ const text_wrapper = cell.first_child().next_sibling();
+
+ if (is_expanded)
+ {
+ // Collapse: show first line only
+ const first_line = this._get_first_line(full_data);
+ text_wrapper.text(first_line);
+ cell.attr("expanded", null);
+ }
+ else
+ {
+ // Expand: show formatted JSON
+ const formatted = this._format_json(full_data);
+ text_wrapper.text(formatted);
+ cell.attr("expanded", "");
+ }
+ }
+
+ // Format dependency data based on its structure
+ _format_dependency(dep_array)
+ {
+ const type = dep_array[0];
+ const formatted = {};
+
+ // Common patterns based on the example data:
+ // Type 2 (Function): [type, name, array, hash]
+ // Type 4 (Package): [type, path, hash]
+ // Type 5 (ConsoleVariable): [type, bool, array, hash]
+ // Type 8 (NativeClass): [type, path, hash]
+ // Type 9 (AssetRegistryQuery): [type, bool, object, hash]
+ // Type 10 (RedirectionTarget): [type, path, hash]
+
+ if (dep_array.length > 1)
+ {
+ // Most types have a name/path as second element
+ if (typeof dep_array[1] === "string")
+ {
+ formatted.Name = dep_array[1];
+ }
+ else if (typeof dep_array[1] === "boolean")
+ {
+ formatted.Value = dep_array[1].toString();
+ }
+ }
+
+ if (dep_array.length > 2)
+ {
+ // Third element varies
+ if (Array.isArray(dep_array[2]))
+ {
+ formatted.Data = JSON.stringify(dep_array[2]);
+ }
+ else if (typeof dep_array[2] === "object")
+ {
+ formatted.Data = JSON.stringify(dep_array[2]);
+ }
+ else if (typeof dep_array[2] === "string")
+ {
+ formatted.Hash = dep_array[2];
+ }
+ }
+
+ if (dep_array.length > 3)
+ {
+ // Fourth element is usually the hash
+ if (typeof dep_array[3] === "string")
+ {
+ formatted.Hash = dep_array[3];
+ }
+ }
+
+ return formatted;
+ }
+
+ async _build_page()
+ {
+ const project = this.get_param("project");
+ const oplog = this.get_param("oplog");
+ const opkey = this.get_param("opkey");
+ const artifact_hash = this.get_param("hash");
+
+ // Build page title
+ let title = "Cook Artifacts";
+ if (this._entry)
+ {
+ try
+ {
+ const entry = await this._entry;
+ const entry_obj = entry.as_object().find("entry").as_object();
+ const key = entry_obj.find("key").as_value();
+ title = `Cook Artifacts`;
+ }
+ catch (e)
+ {
+ console.error("Failed to fetch entry:", e);
+ }
+ }
+
+ const section = this.add_section(title);
+
+ // Fetch and parse artifact
+ let artifact;
+ try
+ {
+ artifact = await this._artifact;
+ }
+ catch (e)
+ {
+ section.text(`Failed to load artifact: ${e.message}`);
+ return;
+ }
+
+ // Display artifact info
+ const info_section = section.add_section("Artifact Info");
+ const info_table = info_section.add_widget(Table, ["Property", "Value"], Table.Flag_PackRight);
+
+ if (artifact.Version !== undefined)
+ info_table.add_row("Version", artifact.Version.toString());
+ if (artifact.HasSaveResults !== undefined)
+ info_table.add_row("HasSaveResults", artifact.HasSaveResults.toString());
+ if (artifact.PackageSavedHash !== undefined)
+ info_table.add_row("PackageSavedHash", artifact.PackageSavedHash);
+
+ // Process SaveBuildDependencies
+ if (artifact.SaveBuildDependencies && artifact.SaveBuildDependencies.Dependencies)
+ {
+ this._build_dependency_section(
+ section,
+ "Save Build Dependencies",
+ artifact.SaveBuildDependencies.Dependencies,
+ artifact.SaveBuildDependencies.StoredKey
+ );
+ }
+
+ // Process LoadBuildDependencies
+ if (artifact.LoadBuildDependencies && artifact.LoadBuildDependencies.Dependencies)
+ {
+ this._build_dependency_section(
+ section,
+ "Load Build Dependencies",
+ artifact.LoadBuildDependencies.Dependencies,
+ artifact.LoadBuildDependencies.StoredKey
+ );
+ }
+
+ // Process RuntimeDependencies
+ if (artifact.RuntimeDependencies && artifact.RuntimeDependencies.length > 0)
+ {
+ const runtime_section = section.add_section("Runtime Dependencies");
+ const runtime_table = runtime_section.add_widget(Table, ["Path"], Table.Flag_PackRight);
+ for (const dep of artifact.RuntimeDependencies)
+ {
+ const row = runtime_table.add_row(dep);
+ // Make Path clickable to navigate to entry
+ if (this._should_link_dependency(dep))
+ {
+ row.get_cell(0).text(dep).on_click((opkey) => {
+ window.location = `?page=entry&project=${project}&oplog=${oplog}&opkey=${opkey.toLowerCase()}`;
+ }, dep);
+ }
+ }
+ }
+ }
+
+ _should_link_dependency(name)
+ {
+ // Exclude dependencies starting with /Script/ (code-defined entries) - case insensitive
+ if (name && name.toLowerCase().startsWith("/script/"))
+ return false;
+
+ return true;
+ }
+
+ _build_dependency_section(parent_section, title, dependencies, stored_key)
+ {
+ const section = parent_section.add_section(title);
+
+ // Add stored key info
+ if (stored_key)
+ {
+ const key_toolbar = section.add_widget(Toolbar);
+ key_toolbar.left().add(`Key: ${stored_key}`);
+ }
+
+ // Group dependencies by type
+ const dependencies_by_type = {};
+
+ for (const dep_array of dependencies)
+ {
+ if (!Array.isArray(dep_array) || dep_array.length === 0)
+ continue;
+
+ const type = dep_array[0];
+ if (!dependencies_by_type[type])
+ dependencies_by_type[type] = [];
+
+ dependencies_by_type[type].push(this._format_dependency(dep_array));
+ }
+
+ // Sort types numerically
+ const sorted_types = Object.keys(dependencies_by_type).map(Number).sort((a, b) => a - b);
+
+ for (const type_value of sorted_types)
+ {
+ const type_name = this._get_dependency_type_name(type_value);
+ const deps = dependencies_by_type[type_value];
+
+ const type_section = section.add_section(type_name);
+
+ // Determine columns based on available fields
+ const all_fields = new Set();
+ for (const dep of deps)
+ {
+ for (const field in dep)
+ all_fields.add(field);
+ }
+ let columns = Array.from(all_fields);
+
+ // Remove Hash column for RedirectionTarget as it's not useful
+ if (type_value === 10)
+ {
+ columns = columns.filter(col => col !== "Hash");
+ }
+
+ if (columns.length === 0)
+ {
+ type_section.text("No data fields");
+ continue;
+ }
+
+ // Create table with dynamic columns
+ const table = type_section.add_widget(Table, columns, Table.Flag_PackRight);
+
+ // Check if this type should have clickable Name links
+ const should_link = (type_value === 3 || type_value === 4 || type_value === 10);
+ const name_col_index = columns.indexOf("Name");
+
+ for (const dep of deps)
+ {
+ const row_values = columns.map(col => dep[col] || "");
+ const row = table.add_row(...row_values);
+
+ // Make Name field clickable for Package, TransitiveBuild, and RedirectionTarget
+ if (should_link && name_col_index >= 0 && dep.Name && this._should_link_dependency(dep.Name))
+ {
+ const project = this.get_param("project");
+ const oplog = this.get_param("oplog");
+ row.get_cell(name_col_index).text(dep.Name).on_click((opkey) => {
+ window.location = `?page=entry&project=${project}&oplog=${oplog}&opkey=${opkey.toLowerCase()}`;
+ }, dep.Name);
+ }
+
+ // Make Data field expandable/collapsible if needed
+ const data_col_index = columns.indexOf("Data");
+ if (data_col_index >= 0 && dep.Data)
+ {
+ const data_cell = row.get_cell(data_col_index);
+
+ if (this._should_make_expandable(dep.Data))
+ {
+ // Store full data in attribute
+ data_cell.attr("data-full", dep.Data);
+
+ // Clear the cell and rebuild with icon + text
+ data_cell.inner().innerHTML = "";
+
+ // Create expand/collapse icon
+ const icon = data_cell.tag("span").classify("zen_expand_icon").text("+");
+ icon.on_click(() => {
+ this._toggle_data_cell(data_cell);
+ // Update icon text
+ const is_expanded = data_cell.attr("expanded") !== null;
+ icon.text(is_expanded ? "-" : "+");
+ });
+
+ // Add text content wrapper
+ const text_wrapper = data_cell.tag("span").classify("zen_data_text");
+ const first_line = this._get_first_line(dep.Data);
+ text_wrapper.text(first_line);
+
+ // Store reference to text wrapper for updates
+ data_cell.attr("data-text-wrapper", "true");
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js
index 08589b090..1e4c82e3f 100644
--- a/src/zenserver/frontend/html/pages/entry.js
+++ b/src/zenserver/frontend/html/pages/entry.js
@@ -26,6 +26,9 @@ export class Page extends ZenPage
this._indexer = this.load_indexer(project, oplog);
+ this._files_index_start = Number(this.get_param("files_start", 0)) || 0;
+ this._files_index_count = Number(this.get_param("files_count", 50)) || 0;
+
this._build_page();
}
@@ -40,25 +43,39 @@ export class Page extends ZenPage
return indexer;
}
- async _build_deps(section, tree)
+ _build_deps(section, tree)
{
- const indexer = await this._indexer;
+ const project = this.get_param("project");
+ const oplog = this.get_param("oplog");
for (const dep_name in tree)
{
const dep_section = section.add_section(dep_name);
const table = dep_section.add_widget(Table, ["name", "id"], Table.Flag_PackRight);
+
for (const dep_id of tree[dep_name])
{
- const cell_values = ["", dep_id.toString(16).padStart(16, "0")];
+ const hex_id = dep_id.toString(16).padStart(16, "0");
+ const cell_values = ["loading...", hex_id];
const row = table.add_row(...cell_values);
- var opkey = indexer.lookup_id(dep_id);
- row.get_cell(0).text(opkey).on_click((k) => this.view_opkey(k), opkey);
+ // Asynchronously resolve the name
+ this._resolve_dep_name(row.get_cell(0), dep_id, project, oplog);
}
}
}
+ async _resolve_dep_name(cell, dep_id, project, oplog)
+ {
+ const indexer = await this._indexer;
+ const opkey = indexer.lookup_id(dep_id);
+
+ if (opkey)
+ {
+ cell.text(opkey).on_click((k) => this.view_opkey(k), opkey);
+ }
+ }
+
_find_iohash_field(container, name)
{
const found_field = container.find(name);
@@ -76,6 +93,21 @@ export class Page extends ZenPage
return null;
}
+ _is_null_io_hash_string(io_hash)
+ {
+ if (!io_hash)
+ return true;
+
+ for (let char of io_hash)
+ {
+ if (char != '0')
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+
async _build_meta(section, entry)
{
var tree = {}
@@ -123,11 +155,23 @@ export class Page extends ZenPage
const project = this.get_param("project");
const oplog = this.get_param("oplog");
+ const opkey = this.get_param("opkey");
const link = row.get_cell(0).link(
- "/" + ["prj", project, "oplog", oplog, value+".json"].join("/")
+ (key === "cook.artifacts") ?
+ `?page=cookartifacts&project=${project}&oplog=${oplog}&opkey=${opkey}&hash=${value}`
+ : "/" + ["prj", project, "oplog", oplog, value+".json"].join("/")
);
const action_tb = new Toolbar(row.get_cell(-1), true);
+
+ // Add "view-raw" button for cook.artifacts
+ if (key === "cook.artifacts")
+ {
+ action_tb.left().add("view-raw").on_click(() => {
+ window.location = "/" + ["prj", project, "oplog", oplog, value+".json"].join("/");
+ });
+ }
+
action_tb.left().add("copy-hash").on_click(async (v) => {
await navigator.clipboard.writeText(v);
}, value);
@@ -137,35 +181,55 @@ export class Page extends ZenPage
async _build_page()
{
var entry = await this._entry;
+
+ // Check if entry exists
+ if (!entry || entry.as_object().find("entry") == null)
+ {
+ const opkey = this.get_param("opkey");
+ var section = this.add_section("Entry Not Found");
+ section.tag("p").text(`The entry "${opkey}" is not present in this dataset.`);
+ section.tag("p").text("This could mean:");
+ const list = section.tag("ul");
+ list.tag("li").text("The entry is for an instance defined in code");
+ list.tag("li").text("The entry has not been added to the oplog yet");
+ list.tag("li").text("The entry key is misspelled");
+ list.tag("li").text("The entry was removed or never existed");
+ return;
+ }
+
entry = entry.as_object().find("entry").as_object();
const name = entry.find("key").as_value();
var section = this.add_section(name);
+ var has_package_data = false;
// tree
{
var tree = entry.find("$tree");
if (tree == undefined)
tree = this._convert_legacy_to_tree(entry);
- if (tree == undefined)
- return this._display_unsupported(section, entry);
-
- delete tree["$id"];
-
- if (Object.keys(tree).length != 0)
+ if (tree != undefined)
{
- const sub_section = section.add_section("deps");
- this._build_deps(sub_section, tree);
+ delete tree["$id"];
+
+ if (Object.keys(tree).length != 0)
+ {
+ const sub_section = section.add_section("dependencies");
+ this._build_deps(sub_section, tree);
+ }
+ has_package_data = true;
}
}
// meta
+ if (has_package_data)
{
this._build_meta(section, entry);
}
// data
+ if (has_package_data)
{
const sub_section = section.add_section("data");
const table = sub_section.add_widget(
@@ -181,7 +245,7 @@ export class Page extends ZenPage
for (const item of pkg_data.as_array())
{
- var io_hash, size, raw_size, file_name;
+ var io_hash = undefined, size = undefined, raw_size = undefined, file_name = undefined;
for (const field of item.as_object())
{
if (field.is_named("data")) io_hash = field.as_value();
@@ -198,8 +262,8 @@ export class Page extends ZenPage
io_hash = ret;
}
- size = (size !== undefined) ? Friendly.kib(size) : "";
- raw_size = (raw_size !== undefined) ? Friendly.kib(raw_size) : "";
+ size = (size !== undefined) ? Friendly.bytes(size) : "";
+ raw_size = (raw_size !== undefined) ? Friendly.bytes(raw_size) : "";
const row = table.add_row(file_name, size, raw_size);
@@ -219,12 +283,76 @@ export class Page extends ZenPage
}
}
+ // files
+ var has_file_data = false;
+ {
+ var file_data = entry.find("files");
+ if (file_data != undefined)
+ {
+ has_file_data = true;
+
+ // Extract files into array
+ this._files_data = [];
+ for (const item of file_data.as_array())
+ {
+ var io_hash = undefined, cid = undefined, server_path = undefined, client_path = undefined;
+ for (const field of item.as_object())
+ {
+ if (field.is_named("data")) io_hash = field.as_value();
+ else if (field.is_named("id")) cid = field.as_value();
+ else if (field.is_named("serverpath")) server_path = field.as_value();
+ else if (field.is_named("clientpath")) client_path = field.as_value();
+ }
+
+ if (io_hash instanceof Uint8Array)
+ {
+ var ret = "";
+ for (var x of io_hash)
+ ret += x.toString(16).padStart(2, "0");
+ io_hash = ret;
+ }
+
+ if (cid instanceof Uint8Array)
+ {
+ var ret = "";
+ for (var x of cid)
+ ret += x.toString(16).padStart(2, "0");
+ cid = ret;
+ }
+
+ this._files_data.push({
+ server_path: server_path,
+ client_path: client_path,
+ io_hash: io_hash,
+ cid: cid
+ });
+ }
+
+ this._files_index_max = this._files_data.length;
+
+ const sub_section = section.add_section("files");
+ this._build_files_nav(sub_section);
+
+ this._files_table = sub_section.add_widget(
+ Table,
+ ["name", "actions"], Table.Flag_PackRight
+ );
+ this._files_table.id("filetable");
+
+ this._build_files_table(this._files_index_start);
+ }
+ }
+
// props
+ if (has_package_data)
{
const object = entry.to_js_object();
var sub_section = section.add_section("props");
sub_section.add_widget(PropTable).add_object(object);
}
+
+ if (!has_package_data && !has_file_data)
+ return this._display_unsupported(section, entry);
}
_display_unsupported(section, entry)
@@ -271,16 +399,30 @@ export class Page extends ZenPage
for (const field of pkgst_entry)
{
const field_name = field.get_name();
- if (!field_name.endsWith("importedpackageids"))
- continue;
-
- var dep_name = field_name.slice(0, -18);
- if (dep_name.length == 0)
- dep_name = "imported";
-
- var out = tree[dep_name] = [];
- for (var item of field.as_array())
- out.push(item.as_value(BigInt));
+ if (field_name.endsWith("importedpackageids"))
+ {
+ var dep_name = field_name.slice(0, -18);
+ if (dep_name.length == 0)
+ dep_name = "hard";
+ else
+ dep_name = "hard." + dep_name;
+
+ var out = tree[dep_name] = [];
+ for (var item of field.as_array())
+ out.push(item.as_value(BigInt));
+ }
+ else if (field_name.endsWith("softpackagereferences"))
+ {
+ var dep_name = field_name.slice(0, -21);
+ if (dep_name.length == 0)
+ dep_name = "soft";
+ else
+ dep_name = "soft." + dep_name;
+
+ var out = tree[dep_name] = [];
+ for (var item of field.as_array())
+ out.push(item.as_value(BigInt));
+ }
}
return tree;
@@ -292,4 +434,149 @@ export class Page extends ZenPage
params.set("opkey", opkey);
window.location.search = params;
}
+
+ _build_files_nav(section)
+ {
+ const nav = section.add_widget(Toolbar);
+ const left = nav.left();
+ left.add("|<") .on_click(() => this._on_files_next_prev(-10e10));
+ left.add("<<") .on_click(() => this._on_files_next_prev(-10));
+ left.add("prev").on_click(() => this._on_files_next_prev( -1));
+ left.add("next").on_click(() => this._on_files_next_prev( 1));
+ left.add(">>") .on_click(() => this._on_files_next_prev( 10));
+ left.add(">|") .on_click(() => this._on_files_next_prev( 10e10));
+
+ left.sep();
+ for (var count of [10, 25, 50, 100])
+ {
+ var handler = (n) => this._on_files_change_count(n);
+ left.add(count).on_click(handler, count);
+ }
+
+ const right = nav.right();
+ right.add(Friendly.sep(this._files_index_max));
+
+ right.sep();
+ var search_input = right.add("search:", "label").tag("input");
+ search_input.on("change", (x) => this._search_files(x.inner().value), search_input);
+ }
+
+ _build_files_table(index)
+ {
+ this._files_index_count = Math.max(this._files_index_count, 1);
+ index = Math.min(index, this._files_index_max - this._files_index_count);
+ index = Math.max(index, 0);
+
+ const project = this.get_param("project");
+ const oplog = this.get_param("oplog");
+
+ const end_index = Math.min(index + this._files_index_count, this._files_index_max);
+
+ this._files_table.clear(index);
+ for (var i = index; i < end_index; i++)
+ {
+ const file_item = this._files_data[i];
+ const row = this._files_table.add_row(file_item.server_path);
+
+ var base_name = file_item.server_path.split("/").pop().split("\\").pop();
+ if (this._is_null_io_hash_string(file_item.io_hash))
+ {
+ const link = row.get_cell(0).link(
+ "/" + ["prj", project, "oplog", oplog, file_item.cid].join("/")
+ );
+ link.first_child().attr("download", `${file_item.cid}_${base_name}`);
+
+ const action_tb = new Toolbar(row.get_cell(-1), true);
+ action_tb.left().add("copy-id").on_click(async (v) => {
+ await navigator.clipboard.writeText(v);
+ }, file_item.cid);
+ }
+ else
+ {
+ const link = row.get_cell(0).link(
+ "/" + ["prj", project, "oplog", oplog, file_item.io_hash].join("/")
+ );
+ link.first_child().attr("download", `${file_item.io_hash}_${base_name}`);
+
+ const action_tb = new Toolbar(row.get_cell(-1), true);
+ action_tb.left().add("copy-hash").on_click(async (v) => {
+ await navigator.clipboard.writeText(v);
+ }, file_item.io_hash);
+ }
+ }
+
+ this.set_param("files_start", index);
+ this.set_param("files_count", this._files_index_count);
+ this._files_index_start = index;
+ }
+
+ _on_files_change_count(value)
+ {
+ this._files_index_count = parseInt(value);
+ this._build_files_table(this._files_index_start);
+ }
+
+ _on_files_next_prev(direction)
+ {
+ var index = this._files_index_start + (this._files_index_count * direction);
+ index = Math.max(0, index);
+ this._build_files_table(index);
+ }
+
+ _search_files(needle)
+ {
+ if (needle.length == 0)
+ {
+ this._build_files_table(this._files_index_start);
+ return;
+ }
+ needle = needle.trim().toLowerCase();
+
+ this._files_table.clear(this._files_index_start);
+
+ const project = this.get_param("project");
+ const oplog = this.get_param("oplog");
+
+ var added = 0;
+ const truncate_at = this.get_param("searchmax") || 250;
+ for (const file_item of this._files_data)
+ {
+ if (!file_item.server_path.toLowerCase().includes(needle))
+ continue;
+
+ const row = this._files_table.add_row(file_item.server_path);
+
+ var base_name = file_item.server_path.split("/").pop().split("\\").pop();
+ if (this._is_null_io_hash_string(file_item.io_hash))
+ {
+ const link = row.get_cell(0).link(
+ "/" + ["prj", project, "oplog", oplog, file_item.cid].join("/")
+ );
+ link.first_child().attr("download", `${file_item.cid}_${base_name}`);
+
+ const action_tb = new Toolbar(row.get_cell(-1), true);
+ action_tb.left().add("copy-id").on_click(async (v) => {
+ await navigator.clipboard.writeText(v);
+ }, file_item.cid);
+ }
+ else
+ {
+ const link = row.get_cell(0).link(
+ "/" + ["prj", project, "oplog", oplog, file_item.io_hash].join("/")
+ );
+ link.first_child().attr("download", `${file_item.io_hash}_${base_name}`);
+
+ const action_tb = new Toolbar(row.get_cell(-1), true);
+ action_tb.left().add("copy-hash").on_click(async (v) => {
+ await navigator.clipboard.writeText(v);
+ }, file_item.io_hash);
+ }
+
+ if (++added >= truncate_at)
+ {
+ this._files_table.add_row("...truncated");
+ break;
+ }
+ }
+ }
}
diff --git a/src/zenserver/frontend/html/pages/hub.js b/src/zenserver/frontend/html/pages/hub.js
new file mode 100644
index 000000000..f9e4fff33
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/hub.js
@@ -0,0 +1,122 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
+import { Table } from "../util/widgets.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+ async main()
+ {
+ this.set_title("hub");
+
+ // Capacity
+ const stats_section = this.add_section("Capacity");
+ this._stats_grid = stats_section.tag().classify("grid").classify("stats-tiles");
+
+ // Modules
+ const mod_section = this.add_section("Modules");
+ this._mod_host = mod_section;
+ this._mod_table = null;
+
+ await this._update();
+ this._poll_timer = setInterval(() => this._update(), 2000);
+ }
+
+ async _update()
+ {
+ try
+ {
+ const [stats, status] = await Promise.all([
+ new Fetcher().resource("/hub/stats").json(),
+ new Fetcher().resource("/hub/status").json(),
+ ]);
+
+ this._render_capacity(stats);
+ this._render_modules(status);
+ }
+ catch (e) { /* service unavailable */ }
+ }
+
+ _render_capacity(data)
+ {
+ const grid = this._stats_grid;
+ grid.inner().innerHTML = "";
+
+ const current = data.currentInstanceCount || 0;
+ const max = data.maxInstanceCount || 0;
+ const limit = data.instanceLimit || 0;
+
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Active Modules");
+ const body = tile.tag().classify("tile-metrics");
+ this._metric(body, Friendly.sep(current), "currently provisioned", true);
+ }
+
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Peak Modules");
+ const body = tile.tag().classify("tile-metrics");
+ this._metric(body, Friendly.sep(max), "high watermark", true);
+ }
+
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Instance Limit");
+ const body = tile.tag().classify("tile-metrics");
+ this._metric(body, Friendly.sep(limit), "maximum allowed", true);
+ if (limit > 0)
+ {
+ const pct = ((current / limit) * 100).toFixed(0) + "%";
+ this._metric(body, pct, "utilization");
+ }
+ }
+ }
+
+ _render_modules(data)
+ {
+ const modules = data.modules || [];
+
+ if (this._mod_table)
+ {
+ this._mod_table.clear();
+ }
+ else
+ {
+ this._mod_table = this._mod_host.add_widget(
+ Table,
+ ["module ID", "status"],
+ Table.Flag_FitLeft|Table.Flag_PackRight
+ );
+ }
+
+ if (modules.length === 0)
+ {
+ return;
+ }
+
+ for (const m of modules)
+ {
+ this._mod_table.add_row(
+ m.moduleId || "",
+ m.provisioned ? "provisioned" : "inactive",
+ );
+ }
+ }
+
+ _metric(parent, value, label, hero = false)
+ {
+ const m = parent.tag().classify("tile-metric");
+ if (hero)
+ {
+ m.classify("tile-metric-hero");
+ }
+ m.tag().classify("metric-value").text(value);
+ m.tag().classify("metric-label").text(label);
+ }
+}
diff --git a/src/zenserver/frontend/html/pages/info.js b/src/zenserver/frontend/html/pages/info.js
new file mode 100644
index 000000000..f92765c78
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/info.js
@@ -0,0 +1,261 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+ async main()
+ {
+ this.set_title("info");
+
+ const [info, gc, services, version] = await Promise.all([
+ new Fetcher().resource("/health/info").json(),
+ new Fetcher().resource("/admin/gc").json().catch(() => null),
+ new Fetcher().resource("/api/").json().catch(() => ({})),
+ new Fetcher().resource("/health/version").param("detailed", "true").text(),
+ ]);
+
+ const section = this.add_section("Server Info");
+ const grid = section.tag().classify("grid").classify("info-tiles");
+
+ // Application
+ {
+ const tile = grid.tag().classify("card").classify("info-tile");
+ tile.tag().classify("card-title").text("Application");
+ const list = tile.tag().classify("info-props");
+
+ this._prop(list, "version", version || info.BuildVersion || "-");
+ this._prop(list, "http server", info.HttpServerClass || "-");
+ this._prop(list, "port", info.Port || "-");
+ this._prop(list, "pid", info.Pid || "-");
+ this._prop(list, "dedicated", info.IsDedicated ? "yes" : "no");
+
+ if (info.StartTimeMs)
+ {
+ const start = new Date(info.StartTimeMs);
+ const elapsed = Date.now() - info.StartTimeMs;
+ this._prop(list, "started", start.toLocaleString());
+ this._prop(list, "uptime", this._format_duration(elapsed));
+ }
+
+ this._prop(list, "data root", info.DataRoot || "-");
+ this._prop(list, "log path", info.AbsLogPath || "-");
+ }
+
+ // System
+ {
+ const tile = grid.tag().classify("card").classify("info-tile");
+ tile.tag().classify("card-title").text("System");
+ const list = tile.tag().classify("info-props");
+
+ this._prop(list, "hostname", info.Hostname || "-");
+ this._prop(list, "platform", info.Platform || "-");
+ this._prop(list, "os", info.OS || "-");
+ this._prop(list, "arch", info.Arch || "-");
+
+ const sys = info.System;
+ if (sys)
+ {
+ this._prop(list, "cpus", sys.cpu_count || "-");
+ this._prop(list, "cores", sys.core_count || "-");
+ this._prop(list, "logical processors", sys.lp_count || "-");
+ this._prop(list, "total memory", sys.total_memory_mb ? Friendly.bytes(sys.total_memory_mb * 1048576) : "-");
+ this._prop(list, "available memory", sys.avail_memory_mb ? Friendly.bytes(sys.avail_memory_mb * 1048576) : "-");
+ if (sys.uptime_seconds)
+ {
+ this._prop(list, "system uptime", this._format_duration(sys.uptime_seconds * 1000));
+ }
+ }
+ }
+
+ // Runtime Configuration
+ if (info.RuntimeConfig)
+ {
+ const tile = grid.tag().classify("card").classify("info-tile");
+ tile.tag().classify("card-title").text("Runtime Configuration");
+ const list = tile.tag().classify("info-props");
+
+ for (const key in info.RuntimeConfig)
+ {
+ this._prop(list, key, info.RuntimeConfig[key] || "-");
+ }
+ }
+
+ // Build Configuration
+ if (info.BuildConfig)
+ {
+ const tile = grid.tag().classify("card").classify("info-tile");
+ tile.tag().classify("card-title").text("Build Configuration");
+ const list = tile.tag().classify("info-props");
+
+ for (const key in info.BuildConfig)
+ {
+ this._prop(list, key, info.BuildConfig[key] ? "yes" : "no");
+ }
+ }
+
+ // Services
+ {
+ const tile = grid.tag().classify("card").classify("info-tile");
+ tile.tag().classify("card-title").text("Services");
+ const list = tile.tag().classify("info-props");
+
+ const svc_list = (services.services || []).map(s => s.base_uri).sort();
+ for (const uri of svc_list)
+ {
+ this._prop(list, uri, "registered");
+ }
+ }
+
+ // Garbage Collection
+ if (gc)
+ {
+ const tile = grid.tag().classify("card").classify("info-tile");
+ tile.tag().classify("card-title").text("Garbage Collection");
+ const list = tile.tag().classify("info-props");
+
+ this._prop(list, "status", gc.Status || "-");
+
+ if (gc.AreDiskWritesBlocked !== undefined)
+ {
+ this._prop(list, "disk writes blocked", gc.AreDiskWritesBlocked ? "yes" : "no");
+ }
+
+ if (gc.DiskSize)
+ {
+ this._prop(list, "disk size", gc.DiskSize);
+ this._prop(list, "disk used", gc.DiskUsed);
+ this._prop(list, "disk free", gc.DiskFree);
+ }
+
+ const cfg = gc.Config;
+ if (cfg)
+ {
+ this._prop(list, "gc enabled", cfg.Enabled ? "yes" : "no");
+ if (cfg.Interval)
+ {
+ this._prop(list, "interval", this._friendly_duration(cfg.Interval));
+ }
+ if (cfg.LightweightInterval)
+ {
+ this._prop(list, "lightweight interval", this._friendly_duration(cfg.LightweightInterval));
+ }
+ if (cfg.MaxCacheDuration)
+ {
+ this._prop(list, "max cache duration", this._friendly_duration(cfg.MaxCacheDuration));
+ }
+ if (cfg.MaxProjectStoreDuration)
+ {
+ this._prop(list, "max project duration", this._friendly_duration(cfg.MaxProjectStoreDuration));
+ }
+ if (cfg.MaxBuildStoreDuration)
+ {
+ this._prop(list, "max build duration", this._friendly_duration(cfg.MaxBuildStoreDuration));
+ }
+ }
+
+ if (gc.FullGC)
+ {
+ if (gc.FullGC.LastTime)
+ {
+ this._prop(list, "last full gc", this._friendly_timestamp(gc.FullGC.LastTime));
+ }
+ if (gc.FullGC.TimeToNext)
+ {
+ this._prop(list, "next full gc", this._friendly_duration(gc.FullGC.TimeToNext));
+ }
+ }
+
+ if (gc.LightweightGC)
+ {
+ if (gc.LightweightGC.LastTime)
+ {
+ this._prop(list, "last lightweight gc", this._friendly_timestamp(gc.LightweightGC.LastTime));
+ }
+ if (gc.LightweightGC.TimeToNext)
+ {
+ this._prop(list, "next lightweight gc", this._friendly_duration(gc.LightweightGC.TimeToNext));
+ }
+ }
+ }
+ }
+
+ _prop(parent, label, value)
+ {
+ const row = parent.tag().classify("info-prop");
+ row.tag().classify("info-prop-label").text(label);
+ const val = row.tag().classify("info-prop-value");
+ const str = String(value);
+ if (str.match(/^[A-Za-z]:[\\/]/) || str.startsWith("/"))
+ {
+ val.tag("a").text(str).attr("href", "vscode://" + str.replace(/\\/g, "/"));
+ }
+ else
+ {
+ val.text(str);
+ }
+ }
+
+ _friendly_timestamp(value)
+ {
+ const d = new Date(value);
+ if (isNaN(d.getTime()))
+ {
+ return String(value);
+ }
+ return d.toLocaleString(undefined, {
+ year: "numeric", month: "short", day: "numeric",
+ hour: "2-digit", minute: "2-digit", second: "2-digit",
+ });
+ }
+
+ _friendly_duration(value)
+ {
+ if (typeof value === "number")
+ {
+ return this._format_duration(value);
+ }
+
+ const str = String(value);
+ const match = str.match(/^[+-]?(?:(\d+)\.)?(\d+):(\d+):(\d+)(?:\.(\d+))?$/);
+ if (!match)
+ {
+ return str;
+ }
+
+ const days = parseInt(match[1] || "0", 10);
+ const hours = parseInt(match[2], 10);
+ const minutes = parseInt(match[3], 10);
+ const seconds = parseInt(match[4], 10);
+ const total_seconds = days * 86400 + hours * 3600 + minutes * 60 + seconds;
+
+ return this._format_duration(total_seconds * 1000);
+ }
+
+ _format_duration(ms)
+ {
+ const seconds = Math.floor(ms / 1000);
+ const minutes = Math.floor(seconds / 60);
+ const hours = Math.floor(minutes / 60);
+ const days = Math.floor(hours / 24);
+
+ if (days > 0)
+ {
+ return `${days}d ${hours % 24}h ${minutes % 60}m`;
+ }
+ if (hours > 0)
+ {
+ return `${hours}h ${minutes % 60}m`;
+ }
+ if (minutes > 0)
+ {
+ return `${minutes}m ${seconds % 60}s`;
+ }
+ return `${seconds}s`;
+ }
+}
diff --git a/src/zenserver/frontend/html/pages/map.js b/src/zenserver/frontend/html/pages/map.js
index 58046b255..ac8f298aa 100644
--- a/src/zenserver/frontend/html/pages/map.js
+++ b/src/zenserver/frontend/html/pages/map.js
@@ -116,9 +116,9 @@ export class Page extends ZenPage
for (const name of sorted_keys)
nodes.push(new_nodes[name] / branch_size);
- var stats = Friendly.kib(branch_size);
+ var stats = Friendly.bytes(branch_size);
stats += " / ";
- stats += Friendly.kib(total_size);
+ stats += Friendly.bytes(total_size);
stats += " (";
stats += 0|((branch_size * 100) / total_size);
stats += "%)";
diff --git a/src/zenserver/frontend/html/pages/metrics.js b/src/zenserver/frontend/html/pages/metrics.js
new file mode 100644
index 000000000..e7a2eca67
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/metrics.js
@@ -0,0 +1,232 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
+import { PropTable, Toolbar } from "../util/widgets.js"
+
+////////////////////////////////////////////////////////////////////////////////
+class TemporalStat
+{
+ constructor(data, as_bytes)
+ {
+ this._data = data;
+ this._as_bytes = as_bytes;
+ }
+
+ toString()
+ {
+ const columns = [
+ /* count */ {},
+ /* rate */ {},
+ /* t */ {}, {},
+ ];
+ const data = this._data;
+ for (var key in data)
+ {
+ var out = columns[0];
+ if (key.startsWith("rate_")) out = columns[1];
+ else if (key.startsWith("t_p")) out = columns[3];
+ else if (key.startsWith("t_")) out = columns[2];
+ out[key] = data[key];
+ }
+
+ var friendly = this._as_bytes ? Friendly.bytes : Friendly.sep;
+
+ var content = "";
+ for (var i = 0; i < columns.length; ++i)
+ {
+ const column = columns[i];
+ for (var key in column)
+ {
+ var value = column[key];
+ if (i)
+ {
+ value = Friendly.sep(value, 2);
+ key = key.padStart(9);
+ content += key + ": " + value;
+ }
+ else
+ content += friendly(value);
+ content += "\r\n";
+ }
+ }
+
+ return content;
+ }
+
+ tag()
+ {
+ return "pre";
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+ async main()
+ {
+ this.set_title("metrics");
+
+ const metrics_section = this.add_section("metrics");
+ const top_toolbar = metrics_section.add_widget(Toolbar);
+ const tb_right = top_toolbar.right();
+ this._refresh_label = tb_right.add("", "label");
+ this._pause_btn = tb_right.add("pause").on_click(() => this._toggle_pause());
+
+ this._paused = false;
+ this._last_refresh = Date.now();
+ this._provider_views = [];
+
+ const providers_data = await new Fetcher().resource("stats").json();
+ const providers = providers_data["providers"] || [];
+
+ const stats_list = await Promise.all(providers.map((provider) =>
+ new Fetcher()
+ .resource("stats", provider)
+ .param("cidstorestats", "true")
+ .param("cachestorestats", "true")
+ .json()
+ .then((stats) => ({ provider, stats }))
+ ));
+
+ for (const { provider, stats } of stats_list)
+ {
+ this._condense(stats);
+ this._provider_views.push(this._render_provider(provider, stats));
+ }
+
+ this._last_refresh = Date.now();
+ this._update_refresh_label();
+
+ this._timer_id = setInterval(() => this._refresh(), 5000);
+ this._tick_id = setInterval(() => this._update_refresh_label(), 1000);
+
+ document.addEventListener("visibilitychange", () => {
+ if (document.hidden)
+ this._pause_timer(false);
+ else if (!this._paused)
+ this._resume_timer();
+ });
+ }
+
+ _render_provider(provider, stats)
+ {
+ const section = this.add_section(provider);
+ const toolbar = section.add_widget(Toolbar);
+
+ toolbar.right().add("detailed →").on_click(() => {
+ window.location = "?page=stat&provider=" + provider;
+ });
+
+ const table = section.add_widget(PropTable);
+ let current_stats = stats;
+ let current_category = undefined;
+
+ const show_category = (cat) => {
+ current_category = cat;
+ table.clear();
+ table.add_object(current_stats[cat], true, 3);
+ };
+
+ var first = undefined;
+ for (var name in stats)
+ {
+ first = first || name;
+ toolbar.left().add(name).on_click(show_category, name);
+ }
+
+ if (first)
+ show_category(first);
+
+ return {
+ provider,
+ set_stats: (new_stats) => {
+ current_stats = new_stats;
+ if (current_category && current_stats[current_category])
+ show_category(current_category);
+ },
+ };
+ }
+
+ async _refresh()
+ {
+ const updates = await Promise.all(this._provider_views.map((view) =>
+ new Fetcher()
+ .resource("stats", view.provider)
+ .param("cidstorestats", "true")
+ .param("cachestorestats", "true")
+ .json()
+ .then((stats) => ({ view, stats }))
+ ));
+
+ for (const { view, stats } of updates)
+ {
+ this._condense(stats);
+ view.set_stats(stats);
+ }
+
+ this._last_refresh = Date.now();
+ this._update_refresh_label();
+ }
+
+ _update_refresh_label()
+ {
+ const elapsed = Math.floor((Date.now() - this._last_refresh) / 1000);
+ this._refresh_label.inner().textContent = "refreshed " + elapsed + "s ago";
+ }
+
+ _toggle_pause()
+ {
+ if (this._paused)
+ this._resume_timer();
+ else
+ this._pause_timer(true);
+ }
+
+ _pause_timer(user_paused=true)
+ {
+ clearInterval(this._timer_id);
+ this._timer_id = undefined;
+ if (user_paused)
+ {
+ this._paused = true;
+ this._pause_btn.inner().textContent = "resume";
+ }
+ }
+
+ _resume_timer()
+ {
+ this._paused = false;
+ this._pause_btn.inner().textContent = "pause";
+ this._timer_id = setInterval(() => this._refresh(), 5000);
+ this._refresh();
+ }
+
+ _condense(stats)
+ {
+ const impl = function(node)
+ {
+ for (var name in node)
+ {
+ const candidate = node[name];
+ if (!(candidate instanceof Object))
+ continue;
+
+ if (candidate["rate_mean"] != undefined)
+ {
+ const as_bytes = (name.indexOf("bytes") >= 0);
+ node[name] = new TemporalStat(candidate, as_bytes);
+ continue;
+ }
+
+ impl(candidate);
+ }
+ }
+
+ for (var name in stats)
+ impl(stats[name]);
+ }
+}
diff --git a/src/zenserver/frontend/html/pages/oplog.js b/src/zenserver/frontend/html/pages/oplog.js
index 879fc4c97..fb857affb 100644
--- a/src/zenserver/frontend/html/pages/oplog.js
+++ b/src/zenserver/frontend/html/pages/oplog.js
@@ -32,7 +32,7 @@ export class Page extends ZenPage
this.set_title("oplog - " + oplog);
- var section = this.add_section(project + " - " + oplog);
+ var section = this.add_section(oplog);
oplog_info = await oplog_info;
this._index_max = oplog_info["opcount"];
@@ -81,7 +81,7 @@ export class Page extends ZenPage
const right = nav.right();
right.add(Friendly.sep(oplog_info["opcount"]));
- right.add("(" + Friendly.kib(oplog_info["totalsize"]) + ")");
+ right.add("(" + Friendly.bytes(oplog_info["totalsize"]) + ")");
right.sep();
var search_input = right.add("search:", "label").tag("input")
diff --git a/src/zenserver/frontend/html/pages/orchestrator.js b/src/zenserver/frontend/html/pages/orchestrator.js
new file mode 100644
index 000000000..24805c722
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/orchestrator.js
@@ -0,0 +1,405 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
+import { Table } from "../util/widgets.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+ async main()
+ {
+ this.set_title("orchestrator");
+
+ // Agents section
+ const agents_section = this._collapsible_section("Compute Agents");
+ this._agents_host = agents_section;
+ this._agents_table = null;
+
+ // Clients section
+ const clients_section = this._collapsible_section("Connected Clients");
+ this._clients_host = clients_section;
+ this._clients_table = null;
+
+ // Event history
+ const history_section = this._collapsible_section("Worker Events");
+ this._history_host = history_section;
+ this._history_table = null;
+
+ const client_history_section = this._collapsible_section("Client Events");
+ this._client_history_host = client_history_section;
+ this._client_history_table = null;
+
+ this._ws_paused = false;
+ try { this._ws_paused = localStorage.getItem("zen-ws-paused") === "true"; } catch (e) {}
+ document.addEventListener("zen-ws-toggle", (e) => {
+ this._ws_paused = e.detail.paused;
+ });
+
+ // Initial fetch
+ await this._fetch_all();
+
+ // Connect WebSocket for live updates, fall back to polling
+ this._connect_ws();
+ }
+
+ _collapsible_section(name)
+ {
+ const section = this.add_section(name);
+ const container = section._parent.inner();
+ const heading = container.firstElementChild;
+
+ heading.style.cursor = "pointer";
+ heading.style.userSelect = "none";
+
+ const indicator = document.createElement("span");
+ indicator.textContent = " \u25BC";
+ indicator.style.fontSize = "0.7em";
+ heading.appendChild(indicator);
+
+ let collapsed = false;
+ heading.addEventListener("click", (e) => {
+ if (e.target !== heading && e.target !== indicator)
+ {
+ return;
+ }
+ collapsed = !collapsed;
+ indicator.textContent = collapsed ? " \u25B6" : " \u25BC";
+ let sibling = heading.nextElementSibling;
+ while (sibling)
+ {
+ sibling.style.display = collapsed ? "none" : "";
+ sibling = sibling.nextElementSibling;
+ }
+ });
+
+ return section;
+ }
+
+ async _fetch_all()
+ {
+ try
+ {
+ const [agents, history, clients, client_history] = await Promise.all([
+ new Fetcher().resource("/orch/agents").json(),
+ new Fetcher().resource("/orch/history").param("limit", "50").json().catch(() => null),
+ new Fetcher().resource("/orch/clients").json().catch(() => null),
+ new Fetcher().resource("/orch/clients/history").param("limit", "50").json().catch(() => null),
+ ]);
+
+ this._render_agents(agents);
+ if (history)
+ {
+ this._render_history(history.events || []);
+ }
+ if (clients)
+ {
+ this._render_clients(clients.clients || []);
+ }
+ if (client_history)
+ {
+ this._render_client_history(client_history.client_events || []);
+ }
+ }
+ catch (e) { /* service unavailable */ }
+ }
+
+ _connect_ws()
+ {
+ try
+ {
+ const proto = location.protocol === "https:" ? "wss:" : "ws:";
+ const ws = new WebSocket(`${proto}//${location.host}/orch/ws`);
+
+ ws.onopen = () => {
+ if (this._poll_timer)
+ {
+ clearInterval(this._poll_timer);
+ this._poll_timer = null;
+ }
+ };
+
+ ws.onmessage = (ev) => {
+ if (this._ws_paused)
+ {
+ return;
+ }
+ try
+ {
+ const data = JSON.parse(ev.data);
+ this._render_agents(data);
+ if (data.events)
+ {
+ this._render_history(data.events);
+ }
+ if (data.clients)
+ {
+ this._render_clients(data.clients);
+ }
+ if (data.client_events)
+ {
+ this._render_client_history(data.client_events);
+ }
+ }
+ catch (e) { /* ignore parse errors */ }
+ };
+
+ ws.onclose = () => {
+ this._start_polling();
+ setTimeout(() => this._connect_ws(), 3000);
+ };
+
+ ws.onerror = () => { /* onclose will fire */ };
+ }
+ catch (e)
+ {
+ this._start_polling();
+ }
+ }
+
+ _start_polling()
+ {
+ if (!this._poll_timer)
+ {
+ this._poll_timer = setInterval(() => this._fetch_all(), 2000);
+ }
+ }
+
+ _render_agents(data)
+ {
+ const workers = data.workers || [];
+
+ if (this._agents_table)
+ {
+ this._agents_table.clear();
+ }
+ else
+ {
+ this._agents_table = this._agents_host.add_widget(
+ Table,
+ ["hostname", "CPUs", "CPU usage", "memory", "queues", "pending", "running", "completed", "traffic", "last seen"],
+ Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric, -1
+ );
+ }
+
+ if (workers.length === 0)
+ {
+ return;
+ }
+
+ let totalCpus = 0, totalWeightedCpu = 0;
+ let totalMemUsed = 0, totalMemTotal = 0;
+ let totalQueues = 0, totalPending = 0, totalRunning = 0, totalCompleted = 0;
+ let totalRecv = 0, totalSent = 0;
+
+ for (const w of workers)
+ {
+ const cpus = w.cpus || 0;
+ const cpuUsage = w.cpu_usage;
+ const memUsed = w.memory_used || 0;
+ const memTotal = w.memory_total || 0;
+ const queues = w.active_queues || 0;
+ const pending = w.actions_pending || 0;
+ const running = w.actions_running || 0;
+ const completed = w.actions_completed || 0;
+ const recv = w.bytes_received || 0;
+ const sent = w.bytes_sent || 0;
+
+ totalCpus += cpus;
+ if (cpus > 0 && typeof cpuUsage === "number")
+ {
+ totalWeightedCpu += cpuUsage * cpus;
+ }
+ totalMemUsed += memUsed;
+ totalMemTotal += memTotal;
+ totalQueues += queues;
+ totalPending += pending;
+ totalRunning += running;
+ totalCompleted += completed;
+ totalRecv += recv;
+ totalSent += sent;
+
+ const hostname = w.hostname || "";
+ const row = this._agents_table.add_row(
+ hostname,
+ cpus > 0 ? Friendly.sep(cpus) : "-",
+ typeof cpuUsage === "number" ? cpuUsage.toFixed(1) + "%" : "-",
+ memTotal > 0 ? Friendly.bytes(memUsed) + " / " + Friendly.bytes(memTotal) : "-",
+ queues > 0 ? Friendly.sep(queues) : "-",
+ Friendly.sep(pending),
+ Friendly.sep(running),
+ Friendly.sep(completed),
+ this._format_traffic(recv, sent),
+ this._format_last_seen(w.dt),
+ );
+
+ // Link hostname to worker dashboard
+ if (w.uri)
+ {
+ const cell = row.get_cell(0);
+ cell.inner().textContent = "";
+ cell.tag("a").text(hostname).attr("href", w.uri + "/dashboard/compute/").attr("target", "_blank");
+ }
+ }
+
+ // Total row
+ const total = this._agents_table.add_row(
+ "TOTAL",
+ Friendly.sep(totalCpus),
+ "",
+ totalMemTotal > 0 ? Friendly.bytes(totalMemUsed) + " / " + Friendly.bytes(totalMemTotal) : "-",
+ Friendly.sep(totalQueues),
+ Friendly.sep(totalPending),
+ Friendly.sep(totalRunning),
+ Friendly.sep(totalCompleted),
+ this._format_traffic(totalRecv, totalSent),
+ "",
+ );
+ total.get_cell(0).style("fontWeight", "bold");
+ }
+
+ _render_clients(clients)
+ {
+ if (this._clients_table)
+ {
+ this._clients_table.clear();
+ }
+ else
+ {
+ this._clients_table = this._clients_host.add_widget(
+ Table,
+ ["client ID", "hostname", "address", "last seen"],
+ Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable, -1
+ );
+ }
+
+ for (const c of clients)
+ {
+ this._clients_table.add_row(
+ c.id || "",
+ c.hostname || "",
+ c.address || "",
+ this._format_last_seen(c.dt),
+ );
+ }
+ }
+
+ _render_history(events)
+ {
+ if (this._history_table)
+ {
+ this._history_table.clear();
+ }
+ else
+ {
+ this._history_table = this._history_host.add_widget(
+ Table,
+ ["time", "event", "worker", "hostname"],
+ Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable, -1
+ );
+ }
+
+ for (const evt of events)
+ {
+ this._history_table.add_row(
+ this._format_timestamp(evt.ts),
+ evt.type || "",
+ evt.worker_id || "",
+ evt.hostname || "",
+ );
+ }
+ }
+
+ _render_client_history(events)
+ {
+ if (this._client_history_table)
+ {
+ this._client_history_table.clear();
+ }
+ else
+ {
+ this._client_history_table = this._client_history_host.add_widget(
+ Table,
+ ["time", "event", "client", "hostname"],
+ Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable, -1
+ );
+ }
+
+ for (const evt of events)
+ {
+ this._client_history_table.add_row(
+ this._format_timestamp(evt.ts),
+ evt.type || "",
+ evt.client_id || "",
+ evt.hostname || "",
+ );
+ }
+ }
+
+ _metric(parent, value, label, hero = false)
+ {
+ const m = parent.tag().classify("tile-metric");
+ if (hero)
+ {
+ m.classify("tile-metric-hero");
+ }
+ m.tag().classify("metric-value").text(value);
+ m.tag().classify("metric-label").text(label);
+ }
+
+ _format_last_seen(dtMs)
+ {
+ if (dtMs == null)
+ {
+ return "-";
+ }
+ const seconds = Math.floor(dtMs / 1000);
+ if (seconds < 60)
+ {
+ return seconds + "s ago";
+ }
+ const minutes = Math.floor(seconds / 60);
+ if (minutes < 60)
+ {
+ return minutes + "m " + (seconds % 60) + "s ago";
+ }
+ const hours = Math.floor(minutes / 60);
+ return hours + "h " + (minutes % 60) + "m ago";
+ }
+
+ _format_traffic(recv, sent)
+ {
+ if (!recv && !sent)
+ {
+ return "-";
+ }
+ return Friendly.bytes(recv) + " / " + Friendly.bytes(sent);
+ }
+
+ _format_timestamp(ts)
+ {
+ if (!ts)
+ {
+ return "-";
+ }
+ let date;
+ if (typeof ts === "number")
+ {
+ // .NET-style ticks: convert to Unix ms
+ const unixMs = (ts - 621355968000000000) / 10000;
+ date = new Date(unixMs);
+ }
+ else
+ {
+ date = new Date(ts);
+ }
+ if (isNaN(date.getTime()))
+ {
+ return "-";
+ }
+ return date.toLocaleTimeString();
+ }
+}
diff --git a/src/zenserver/frontend/html/pages/page.js b/src/zenserver/frontend/html/pages/page.js
index 9a9541904..dd8032c28 100644
--- a/src/zenserver/frontend/html/pages/page.js
+++ b/src/zenserver/frontend/html/pages/page.js
@@ -3,6 +3,7 @@
"use strict";
import { WidgetHost } from "../util/widgets.js"
+import { Fetcher } from "../util/fetcher.js"
////////////////////////////////////////////////////////////////////////////////
export class PageBase extends WidgetHost
@@ -63,31 +64,85 @@ export class ZenPage extends PageBase
super(parent, ...args);
super.set_title("zen");
this.add_branding(parent);
+ this.add_service_nav(parent);
this.generate_crumbs();
}
add_branding(parent)
{
- var root = parent.tag().id("branding");
-
- const zen_store = root.tag("pre").id("logo").text(
- "_________ _______ __\n" +
- "\\____ /___ ___ / ___// |__ ___ ______ ____\n" +
- " / __/ __ \\ / \\ \\___ \\\\_ __// \\\\_ \\/ __ \\\n" +
- " / \\ __// | \\/ \\| | ( - )| |\\/\\ __/\n" +
- "/______/\\___/\\__|__/\\______/|__| \\___/ |__| \\___|"
- );
- zen_store.tag().id("go_home").on_click(() => window.location.search = "");
-
- root.tag("img").attr("src", "favicon.ico").id("ue_logo");
-
- /*
- _________ _______ __
- \____ /___ ___ / ___// |__ ___ ______ ____
- / __/ __ \ / \ \___ \\_ __// \\_ \/ __ \
- / \ __// | \/ \| | ( - )| |\/\ __/
- /______/\___/\__|__/\______/|__| \___/ |__| \___|
- */
+ var banner = parent.tag("zen-banner");
+ banner.attr("subtitle", "SERVER");
+ banner.attr("tagline", "Local Storage Service");
+ banner.attr("logo-src", "favicon.ico");
+ banner.attr("load", "0");
+
+ this._banner = banner;
+ this._poll_status();
+ }
+
+ async _poll_status()
+ {
+ try
+ {
+ var cbo = await new Fetcher().resource("/status/status").cbo();
+ if (cbo)
+ {
+ var obj = cbo.as_object();
+
+ var hostname = obj.find("hostname");
+ if (hostname)
+ {
+ this._banner.attr("tagline", "Local Storage Service \u2014 " + hostname.as_value());
+ }
+
+ var cpu = obj.find("cpuUsagePercent");
+ if (cpu)
+ {
+ this._banner.attr("load", cpu.as_value().toFixed(1));
+ }
+ }
+ }
+ catch (e) { console.warn("status poll:", e); }
+
+ setTimeout(() => this._poll_status(), 2000);
+ }
+
+ add_service_nav(parent)
+ {
+ const nav = parent.tag().id("service_nav");
+
+ // Map service base URIs to dashboard links, this table is also used to detemine
+ // which links to show based on the services that are currently registered.
+
+ const service_dashboards = [
+ { base_uri: "/compute/", label: "Compute", href: "/dashboard/?page=compute" },
+ { base_uri: "/orch/", label: "Orchestrator", href: "/dashboard/?page=orchestrator" },
+ { base_uri: "/hub/", label: "Hub", href: "/dashboard/?page=hub" },
+ ];
+
+ nav.tag("a").text("Home").attr("href", "/dashboard/");
+
+ nav.tag("a").text("Sessions").attr("href", "/dashboard/?page=sessions");
+ nav.tag("a").text("Cache").attr("href", "/dashboard/?page=cache");
+ nav.tag("a").text("Projects").attr("href", "/dashboard/?page=projects");
+ this._info_link = nav.tag("a").text("Info").attr("href", "/dashboard/?page=info");
+
+ new Fetcher().resource("/api/").json().then((data) => {
+ const services = data.services || [];
+ const uris = new Set(services.map(s => s.base_uri));
+
+ const links = service_dashboards.filter(d => uris.has(d.base_uri));
+
+ // Insert service links before the Info link
+ const info_elem = this._info_link.inner();
+ for (const link of links)
+ {
+ const a = document.createElement("a");
+ a.textContent = link.label;
+ a.href = link.href;
+ info_elem.parentNode.insertBefore(a, info_elem);
+ }
+ }).catch(() => {});
}
set_title(...args)
@@ -97,7 +152,7 @@ export class ZenPage extends PageBase
generate_crumbs()
{
- const auto_name = this.get_param("page") || "start";
+ var auto_name = this.get_param("page") || "start";
if (auto_name == "start")
return;
@@ -114,15 +169,30 @@ export class ZenPage extends PageBase
var project = this.get_param("project");
if (project != undefined)
{
+ auto_name = project;
var oplog = this.get_param("oplog");
if (oplog != undefined)
{
- new_crumb("project", `?page=project&project=${project}`);
- if (this.get_param("opkey"))
- new_crumb("oplog", `?page=oplog&project=${project}&oplog=${oplog}`);
+ new_crumb(auto_name, `?page=project&project=${project}`);
+ auto_name = oplog;
+ var opkey = this.get_param("opkey")
+ if (opkey != undefined)
+ {
+ new_crumb(auto_name, `?page=oplog&project=${project}&oplog=${oplog}`);
+ auto_name = opkey.split("/").pop().split("\\").pop();
+
+ // Check if we're viewing cook artifacts
+ var page = this.get_param("page");
+ var hash = this.get_param("hash");
+ if (hash != undefined && page == "cookartifacts")
+ {
+ new_crumb(auto_name, `?page=entry&project=${project}&oplog=${oplog}&opkey=${opkey}`);
+ auto_name = "cook artifacts";
+ }
+ }
}
}
- new_crumb(auto_name.toLowerCase());
+ new_crumb(auto_name);
}
}
diff --git a/src/zenserver/frontend/html/pages/project.js b/src/zenserver/frontend/html/pages/project.js
index 42ae30c8c..3a7a45527 100644
--- a/src/zenserver/frontend/html/pages/project.js
+++ b/src/zenserver/frontend/html/pages/project.js
@@ -59,7 +59,7 @@ export class Page extends ZenPage
info = await info;
row.get_cell(1).text(info["markerpath"]);
- row.get_cell(2).text(Friendly.kib(info["totalsize"]));
+ row.get_cell(2).text(Friendly.bytes(info["totalsize"]));
row.get_cell(3).text(Friendly.sep(info["opcount"]));
row.get_cell(4).text(info["expired"]);
}
diff --git a/src/zenserver/frontend/html/pages/projects.js b/src/zenserver/frontend/html/pages/projects.js
new file mode 100644
index 000000000..9c1e519d4
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/projects.js
@@ -0,0 +1,447 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
+import { Modal } from "../util/modal.js"
+import { Table, Toolbar } from "../util/widgets.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+ async main()
+ {
+ this.set_title("projects");
+
+ // Project Service Stats
+ const stats_section = this._collapsible_section("Project Service Stats");
+ stats_section.tag().classify("dropall").text("raw yaml \u2192").on_click(() => {
+ window.open("/stats/prj.yaml", "_blank");
+ });
+ this._stats_grid = stats_section.tag().classify("grid").classify("stats-tiles");
+
+ const stats = await new Fetcher().resource("stats", "prj").json();
+ if (stats)
+ {
+ this._render_stats(stats);
+ }
+
+ this._connect_stats_ws();
+
+ // Projects list
+ var section = this._collapsible_section("Projects");
+
+ section.tag().classify("dropall").text("drop-all").on_click(() => this.drop_all());
+
+ var columns = [
+ "name",
+ "project dir",
+ "engine dir",
+ "oplogs",
+ "actions",
+ ];
+
+ this._project_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric);
+
+ var projects = await new Fetcher().resource("/prj/list").json();
+ projects.sort((a, b) => (b.LastAccessTime || 0) - (a.LastAccessTime || 0));
+
+ for (const project of projects)
+ {
+ var row = this._project_table.add_row(
+ "",
+ "",
+ "",
+ "",
+ );
+
+ var cell = row.get_cell(0);
+ cell.tag().text(project.Id).on_click(() => this.view_project(project.Id));
+
+ if (project.ProjectRootDir)
+ {
+ row.get_cell(1).tag("a").text(project.ProjectRootDir)
+ .attr("href", "vscode://" + project.ProjectRootDir.replace(/\\/g, "/"));
+ }
+ if (project.EngineRootDir)
+ {
+ row.get_cell(2).tag("a").text(project.EngineRootDir)
+ .attr("href", "vscode://" + project.EngineRootDir.replace(/\\/g, "/"));
+ }
+
+ cell = row.get_cell(-1);
+ const action_tb = new Toolbar(cell, true).left();
+ action_tb.add("view").on_click(() => this.view_project(project.Id));
+ action_tb.add("drop").on_click(() => this.drop_project(project.Id));
+
+ row.attr("zs_name", project.Id);
+
+ // Fetch project details to get oplog count
+ new Fetcher().resource("prj", project.Id).json().then((info) => {
+ const oplogs = info["oplogs"] || [];
+ row.get_cell(3).text(Friendly.sep(oplogs.length)).style("textAlign", "right");
+ // Right-align the corresponding header cell
+ const header = this._project_table._element.firstElementChild;
+ if (header && header.children[4])
+ {
+ header.children[4].style.textAlign = "right";
+ }
+ }).catch(() => {});
+ }
+
+ // Project detail area (inside projects section so it collapses together)
+ this._project_host = section;
+ this._project_container = null;
+ this._selected_project = null;
+
+ // Restore project from URL if present
+ const prj_param = this.get_param("project");
+ if (prj_param)
+ {
+ this.view_project(prj_param);
+ }
+ }
+
+ _collapsible_section(name)
+ {
+ const section = this.add_section(name);
+ const container = section._parent.inner();
+ const heading = container.firstElementChild;
+
+ heading.style.cursor = "pointer";
+ heading.style.userSelect = "none";
+
+ const indicator = document.createElement("span");
+ indicator.textContent = " \u25BC";
+ indicator.style.fontSize = "0.7em";
+ heading.appendChild(indicator);
+
+ let collapsed = false;
+ heading.addEventListener("click", (e) => {
+ if (e.target !== heading && e.target !== indicator)
+ {
+ return;
+ }
+ collapsed = !collapsed;
+ indicator.textContent = collapsed ? " \u25B6" : " \u25BC";
+ let sibling = heading.nextElementSibling;
+ while (sibling)
+ {
+ sibling.style.display = collapsed ? "none" : "";
+ sibling = sibling.nextElementSibling;
+ }
+ });
+
+ return section;
+ }
+
+ _clear_param(name)
+ {
+ this._params.delete(name);
+ const url = new URL(window.location);
+ url.searchParams.delete(name);
+ history.replaceState(null, "", url);
+ }
+
+ _connect_stats_ws()
+ {
+ try
+ {
+ const proto = location.protocol === "https:" ? "wss:" : "ws:";
+ const ws = new WebSocket(`${proto}//${location.host}/stats`);
+
+ try { this._ws_paused = localStorage.getItem("zen-ws-paused") === "true"; } catch (e) { this._ws_paused = false; }
+ document.addEventListener("zen-ws-toggle", (e) => {
+ this._ws_paused = e.detail.paused;
+ });
+
+ ws.onmessage = (ev) => {
+ if (this._ws_paused)
+ {
+ return;
+ }
+ try
+ {
+ const all_stats = JSON.parse(ev.data);
+ const stats = all_stats["prj"];
+ if (stats)
+ {
+ this._render_stats(stats);
+ }
+ }
+ catch (e) { /* ignore parse errors */ }
+ };
+
+ ws.onclose = () => { this._stats_ws = null; };
+ ws.onerror = () => { ws.close(); };
+
+ this._stats_ws = ws;
+ }
+ catch (e) { /* WebSocket not available */ }
+ }
+
+ _render_stats(stats)
+ {
+ const safe = (obj, path) => path.split(".").reduce((a, b) => a && a[b], obj);
+ const grid = this._stats_grid;
+
+ grid.inner().innerHTML = "";
+
+ // HTTP Requests tile
+ {
+ const req = safe(stats, "requests");
+ if (req)
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("HTTP Requests");
+ const columns = tile.tag().classify("tile-columns");
+
+ const left = columns.tag().classify("tile-metrics");
+ const reqData = req.requests || req;
+ this._metric(left, Friendly.sep(safe(stats, "store.requestcount") || 0), "total requests", true);
+ if (reqData.rate_mean > 0)
+ {
+ this._metric(left, Friendly.sep(reqData.rate_mean, 1) + "/s", "req/sec (mean)");
+ }
+ if (reqData.rate_1 > 0)
+ {
+ this._metric(left, Friendly.sep(reqData.rate_1, 1) + "/s", "req/sec (1m)");
+ }
+ const badRequests = safe(stats, "store.badrequestcount") || 0;
+ this._metric(left, Friendly.sep(badRequests), "bad requests");
+
+ const right = columns.tag().classify("tile-metrics");
+ this._metric(right, Friendly.duration(reqData.t_avg || 0), "avg latency", true);
+ if (reqData.t_p75)
+ {
+ this._metric(right, Friendly.duration(reqData.t_p75), "p75");
+ }
+ if (reqData.t_p95)
+ {
+ this._metric(right, Friendly.duration(reqData.t_p95), "p95");
+ }
+ if (reqData.t_p99)
+ {
+ this._metric(right, Friendly.duration(reqData.t_p99), "p99");
+ }
+ }
+ }
+
+ // Store Operations tile
+ {
+ const store = safe(stats, "store");
+ if (store)
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Store Operations");
+ const columns = tile.tag().classify("tile-columns");
+
+ const left = columns.tag().classify("tile-metrics");
+ const proj = store.project || {};
+ this._metric(left, Friendly.sep(proj.readcount || 0), "project reads", true);
+ this._metric(left, Friendly.sep(proj.writecount || 0), "project writes");
+ this._metric(left, Friendly.sep(proj.deletecount || 0), "project deletes");
+
+ const right = columns.tag().classify("tile-metrics");
+ const oplog = store.oplog || {};
+ this._metric(right, Friendly.sep(oplog.readcount || 0), "oplog reads", true);
+ this._metric(right, Friendly.sep(oplog.writecount || 0), "oplog writes");
+ this._metric(right, Friendly.sep(oplog.deletecount || 0), "oplog deletes");
+ }
+ }
+
+ // Op & Chunk tile
+ {
+ const store = safe(stats, "store");
+ if (store)
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Ops & Chunks");
+ const columns = tile.tag().classify("tile-columns");
+
+ const left = columns.tag().classify("tile-metrics");
+ const op = store.op || {};
+ const opTotal = (op.hitcount || 0) + (op.misscount || 0);
+ const opRatio = opTotal > 0 ? (((op.hitcount || 0) / opTotal) * 100).toFixed(1) + "%" : "-";
+ this._metric(left, opRatio, "op hit ratio", true);
+ this._metric(left, Friendly.sep(op.hitcount || 0), "op hits");
+ this._metric(left, Friendly.sep(op.misscount || 0), "op misses");
+ this._metric(left, Friendly.sep(op.writecount || 0), "op writes");
+
+ const right = columns.tag().classify("tile-metrics");
+ const chunk = store.chunk || {};
+ const chunkTotal = (chunk.hitcount || 0) + (chunk.misscount || 0);
+ const chunkRatio = chunkTotal > 0 ? (((chunk.hitcount || 0) / chunkTotal) * 100).toFixed(1) + "%" : "-";
+ this._metric(right, chunkRatio, "chunk hit ratio", true);
+ this._metric(right, Friendly.sep(chunk.hitcount || 0), "chunk hits");
+ this._metric(right, Friendly.sep(chunk.misscount || 0), "chunk misses");
+ this._metric(right, Friendly.sep(chunk.writecount || 0), "chunk writes");
+ }
+ }
+
+ // Storage tile
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Storage");
+ const columns = tile.tag().classify("tile-columns");
+
+ const left = columns.tag().classify("tile-metrics");
+ this._metric(left, safe(stats, "store.size.disk") != null ? Friendly.bytes(safe(stats, "store.size.disk")) : "-", "store disk", true);
+ this._metric(left, safe(stats, "store.size.memory") != null ? Friendly.bytes(safe(stats, "store.size.memory")) : "-", "store memory");
+
+ const right = columns.tag().classify("tile-metrics");
+ this._metric(right, safe(stats, "cid.size.total") != null ? Friendly.bytes(safe(stats, "cid.size.total")) : "-", "cid total", true);
+ this._metric(right, safe(stats, "cid.size.tiny") != null ? Friendly.bytes(safe(stats, "cid.size.tiny")) : "-", "cid tiny");
+ this._metric(right, safe(stats, "cid.size.small") != null ? Friendly.bytes(safe(stats, "cid.size.small")) : "-", "cid small");
+ this._metric(right, safe(stats, "cid.size.large") != null ? Friendly.bytes(safe(stats, "cid.size.large")) : "-", "cid large");
+ }
+ }
+
+ _metric(parent, value, label, hero = false)
+ {
+ const m = parent.tag().classify("tile-metric");
+ if (hero)
+ {
+ m.classify("tile-metric-hero");
+ }
+ m.tag().classify("metric-value").text(value);
+ m.tag().classify("metric-label").text(label);
+ }
+
+ async view_project(project_id)
+ {
+ // Toggle off if already selected
+ if (this._selected_project === project_id)
+ {
+ this._selected_project = null;
+ this._clear_project_detail();
+ this._clear_param("project");
+ return;
+ }
+
+ this._selected_project = project_id;
+ this._clear_project_detail();
+ this.set_param("project", project_id);
+
+ const info = await new Fetcher().resource("prj", project_id).json();
+ if (this._selected_project !== project_id)
+ {
+ return;
+ }
+
+ const section = this._project_host.add_section(project_id);
+ this._project_container = section;
+
+ // Oplogs table
+ const oplog_section = section.add_section("Oplogs");
+ const oplog_table = oplog_section.add_widget(
+ Table,
+ ["name", "marker", "size", "ops", "expired", "actions"],
+ Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric
+ );
+
+ let totalSize = 0, totalOps = 0;
+ const total_row = oplog_table.add_row("TOTAL");
+ total_row.get_cell(0).style("fontWeight", "bold");
+ total_row.get_cell(2).style("textAlign", "right").style("fontWeight", "bold");
+ total_row.get_cell(3).style("textAlign", "right").style("fontWeight", "bold");
+
+ // Right-align header for numeric columns (size, ops)
+ const header = oplog_table._element.firstElementChild;
+ for (let i = 3; i < header.children.length - 1; i++)
+ {
+ header.children[i].style.textAlign = "right";
+ }
+
+ for (const oplog of info["oplogs"] || [])
+ {
+ const name = oplog["id"];
+ const row = oplog_table.add_row("");
+
+ var cell = row.get_cell(0);
+ cell.tag().text(name).link("", {
+ "page": "oplog",
+ "project": project_id,
+ "oplog": name,
+ });
+
+ cell = row.get_cell(-1);
+ const action_tb = new Toolbar(cell, true).left();
+ action_tb.add("list").link("", { "page": "oplog", "project": project_id, "oplog": name });
+ action_tb.add("tree").link("", { "page": "tree", "project": project_id, "oplog": name });
+ action_tb.add("drop").on_click(() => this.drop_oplog(project_id, name));
+
+ new Fetcher().resource("prj", project_id, "oplog", name).json().then((data) => {
+ row.get_cell(1).text(data["markerpath"]);
+ row.get_cell(2).text(Friendly.bytes(data["totalsize"])).style("textAlign", "right");
+ row.get_cell(3).text(Friendly.sep(data["opcount"])).style("textAlign", "right");
+ row.get_cell(4).text(data["expired"]);
+
+ totalSize += data["totalsize"] || 0;
+ totalOps += data["opcount"] || 0;
+ total_row.get_cell(2).text(Friendly.bytes(totalSize)).style("textAlign", "right").style("fontWeight", "bold");
+ total_row.get_cell(3).text(Friendly.sep(totalOps)).style("textAlign", "right").style("fontWeight", "bold");
+ }).catch(() => {});
+ }
+ }
+
+ _clear_project_detail()
+ {
+ if (this._project_container)
+ {
+ this._project_container._parent.inner().remove();
+ this._project_container = null;
+ }
+ }
+
+ drop_oplog(project_id, oplog_id)
+ {
+ const drop = async () => {
+ await new Fetcher().resource("prj", project_id, "oplog", oplog_id).delete();
+ // Refresh the project view
+ this._selected_project = null;
+ this._clear_project_detail();
+ this.view_project(project_id);
+ };
+
+ new Modal()
+ .title("Confirmation")
+ .message(`Drop oplog '${oplog_id}'?`)
+ .option("Yes", () => drop())
+ .option("No");
+ }
+
+ drop_project(project_id)
+ {
+ const drop = async () => {
+ await new Fetcher().resource("prj", project_id).delete();
+ this.reload();
+ };
+
+ new Modal()
+ .title("Confirmation")
+ .message(`Drop project '${project_id}'?`)
+ .option("Yes", () => drop())
+ .option("No");
+ }
+
+ async drop_all()
+ {
+ const drop = async () => {
+ for (const row of this._project_table)
+ {
+ const project_id = row.attr("zs_name");
+ await new Fetcher().resource("prj", project_id).delete();
+ }
+ this.reload();
+ };
+
+ new Modal()
+ .title("Confirmation")
+ .message("Drop every project?")
+ .option("Yes", () => drop())
+ .option("No");
+ }
+}
diff --git a/src/zenserver/frontend/html/pages/sessions.js b/src/zenserver/frontend/html/pages/sessions.js
new file mode 100644
index 000000000..95533aa96
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/sessions.js
@@ -0,0 +1,61 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Table } from "../util/widgets.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+ async main()
+ {
+ this.set_title("sessions");
+
+ const data = await new Fetcher().resource("/sessions/").json();
+ const sessions = data.sessions || [];
+
+ const section = this.add_section("Sessions");
+
+ if (sessions.length === 0)
+ {
+ section.tag().classify("empty-state").text("No active sessions.");
+ return;
+ }
+
+ const columns = [
+ "id",
+ "created",
+ "updated",
+ "metadata",
+ ];
+ const table = section.add_widget(Table, columns, Table.Flag_FitLeft);
+
+ for (const session of sessions)
+ {
+ const created = session.created_at ? new Date(session.created_at).toLocaleString() : "-";
+ const updated = session.updated_at ? new Date(session.updated_at).toLocaleString() : "-";
+ const meta = this._format_metadata(session.metadata);
+
+ const row = table.add_row(
+ session.id || "-",
+ created,
+ updated,
+ meta,
+ );
+ }
+ }
+
+ _format_metadata(metadata)
+ {
+ if (!metadata || Object.keys(metadata).length === 0)
+ {
+ return "-";
+ }
+
+ return Object.entries(metadata)
+ .map(([k, v]) => `${k}: ${v}`)
+ .join(", ");
+ }
+}
diff --git a/src/zenserver/frontend/html/pages/start.js b/src/zenserver/frontend/html/pages/start.js
index 4c8789431..3a68a725d 100644
--- a/src/zenserver/frontend/html/pages/start.js
+++ b/src/zenserver/frontend/html/pages/start.js
@@ -13,109 +13,117 @@ export class Page extends ZenPage
{
async main()
{
+ // Discover which services are available
+ const api_data = await new Fetcher().resource("/api/").json();
+ const available = new Set((api_data.services || []).map(s => s.base_uri));
+
// project list
- var section = this.add_section("projects");
+ var project_table = null;
+ if (available.has("/prj/"))
+ {
+ var section = this.add_section("Cooked Projects");
- section.tag().classify("dropall").text("drop-all").on_click(() => this.drop_all("projects"));
+ section.tag().classify("dropall").text("drop-all").on_click(() => this.drop_all("projects"));
- var columns = [
- "name",
- "project_dir",
- "engine_dir",
- "actions",
- ];
- var project_table = section.add_widget(Table, columns);
+ var columns = [
+ "name",
+ "project_dir",
+ "engine_dir",
+ "actions",
+ ];
+ project_table = section.add_widget(Table, columns);
- for (const project of await new Fetcher().resource("/prj/list").json())
- {
- var row = project_table.add_row(
- "",
- project.ProjectRootDir,
- project.EngineRootDir,
- );
+ var projects = await new Fetcher().resource("/prj/list").json();
+ projects.sort((a, b) => (b.LastAccessTime || 0) - (a.LastAccessTime || 0));
+ projects = projects.slice(0, 25);
+ projects.sort((a, b) => a.Id.localeCompare(b.Id));
- var cell = row.get_cell(0);
- cell.tag().text(project.Id).on_click((x) => this.view_project(x), project.Id);
+ for (const project of projects)
+ {
+ var row = project_table.add_row(
+ "",
+ project.ProjectRootDir,
+ project.EngineRootDir,
+ );
+
+ var cell = row.get_cell(0);
+ cell.tag().text(project.Id).on_click((x) => this.view_project(x), project.Id);
- var cell = row.get_cell(-1);
- var action_tb = new Toolbar(cell, true);
- action_tb.left().add("view").on_click((x) => this.view_project(x), project.Id);
- action_tb.left().add("drop").on_click((x) => this.drop_project(x), project.Id);
+ var cell = row.get_cell(-1);
+ var action_tb = new Toolbar(cell, true);
+ action_tb.left().add("view").on_click((x) => this.view_project(x), project.Id);
+ action_tb.left().add("drop").on_click((x) => this.drop_project(x), project.Id);
- row.attr("zs_name", project.Id);
+ row.attr("zs_name", project.Id);
+ }
}
// cache
- var section = this.add_section("z$");
-
- section.tag().classify("dropall").text("drop-all").on_click(() => this.drop_all("z$"));
-
- columns = [
- "namespace",
- "dir",
- "buckets",
- "entries",
- "size disk",
- "size mem",
- "actions",
- ]
- var zcache_info = new Fetcher().resource("/z$/").json();
- const cache_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight);
- for (const namespace of (await zcache_info)["Namespaces"])
+ var cache_table = null;
+ if (available.has("/z$/"))
{
- new Fetcher().resource(`/z$/${namespace}/`).json().then((data) => {
- const row = cache_table.add_row(
- "",
- data["Configuration"]["RootDir"],
- data["Buckets"].length,
- data["EntryCount"],
- Friendly.kib(data["StorageSize"].DiskSize),
- Friendly.kib(data["StorageSize"].MemorySize)
- );
- var cell = row.get_cell(0);
- cell.tag().text(namespace).on_click(() => this.view_zcache(namespace));
- row.get_cell(1).tag().text(namespace);
+ var section = this.add_section("Cache");
- cell = row.get_cell(-1);
- const action_tb = new Toolbar(cell, true);
- action_tb.left().add("view").on_click(() => this.view_zcache(namespace));
- action_tb.left().add("drop").on_click(() => this.drop_zcache(namespace));
+ section.tag().classify("dropall").text("drop-all").on_click(() => this.drop_all("z$"));
- row.attr("zs_name", namespace);
- });
+ var columns = [
+ "namespace",
+ "dir",
+ "buckets",
+ "entries",
+ "size disk",
+ "size mem",
+ "actions",
+ ];
+ var zcache_info = await new Fetcher().resource("/z$/").json();
+ cache_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight);
+ for (const namespace of zcache_info["Namespaces"] || [])
+ {
+ new Fetcher().resource(`/z$/${namespace}/`).json().then((data) => {
+ const row = cache_table.add_row(
+ "",
+ data["Configuration"]["RootDir"],
+ data["Buckets"].length,
+ data["EntryCount"],
+ Friendly.bytes(data["StorageSize"].DiskSize),
+ Friendly.bytes(data["StorageSize"].MemorySize)
+ );
+ var cell = row.get_cell(0);
+ cell.tag().text(namespace).on_click(() => this.view_zcache(namespace));
+ row.get_cell(1).tag().text(namespace);
+
+ cell = row.get_cell(-1);
+ const action_tb = new Toolbar(cell, true);
+ action_tb.left().add("view").on_click(() => this.view_zcache(namespace));
+ action_tb.left().add("drop").on_click(() => this.drop_zcache(namespace));
+
+ row.attr("zs_name", namespace);
+ });
+ }
}
- // stats
+ // stats tiles
const safe_lookup = (obj, path, pretty=undefined) => {
const ret = path.split(".").reduce((a,b) => a && a[b], obj);
- if (ret === undefined) return "-";
+ if (ret === undefined) return undefined;
return pretty ? pretty(ret) : ret;
};
- section = this.add_section("stats");
- columns = [
- "name",
- "req count",
- "size disk",
- "size mem",
- "cid total",
- ];
- const stats_table = section.add_widget(Table, columns, Table.Flag_PackRight);
- var providers = new Fetcher().resource("stats").json();
- for (var provider of (await providers)["providers"])
- {
- var stats = await new Fetcher().resource("stats", provider).json();
- var size_stat = (stats.store || stats.cache);
- var values = [
- "",
- safe_lookup(stats, "requests.count"),
- safe_lookup(size_stat, "size.disk", Friendly.kib),
- safe_lookup(size_stat, "size.memory", Friendly.kib),
- safe_lookup(stats, "cid.size.total"),
- ];
- row = stats_table.add_row(...values);
- row.get_cell(0).tag().text(provider).on_click((x) => this.view_stat(x), provider);
- }
+ var section = this.add_section("Stats");
+ section.tag().classify("dropall").text("metrics dashboard →").on_click(() => {
+ window.location = "?page=metrics";
+ });
+
+ var providers_data = await new Fetcher().resource("stats").json();
+ var provider_list = providers_data["providers"] || [];
+ var all_stats = {};
+ await Promise.all(provider_list.map(async (provider) => {
+ all_stats[provider] = await new Fetcher().resource("stats", provider).json();
+ }));
+
+ this._stats_grid = section.tag().classify("grid").classify("stats-tiles");
+ this._safe_lookup = safe_lookup;
+ this._render_stats(all_stats);
// version
var ver_tag = this.tag().id("version");
@@ -125,6 +133,159 @@ export class Page extends ZenPage
this._project_table = project_table;
this._cache_table = cache_table;
+
+ // WebSocket for live stats updates
+ this._connect_stats_ws();
+ }
+
+ _connect_stats_ws()
+ {
+ try
+ {
+ const proto = location.protocol === "https:" ? "wss:" : "ws:";
+ const ws = new WebSocket(`${proto}//${location.host}/stats`);
+
+ try { this._ws_paused = localStorage.getItem("zen-ws-paused") === "true"; } catch (e) { this._ws_paused = false; }
+ document.addEventListener("zen-ws-toggle", (e) => {
+ this._ws_paused = e.detail.paused;
+ });
+
+ ws.onmessage = (ev) => {
+ if (this._ws_paused)
+ {
+ return;
+ }
+ try
+ {
+ const all_stats = JSON.parse(ev.data);
+ this._render_stats(all_stats);
+ }
+ catch (e) { /* ignore parse errors */ }
+ };
+
+ ws.onclose = () => { this._stats_ws = null; };
+ ws.onerror = () => { ws.close(); };
+
+ this._stats_ws = ws;
+ }
+ catch (e) { /* WebSocket not available */ }
+ }
+
+ _render_stats(all_stats)
+ {
+ const grid = this._stats_grid;
+ const safe_lookup = this._safe_lookup;
+
+ // Clear existing tiles
+ grid.inner().innerHTML = "";
+
+ // HTTP tile — aggregate request stats across all providers
+ {
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("HTTP");
+ const columns = tile.tag().classify("tile-columns");
+
+ // Left column: request stats
+ const left = columns.tag().classify("tile-metrics");
+
+ let total_requests = 0;
+ let total_rate = 0;
+ for (const p in all_stats)
+ {
+ total_requests += (safe_lookup(all_stats[p], "requests.count") || 0);
+ total_rate += (safe_lookup(all_stats[p], "requests.rate_1") || 0);
+ }
+
+ this._add_tile_metric(left, Friendly.sep(total_requests), "total requests", true);
+ if (total_rate > 0)
+ this._add_tile_metric(left, Friendly.sep(total_rate, 1) + "/s", "req/sec (1m)");
+
+ // Right column: websocket stats
+ const ws = all_stats["http"] ? (all_stats["http"]["websockets"] || {}) : {};
+ const right = columns.tag().classify("tile-metrics");
+
+ this._add_tile_metric(right, Friendly.sep(ws.active_connections || 0), "ws connections", true);
+ const ws_frames = (ws.frames_received || 0) + (ws.frames_sent || 0);
+ if (ws_frames > 0)
+ this._add_tile_metric(right, Friendly.sep(ws_frames), "ws frames");
+ const ws_bytes = (ws.bytes_received || 0) + (ws.bytes_sent || 0);
+ if (ws_bytes > 0)
+ this._add_tile_metric(right, Friendly.bytes(ws_bytes), "ws traffic");
+
+ tile.on_click(() => { window.location = "?page=metrics"; });
+ }
+
+ // Cache tile (z$)
+ if (all_stats["z$"])
+ {
+ const s = all_stats["z$"];
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Cache");
+ const body = tile.tag().classify("tile-metrics");
+
+ const hits = safe_lookup(s, "cache.hits") || 0;
+ const misses = safe_lookup(s, "cache.misses") || 0;
+ const ratio = (hits + misses) > 0 ? ((hits / (hits + misses)) * 100).toFixed(1) + "%" : "-";
+
+ this._add_tile_metric(body, ratio, "hit ratio", true);
+ this._add_tile_metric(body, safe_lookup(s, "cache.size.disk", Friendly.bytes) || "-", "disk");
+ this._add_tile_metric(body, safe_lookup(s, "cache.size.memory", Friendly.bytes) || "-", "memory");
+
+ tile.on_click(() => { window.location = "?page=stat&provider=z$"; });
+ }
+
+ // Project Store tile (prj)
+ if (all_stats["prj"])
+ {
+ const s = all_stats["prj"];
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Project Store");
+ const body = tile.tag().classify("tile-metrics");
+
+ this._add_tile_metric(body, safe_lookup(s, "requests.count", Friendly.sep) || "-", "requests", true);
+ this._add_tile_metric(body, safe_lookup(s, "store.size.disk", Friendly.bytes) || "-", "disk");
+
+ tile.on_click(() => { window.location = "?page=stat&provider=prj"; });
+ }
+
+ // Build Store tile (builds)
+ if (all_stats["builds"])
+ {
+ const s = all_stats["builds"];
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Build Store");
+ const body = tile.tag().classify("tile-metrics");
+
+ this._add_tile_metric(body, safe_lookup(s, "requests.count", Friendly.sep) || "-", "requests", true);
+ this._add_tile_metric(body, safe_lookup(s, "store.size.disk", Friendly.bytes) || "-", "disk");
+
+ tile.on_click(() => { window.location = "?page=stat&provider=builds"; });
+ }
+
+ // Workspace tile (ws)
+ if (all_stats["ws"])
+ {
+ const s = all_stats["ws"];
+ const tile = grid.tag().classify("card").classify("stats-tile");
+ tile.tag().classify("card-title").text("Workspace");
+ const body = tile.tag().classify("tile-metrics");
+
+ this._add_tile_metric(body, safe_lookup(s, "requests.count", Friendly.sep) || "-", "requests", true);
+ this._add_tile_metric(body, safe_lookup(s, "workspaces.filescount", Friendly.sep) || "-", "files");
+
+ tile.on_click(() => { window.location = "?page=stat&provider=ws"; });
+ }
+ }
+
+ _add_tile_metric(parent, value, label, hero=false)
+ {
+ const m = parent.tag().classify("tile-metric");
+ if (hero)
+ {
+ m.classify("tile-metric-hero");
+ }
+ m.tag().classify("metric-value").text(value);
+ m.tag().classify("metric-label").text(label);
}
view_stat(provider)
diff --git a/src/zenserver/frontend/html/pages/stat.js b/src/zenserver/frontend/html/pages/stat.js
index d6c7fa8e8..4f020ac5e 100644
--- a/src/zenserver/frontend/html/pages/stat.js
+++ b/src/zenserver/frontend/html/pages/stat.js
@@ -33,7 +33,7 @@ class TemporalStat
out[key] = data[key];
}
- var friendly = this._as_bytes ? Friendly.kib : Friendly.sep;
+ var friendly = this._as_bytes ? Friendly.bytes : Friendly.sep;
var content = "";
for (var i = 0; i < columns.length; ++i)
diff --git a/src/zenserver/frontend/html/pages/tree.js b/src/zenserver/frontend/html/pages/tree.js
index 08a578492..b5fece5a3 100644
--- a/src/zenserver/frontend/html/pages/tree.js
+++ b/src/zenserver/frontend/html/pages/tree.js
@@ -106,7 +106,7 @@ export class Page extends ZenPage
for (var i = 0; i < 2; ++i)
{
- const size = Friendly.kib(new_nodes[name][i]);
+ const size = Friendly.bytes(new_nodes[name][i]);
info.tag().text(size);
}
diff --git a/src/zenserver/frontend/html/pages/zcache.js b/src/zenserver/frontend/html/pages/zcache.js
index 974893b21..d8bdc892a 100644
--- a/src/zenserver/frontend/html/pages/zcache.js
+++ b/src/zenserver/frontend/html/pages/zcache.js
@@ -27,8 +27,8 @@ export class Page extends ZenPage
cfg_table.add_object(info["Configuration"], true);
- storage_table.add_property("disk", Friendly.kib(info["StorageSize"]["DiskSize"]));
- storage_table.add_property("mem", Friendly.kib(info["StorageSize"]["MemorySize"]));
+ storage_table.add_property("disk", Friendly.bytes(info["StorageSize"]["DiskSize"]));
+ storage_table.add_property("mem", Friendly.bytes(info["StorageSize"]["MemorySize"]));
storage_table.add_property("entries", Friendly.sep(info["EntryCount"]));
var column_names = ["name", "disk", "mem", "entries", "actions"];
@@ -41,8 +41,8 @@ export class Page extends ZenPage
{
const row = bucket_table.add_row(bucket);
new Fetcher().resource(`/z$/${namespace}/${bucket}`).json().then((data) => {
- row.get_cell(1).text(Friendly.kib(data["StorageSize"]["DiskSize"]));
- row.get_cell(2).text(Friendly.kib(data["StorageSize"]["MemorySize"]));
+ row.get_cell(1).text(Friendly.bytes(data["StorageSize"]["DiskSize"]));
+ row.get_cell(2).text(Friendly.bytes(data["StorageSize"]["MemorySize"]));
row.get_cell(3).text(Friendly.sep(data["DiskEntryCount"]));
const cell = row.get_cell(-1);
diff --git a/src/zenserver/frontend/html/theme.js b/src/zenserver/frontend/html/theme.js
new file mode 100644
index 000000000..52ca116ab
--- /dev/null
+++ b/src/zenserver/frontend/html/theme.js
@@ -0,0 +1,116 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+// Theme toggle: cycles system → light → dark → system.
+// Persists choice in localStorage. Applies data-theme attribute on <html>.
+
+(function() {
+ var KEY = 'zen-theme';
+
+ function getStored() {
+ try { return localStorage.getItem(KEY); } catch (e) { return null; }
+ }
+
+ function setStored(value) {
+ try {
+ if (value) localStorage.setItem(KEY, value);
+ else localStorage.removeItem(KEY);
+ } catch (e) {}
+ }
+
+ function apply(theme) {
+ if (theme)
+ document.documentElement.setAttribute('data-theme', theme);
+ else
+ document.documentElement.removeAttribute('data-theme');
+ }
+
+ function getEffective(stored) {
+ if (stored) return stored;
+ return window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
+ }
+
+ // Apply stored preference immediately (before paint)
+ var stored = getStored();
+ apply(stored);
+
+ // Create toggle button once DOM is ready
+ function createToggle() {
+ var btn = document.createElement('button');
+ btn.id = 'zen_theme_toggle';
+ btn.title = 'Toggle theme';
+
+ function updateIcon() {
+ var effective = getEffective(getStored());
+ // Show sun in dark mode (click to go light), moon in light mode (click to go dark)
+ btn.textContent = effective === 'dark' ? '\u2600' : '\u263E';
+
+ var isManual = getStored() != null;
+ btn.title = isManual
+ ? 'Theme: ' + effective + ' (click to change, double-click for system)'
+ : 'Theme: system (click to change)';
+ }
+
+ btn.addEventListener('click', function() {
+ var current = getStored();
+ var effective = getEffective(current);
+ // Toggle to the opposite
+ var next = effective === 'dark' ? 'light' : 'dark';
+ setStored(next);
+ apply(next);
+ updateIcon();
+ });
+
+ btn.addEventListener('dblclick', function(e) {
+ e.preventDefault();
+ // Reset to system preference
+ setStored(null);
+ apply(null);
+ updateIcon();
+ });
+
+ // Update icon when system preference changes
+ window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', function() {
+ if (!getStored()) updateIcon();
+ });
+
+ updateIcon();
+ document.body.appendChild(btn);
+
+ // WebSocket pause/play toggle
+ var WS_KEY = 'zen-ws-paused';
+ var wsBtn = document.createElement('button');
+ wsBtn.id = 'zen_ws_toggle';
+
+ var initialPaused = false;
+ try { initialPaused = localStorage.getItem(WS_KEY) === 'true'; } catch (e) {}
+
+ function updateWsIcon(paused) {
+ wsBtn.dataset.paused = paused ? 'true' : 'false';
+ wsBtn.textContent = paused ? '\u25B6' : '\u23F8';
+ wsBtn.title = paused ? 'Resume live updates' : 'Pause live updates';
+ }
+
+ updateWsIcon(initialPaused);
+
+ // Fire initial event so pages pick up persisted state
+ document.addEventListener('DOMContentLoaded', function() {
+ if (initialPaused) {
+ document.dispatchEvent(new CustomEvent('zen-ws-toggle', { detail: { paused: true } }));
+ }
+ });
+
+ wsBtn.addEventListener('click', function() {
+ var paused = wsBtn.dataset.paused !== 'true';
+ try { localStorage.setItem(WS_KEY, paused ? 'true' : 'false'); } catch (e) {}
+ updateWsIcon(paused);
+ document.dispatchEvent(new CustomEvent('zen-ws-toggle', { detail: { paused: paused } }));
+ });
+
+ document.body.appendChild(wsBtn);
+ }
+
+ if (document.readyState === 'loading')
+ document.addEventListener('DOMContentLoaded', createToggle);
+ else
+ createToggle();
+})();
diff --git a/src/zenserver/frontend/html/util/compactbinary.js b/src/zenserver/frontend/html/util/compactbinary.js
index 90e4249f6..415fa4be8 100644
--- a/src/zenserver/frontend/html/util/compactbinary.js
+++ b/src/zenserver/frontend/html/util/compactbinary.js
@@ -310,8 +310,8 @@ CbFieldView.prototype.as_value = function(int_type=BigInt)
case CbFieldType.IntegerPositive: return VarInt.read_uint(this._data_view, int_type)[0];
case CbFieldType.IntegerNegative: return VarInt.read_int(this._data_view, int_type)[0];
- case CbFieldType.Float32: return new DataView(this._data_view.subarray(0, 4)).getFloat32(0, false);
- case CbFieldType.Float64: return new DataView(this._data_view.subarray(0, 8)).getFloat64(0, false);
+ case CbFieldType.Float32: { const s = this._data_view; return new DataView(s.buffer, s.byteOffset, 4).getFloat32(0, false); }
+ case CbFieldType.Float64: { const s = this._data_view; return new DataView(s.buffer, s.byteOffset, 8).getFloat64(0, false); }
case CbFieldType.BoolFalse: return false;
case CbFieldType.BoolTrue: return true;
diff --git a/src/zenserver/frontend/html/util/friendly.js b/src/zenserver/frontend/html/util/friendly.js
index a15252faf..5d4586165 100644
--- a/src/zenserver/frontend/html/util/friendly.js
+++ b/src/zenserver/frontend/html/util/friendly.js
@@ -20,4 +20,25 @@ export class Friendly
static kib(x, p=0) { return Friendly.sep((BigInt(x) + 1023n) / (1n << 10n)|0n, p) + " KiB"; }
static mib(x, p=1) { return Friendly.sep( BigInt(x) / (1n << 20n), p) + " MiB"; }
static gib(x, p=2) { return Friendly.sep( BigInt(x) / (1n << 30n), p) + " GiB"; }
+
+ static duration(s)
+ {
+ const v = Number(s);
+ if (v >= 1) return Friendly.sep(v, 2) + " s";
+ if (v >= 0.001) return Friendly.sep(v * 1000, 2) + " ms";
+ if (v >= 0.000001) return Friendly.sep(v * 1000000, 1) + " µs";
+ return Friendly.sep(v * 1000000000, 0) + " ns";
+ }
+
+ static bytes(x)
+ {
+ const v = BigInt(Math.trunc(Number(x)));
+ if (v >= (1n << 60n)) return Friendly.sep(Number(v) / Number(1n << 60n), 2) + " EiB";
+ if (v >= (1n << 50n)) return Friendly.sep(Number(v) / Number(1n << 50n), 2) + " PiB";
+ if (v >= (1n << 40n)) return Friendly.sep(Number(v) / Number(1n << 40n), 2) + " TiB";
+ if (v >= (1n << 30n)) return Friendly.sep(Number(v) / Number(1n << 30n), 2) + " GiB";
+ if (v >= (1n << 20n)) return Friendly.sep(Number(v) / Number(1n << 20n), 1) + " MiB";
+ if (v >= (1n << 10n)) return Friendly.sep(Number(v) / Number(1n << 10n), 0) + " KiB";
+ return Friendly.sep(Number(v), 0) + " B";
+ }
}
diff --git a/src/zenserver/frontend/html/util/widgets.js b/src/zenserver/frontend/html/util/widgets.js
index 32a3f4d28..2964f92f2 100644
--- a/src/zenserver/frontend/html/util/widgets.js
+++ b/src/zenserver/frontend/html/util/widgets.js
@@ -54,6 +54,8 @@ export class Table extends Widget
static Flag_PackRight = 1 << 1;
static Flag_BiasLeft = 1 << 2;
static Flag_FitLeft = 1 << 3;
+ static Flag_Sortable = 1 << 4;
+ static Flag_AlignNumeric = 1 << 5;
constructor(parent, column_names, flags=Table.Flag_EvenSpacing, index_base=0)
{
@@ -81,11 +83,108 @@ export class Table extends Widget
root.style("gridTemplateColumns", column_style);
- this._add_row(column_names, false);
+ this._header_row = this._add_row(column_names, false);
this._index = index_base;
this._num_columns = column_names.length;
this._rows = [];
+ this._flags = flags;
+ this._sort_column = -1;
+ this._sort_ascending = true;
+
+ if (flags & Table.Flag_Sortable)
+ {
+ this._init_sortable();
+ }
+ }
+
+ _init_sortable()
+ {
+ const header_elem = this._element.firstElementChild;
+ if (!header_elem)
+ {
+ return;
+ }
+
+ const cells = header_elem.children;
+ for (let i = 0; i < cells.length; i++)
+ {
+ const cell = cells[i];
+ cell.style.cursor = "pointer";
+ cell.style.userSelect = "none";
+ cell.addEventListener("click", () => this._sort_by(i));
+ }
+ }
+
+ _sort_by(column_index)
+ {
+ if (this._sort_column === column_index)
+ {
+ this._sort_ascending = !this._sort_ascending;
+ }
+ else
+ {
+ this._sort_column = column_index;
+ this._sort_ascending = true;
+ }
+
+ // Update header indicators
+ const header_elem = this._element.firstElementChild;
+ for (const cell of header_elem.children)
+ {
+ const text = cell.textContent.replace(/ [▲▼]$/, "");
+ cell.textContent = text;
+ }
+ const active_cell = header_elem.children[column_index];
+ active_cell.textContent += this._sort_ascending ? " â–˛" : " â–Ľ";
+
+ // Sort rows by comparing cell text content
+ const dir = this._sort_ascending ? 1 : -1;
+ const unit_multipliers = { "B": 1, "KiB": 1024, "MiB": 1048576, "GiB": 1073741824, "TiB": 1099511627776, "PiB": 1125899906842624, "EiB": 1152921504606846976 };
+ const parse_sortable = (text) => {
+ // Try byte units first (e.g. "1,234 KiB", "1.5 GiB")
+ const byte_match = text.match(/^([\d,.]+)\s*(B|[KMGTPE]iB)/);
+ if (byte_match)
+ {
+ const num = parseFloat(byte_match[1].replace(/,/g, ""));
+ const mult = unit_multipliers[byte_match[2]] || 1;
+ return num * mult;
+ }
+ // Try percentage (e.g. "95.5%")
+ const pct_match = text.match(/^([\d,.]+)%/);
+ if (pct_match)
+ {
+ return parseFloat(pct_match[1].replace(/,/g, ""));
+ }
+ // Try plain number (possibly with commas/separators)
+ const num = parseFloat(text.replace(/,/g, ""));
+ if (!isNaN(num))
+ {
+ return num;
+ }
+ return null;
+ };
+ this._rows.sort((a, b) => {
+ const aElem = a.inner().children[column_index];
+ const bElem = b.inner().children[column_index];
+ const aText = aElem ? aElem.textContent : "";
+ const bText = bElem ? bElem.textContent : "";
+
+ const aNum = parse_sortable(aText);
+ const bNum = parse_sortable(bText);
+
+ if (aNum !== null && bNum !== null)
+ {
+ return (aNum - bNum) * dir;
+ }
+ return aText.localeCompare(bText) * dir;
+ });
+
+ // Re-order DOM elements
+ for (const row of this._rows)
+ {
+ this._element.appendChild(row.inner());
+ }
}
*[Symbol.iterator]()
@@ -121,6 +220,18 @@ export class Table extends Widget
ret.push(new TableCell(leaf, row));
}
+ if ((this._flags & Table.Flag_AlignNumeric) && indexed)
+ {
+ for (const c of ret)
+ {
+ const t = c.inner().textContent;
+ if (t && /^\d/.test(t))
+ {
+ c.style("textAlign", "right");
+ }
+ }
+ }
+
if (this._index >= 0)
ret.shift();
@@ -131,9 +242,34 @@ export class Table extends Widget
{
var row = this._add_row(args);
this._rows.push(row);
+
+ if ((this._flags & Table.Flag_AlignNumeric) && this._rows.length === 1)
+ {
+ this._align_header();
+ }
+
return row;
}
+ _align_header()
+ {
+ const first_row = this._rows[0];
+ if (!first_row)
+ {
+ return;
+ }
+ const header_elem = this._element.firstElementChild;
+ const header_cells = header_elem.children;
+ const data_cells = first_row.inner().children;
+ for (let i = 0; i < data_cells.length && i < header_cells.length; i++)
+ {
+ if (data_cells[i].style.textAlign === "right")
+ {
+ header_cells[i].style.textAlign = "right";
+ }
+ }
+ }
+
clear(index=0)
{
const elem = this._element;
diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css
index cc53c0519..a968aecab 100644
--- a/src/zenserver/frontend/html/zen.css
+++ b/src/zenserver/frontend/html/zen.css
@@ -2,66 +2,202 @@
/* theme -------------------------------------------------------------------- */
+/* system preference (default) */
@media (prefers-color-scheme: light) {
:root {
- --theme_g0: #000;
- --theme_g4: #fff;
- --theme_g1: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 45%);
- --theme_g2: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 80%);
- --theme_g3: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 96%);
-
- --theme_p0: #069;
- --theme_p4: hsl(210deg 40% 94%);
+ --theme_g0: #1f2328;
+ --theme_g1: #656d76;
+ --theme_g2: #d0d7de;
+ --theme_g3: #f6f8fa;
+ --theme_g4: #ffffff;
+
+ --theme_p0: #0969da;
+ --theme_p4: #ddf4ff;
--theme_p1: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 35%);
--theme_p2: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 60%);
--theme_p3: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 85%);
--theme_ln: var(--theme_p0);
- --theme_er: #fcc;
+ --theme_er: #ffebe9;
+
+ --theme_ok: #1a7f37;
+ --theme_warn: #9a6700;
+ --theme_fail: #cf222e;
+
+ --theme_bright: #1f2328;
+ --theme_faint: #6e7781;
+ --theme_border_subtle: #d8dee4;
}
}
@media (prefers-color-scheme: dark) {
:root {
- --theme_g0: #ddd;
- --theme_g4: #222;
- --theme_g1: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 35%);
- --theme_g2: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 65%);
- --theme_g3: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 88%);
-
- --theme_p0: #479;
- --theme_p4: #333;
+ --theme_g0: #c9d1d9;
+ --theme_g1: #8b949e;
+ --theme_g2: #30363d;
+ --theme_g3: #161b22;
+ --theme_g4: #0d1117;
+
+ --theme_p0: #58a6ff;
+ --theme_p4: #1c2128;
--theme_p1: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 35%);
--theme_p2: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 60%);
--theme_p3: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 85%);
- --theme_ln: #feb;
- --theme_er: #622;
+ --theme_ln: #58a6ff;
+ --theme_er: #1c1c1c;
+
+ --theme_ok: #3fb950;
+ --theme_warn: #d29922;
+ --theme_fail: #f85149;
+
+ --theme_bright: #f0f6fc;
+ --theme_faint: #6e7681;
+ --theme_border_subtle: #21262d;
}
}
+/* manual overrides (higher specificity than media queries) */
+:root[data-theme="light"] {
+ --theme_g0: #1f2328;
+ --theme_g1: #656d76;
+ --theme_g2: #d0d7de;
+ --theme_g3: #f6f8fa;
+ --theme_g4: #ffffff;
+
+ --theme_p0: #0969da;
+ --theme_p4: #ddf4ff;
+ --theme_p1: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 35%);
+ --theme_p2: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 60%);
+ --theme_p3: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 85%);
+
+ --theme_ln: var(--theme_p0);
+ --theme_er: #ffebe9;
+
+ --theme_ok: #1a7f37;
+ --theme_warn: #9a6700;
+ --theme_fail: #cf222e;
+
+ --theme_bright: #1f2328;
+ --theme_faint: #6e7781;
+ --theme_border_subtle: #d8dee4;
+}
+
+:root[data-theme="dark"] {
+ --theme_g0: #c9d1d9;
+ --theme_g1: #8b949e;
+ --theme_g2: #30363d;
+ --theme_g3: #161b22;
+ --theme_g4: #0d1117;
+
+ --theme_p0: #58a6ff;
+ --theme_p4: #1c2128;
+ --theme_p1: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 35%);
+ --theme_p2: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 60%);
+ --theme_p3: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 85%);
+
+ --theme_ln: #58a6ff;
+ --theme_er: #1c1c1c;
+
+ --theme_ok: #3fb950;
+ --theme_warn: #d29922;
+ --theme_fail: #f85149;
+
+ --theme_bright: #f0f6fc;
+ --theme_faint: #6e7681;
+ --theme_border_subtle: #21262d;
+}
+
+/* theme toggle ------------------------------------------------------------- */
+
+#zen_ws_toggle {
+ position: fixed;
+ top: 16px;
+ right: 60px;
+ z-index: 10;
+ width: 36px;
+ height: 36px;
+ border-radius: 6px;
+ border: 1px solid var(--theme_g2);
+ background: var(--theme_g3);
+ color: var(--theme_g1);
+ cursor: pointer;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ font-size: 18px;
+ line-height: 1;
+ transition: color 0.15s, background 0.15s, border-color 0.15s;
+ user-select: none;
+}
+
+#zen_ws_toggle:hover {
+ color: var(--theme_g0);
+ background: var(--theme_p4);
+ border-color: var(--theme_g1);
+}
+
+#zen_theme_toggle {
+ position: fixed;
+ top: 16px;
+ right: 16px;
+ z-index: 10;
+ width: 36px;
+ height: 36px;
+ border-radius: 6px;
+ border: 1px solid var(--theme_g2);
+ background: var(--theme_g3);
+ color: var(--theme_g1);
+ cursor: pointer;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ font-size: 18px;
+ line-height: 1;
+ transition: color 0.15s, background 0.15s, border-color 0.15s;
+ user-select: none;
+}
+
+#zen_theme_toggle:hover {
+ color: var(--theme_g0);
+ background: var(--theme_p4);
+ border-color: var(--theme_g1);
+}
+
/* page --------------------------------------------------------------------- */
-body, input {
- font-family: consolas, monospace;
- font-size: 11pt;
+body, input, button {
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+ font-size: 14px;
}
body {
overflow-y: scroll;
margin: 0;
+ padding: 20px;
background-color: var(--theme_g4);
color: var(--theme_g0);
}
-pre {
- margin: 0;
+pre, code {
+ font-family: 'SF Mono', 'Cascadia Mono', Consolas, 'DejaVu Sans Mono', monospace;
+ font-size: 13px;
+ margin: 0;
}
input {
color: var(--theme_g0);
background-color: var(--theme_g3);
border: 1px solid var(--theme_g2);
+ border-radius: 4px;
+ padding: 4px 8px;
+}
+
+button {
+ color: var(--theme_g0);
+ background: transparent;
+ border: none;
+ cursor: pointer;
}
* {
@@ -69,17 +205,44 @@ input {
}
#container {
- max-width: 130em;
- min-width: 80em;
+ max-width: 1400px;
margin: auto;
> div {
- margin: 0.0em 2.2em 0.0em 2.2em;
padding-top: 1.0em;
padding-bottom: 1.5em;
}
}
+/* service nav -------------------------------------------------------------- */
+
+#service_nav {
+ display: flex;
+ align-items: center;
+ gap: 4px;
+ margin-bottom: 16px;
+ padding: 4px;
+ background-color: var(--theme_g3);
+ border: 1px solid var(--theme_g2);
+ border-radius: 6px;
+
+ a {
+ padding: 6px 14px;
+ border-radius: 4px;
+ font-size: 13px;
+ font-weight: 500;
+ color: var(--theme_g1);
+ text-decoration: none;
+ transition: color 0.15s, background 0.15s;
+ }
+
+ a:hover {
+ background-color: var(--theme_p4);
+ color: var(--theme_g0);
+ text-decoration: none;
+ }
+}
+
/* links -------------------------------------------------------------------- */
a {
@@ -103,28 +266,37 @@ a {
}
h1 {
- font-size: 1.5em;
+ font-size: 20px;
+ font-weight: 600;
width: 100%;
+ color: var(--theme_bright);
border-bottom: 1px solid var(--theme_g2);
+ padding-bottom: 0.4em;
+ margin-bottom: 16px;
}
h2 {
- font-size: 1.25em;
- margin-bottom: 0.5em;
+ font-size: 16px;
+ font-weight: 600;
+ margin-bottom: 12px;
}
h3 {
- font-size: 1.1em;
+ font-size: 14px;
+ font-weight: 600;
margin: 0em;
- padding: 0.4em;
- background-color: var(--theme_p4);
- border-left: 5px solid var(--theme_p2);
- font-weight: normal;
+ padding: 8px 12px;
+ background-color: var(--theme_g3);
+ border: 1px solid var(--theme_g2);
+ border-radius: 6px 6px 0 0;
+ color: var(--theme_g1);
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
}
- margin-bottom: 3em;
+ margin-bottom: 2em;
> *:not(h1) {
- margin-left: 2em;
+ margin-left: 0;
}
}
@@ -134,23 +306,36 @@ a {
.zen_table {
display: grid;
border: 1px solid var(--theme_g2);
- border-left-style: none;
+ border-radius: 6px;
+ overflow: hidden;
margin-bottom: 1.2em;
+ font-size: 13px;
> div {
display: contents;
}
- > div:nth-of-type(odd) {
+ > div:nth-of-type(odd) > div {
+ background-color: var(--theme_g4);
+ }
+
+ > div:nth-of-type(even) > div {
background-color: var(--theme_g3);
}
> div:first-of-type {
- font-weight: bold;
- background-color: var(--theme_p3);
+ font-weight: 600;
+ > div {
+ background-color: var(--theme_g3);
+ color: var(--theme_g1);
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+ font-size: 11px;
+ border-bottom: 1px solid var(--theme_g2);
+ }
}
- > div:hover {
+ > div:not(:first-of-type):hover > div {
background-color: var(--theme_p4);
}
@@ -160,16 +345,37 @@ a {
}
> div > div {
- padding: 0.3em;
- padding-left: 0.75em;
- padding-right: 0.75em;
+ padding: 8px 12px;
align-content: center;
- border-left: 1px solid var(--theme_g2);
+ border-left: 1px solid var(--theme_border_subtle);
overflow: auto;
overflow-wrap: break-word;
- background-color: inherit;
white-space: pre-wrap;
}
+
+ > div > div:first-child {
+ border-left: none;
+ }
+}
+
+/* expandable cell ---------------------------------------------------------- */
+
+.zen_expand_icon {
+ cursor: pointer;
+ margin-right: 0.5em;
+ color: var(--theme_g1);
+ font-weight: bold;
+ user-select: none;
+}
+
+.zen_expand_icon:hover {
+ color: var(--theme_ln);
+}
+
+.zen_data_text {
+ user-select: text;
+ font-family: 'SF Mono', 'Cascadia Mono', Consolas, 'DejaVu Sans Mono', monospace;
+ font-size: 13px;
}
/* toolbar ------------------------------------------------------------------ */
@@ -178,6 +384,7 @@ a {
display: flex;
margin-top: 0.5em;
margin-bottom: 0.6em;
+ font-size: 13px;
> div {
display: flex;
@@ -225,15 +432,16 @@ a {
z-index: -1;
top: 0;
left: 0;
- width: 100%;
+ width: 100%;
height: 100%;
background: var(--theme_g0);
opacity: 0.4;
}
> div {
- border-radius: 0.5em;
- background-color: var(--theme_g4);
+ border-radius: 6px;
+ background-color: var(--theme_g3);
+ border: 1px solid var(--theme_g2);
opacity: 1.0;
width: 35em;
padding: 0em 2em 2em 2em;
@@ -244,10 +452,11 @@ a {
}
.zen_modal_title {
- font-size: 1.2em;
+ font-size: 16px;
+ font-weight: 600;
border-bottom: 1px solid var(--theme_g2);
padding: 1.2em 0em 0.5em 0em;
- color: var(--theme_g1);
+ color: var(--theme_bright);
}
.zen_modal_buttons {
@@ -257,16 +466,19 @@ a {
> div {
margin: 0em 1em 0em 1em;
- padding: 1em;
+ padding: 10px 16px;
align-content: center;
- border-radius: 0.3em;
- background-color: var(--theme_p3);
+ border-radius: 6px;
+ background-color: var(--theme_p4);
+ border: 1px solid var(--theme_g2);
width: 6em;
cursor: pointer;
+ font-weight: 500;
+ transition: background 0.15s;
}
> div:hover {
- background-color: var(--theme_p4);
+ background-color: var(--theme_p3);
}
}
@@ -284,15 +496,18 @@ a {
top: 0;
left: 0;
width: 100%;
- height: 0.5em;
+ height: 4px;
+ border-radius: 2px;
+ overflow: hidden;
> div:first-of-type {
/* label */
padding: 0.3em;
- padding-top: 0.8em;
- background-color: var(--theme_p4);
+ padding-top: 8px;
+ background-color: var(--theme_g3);
width: max-content;
- font-size: 0.8em;
+ font-size: 12px;
+ color: var(--theme_g1);
}
> div:last-of-type {
@@ -302,7 +517,8 @@ a {
left: 0;
width: 0%;
height: 100%;
- background-color: var(--theme_p1);
+ background-color: var(--theme_p0);
+ transition: width 0.3s ease;
}
> div:nth-of-type(2) {
@@ -312,7 +528,7 @@ a {
left: 0;
width: 100%;
height: 100%;
- background-color: var(--theme_p3);
+ background-color: var(--theme_g3);
}
}
@@ -321,53 +537,25 @@ a {
#crumbs {
display: flex;
position: relative;
- top: -1em;
+ top: -0.5em;
+ font-size: 13px;
+ color: var(--theme_g1);
> div {
padding-right: 0.5em;
}
> div:nth-child(odd)::after {
- content: ":";
- font-weight: bolder;
- color: var(--theme_p2);
+ content: "/";
+ color: var(--theme_g2);
+ padding-left: 0.5em;
}
}
-/* branding ----------------------------------------------------------------- */
-
-#branding {
- font-size: 10pt;
- font-weight: bolder;
- margin-bottom: 2.6em;
- position: relative;
+/* banner ------------------------------------------------------------------- */
- #logo {
- width: min-content;
- margin: auto;
- user-select: none;
- position: relative;
-
- #go_home {
- width: 100%;
- height: 100%;
- position: absolute;
- top: 0;
- left: 0;
- }
- }
-
- #logo:hover {
- filter: drop-shadow(0 0.15em 0.1em var(--theme_p2));
- }
-
- #ue_logo {
- position: absolute;
- top: 1em;
- right: 0;
- width: 5em;
- margin: auto;
- }
+zen-banner {
+ margin-bottom: 24px;
}
/* error -------------------------------------------------------------------- */
@@ -378,26 +566,23 @@ a {
z-index: 1;
color: var(--theme_g0);
background-color: var(--theme_er);
- padding: 1.0em 2em 2em 2em;
+ padding: 12px 20px 16px 20px;
width: 100%;
- border-top: 1px solid var(--theme_g0);
+ border-top: 1px solid var(--theme_g2);
display: flex;
+ gap: 16px;
+ align-items: center;
+ font-size: 13px;
> div:nth-child(1) {
- font-size: 2.5em;
- font-weight: bolder;
- font-family: serif;
- transform: rotate(-13deg);
- color: var(--theme_p0);
- }
-
- > div:nth-child(2) {
- margin-left: 2em;
+ font-size: 24px;
+ font-weight: bold;
+ color: var(--theme_fail);
}
> div:nth-child(2) > pre:nth-child(2) {
- margin-top: 0.5em;
- font-size: 0.8em;
+ margin-top: 4px;
+ font-size: 12px;
color: var(--theme_g1);
}
}
@@ -409,18 +594,144 @@ a {
min-width: 15%;
}
+/* sections ----------------------------------------------------------------- */
+
+.zen_sector {
+ position: relative;
+}
+
+.dropall {
+ position: absolute;
+ top: 16px;
+ right: 0;
+ font-size: 12px;
+ margin: 0;
+}
+
+/* stats tiles -------------------------------------------------------------- */
+
+.stats-tiles {
+ grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+}
+
+.stats-tile {
+ cursor: pointer;
+ transition: border-color 0.15s, background 0.15s;
+}
+
+.stats-tile:hover {
+ border-color: var(--theme_p0);
+ background: var(--theme_p4);
+}
+
+.stats-tile-detailed {
+ position: relative;
+}
+
+.stats-tile-detailed::after {
+ content: "details \203A";
+ position: absolute;
+ bottom: 12px;
+ right: 20px;
+ font-size: 11px;
+ color: var(--theme_g1);
+ opacity: 0.6;
+ transition: opacity 0.15s;
+}
+
+.stats-tile-detailed:hover::after {
+ opacity: 1;
+ color: var(--theme_p0);
+}
+
+.stats-tile-selected {
+ border-color: var(--theme_p0);
+ background: var(--theme_p4);
+ box-shadow: 0 0 0 1px var(--theme_p0);
+}
+
+.stats-tile-selected::after {
+ content: "details \2039";
+ opacity: 1;
+ color: var(--theme_p0);
+}
+
+.tile-metrics {
+ display: flex;
+ flex-direction: column;
+ gap: 12px;
+}
+
+.tile-columns {
+ display: flex;
+ gap: 24px;
+}
+
+.tile-columns > .tile-metrics {
+ flex: 1;
+ min-width: 0;
+}
+
+.tile-metric .metric-value {
+ font-size: 16px;
+}
+
+.tile-metric-hero .metric-value {
+ font-size: 28px;
+}
+
/* start -------------------------------------------------------------------- */
#start {
- .dropall {
- text-align: right;
- font-size: 0.85em;
- margin: -0.5em 0 0.5em 0;
- }
#version {
- color: var(--theme_g1);
+ color: var(--theme_faint);
text-align: center;
- font-size: 0.85em;
+ font-size: 12px;
+ margin-top: 24px;
+ }
+}
+
+/* info --------------------------------------------------------------------- */
+
+#info {
+ .info-tiles {
+ grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
+ }
+
+ .info-tile {
+ overflow: hidden;
+ }
+
+ .info-props {
+ display: flex;
+ flex-direction: column;
+ gap: 1px;
+ font-size: 13px;
+ }
+
+ .info-prop {
+ display: flex;
+ gap: 12px;
+ padding: 4px 0;
+ border-bottom: 1px solid var(--theme_border_subtle);
+ }
+
+ .info-prop:last-child {
+ border-bottom: none;
+ }
+
+ .info-prop-label {
+ color: var(--theme_g1);
+ min-width: 140px;
+ flex-shrink: 0;
+ text-transform: capitalize;
+ }
+
+ .info-prop-value {
+ color: var(--theme_bright);
+ word-break: break-all;
+ margin-left: auto;
+ text-align: right;
}
}
@@ -437,6 +748,8 @@ a {
/* tree --------------------------------------------------------------------- */
#tree {
+ font-size: 13px;
+
#tree_root > ul {
margin-left: 0em;
}
@@ -448,29 +761,33 @@ a {
li > div {
display: flex;
border-bottom: 1px solid transparent;
- padding-left: 0.3em;
- padding-right: 0.3em;
+ padding: 4px 6px;
+ border-radius: 4px;
}
li > div > div[active] {
text-transform: uppercase;
+ color: var(--theme_p0);
+ font-weight: 600;
}
li > div > div:nth-last-child(3) {
margin-left: auto;
}
li > div > div:nth-last-child(-n + 3) {
- font-size: 0.8em;
+ font-size: 12px;
width: 10em;
text-align: right;
+ color: var(--theme_g1);
+ font-family: 'SF Mono', 'Cascadia Mono', Consolas, monospace;
}
li > div > div:nth-last-child(1) {
width: 6em;
}
li > div:hover {
background-color: var(--theme_p4);
- border-bottom: 1px solid var(--theme_g2);
+ border-bottom: 1px solid var(--theme_border_subtle);
}
li a {
- font-weight: bolder;
+ font-weight: 600;
}
li::marker {
content: "+";
@@ -503,3 +820,262 @@ html:has(#map) {
}
}
}
+
+/* ========================================================================== */
+/* Shared classes for compute / dashboard pages */
+/* ========================================================================== */
+
+/* cards -------------------------------------------------------------------- */
+
+.card {
+ background: var(--theme_g3);
+ border: 1px solid var(--theme_g2);
+ border-radius: 6px;
+ padding: 20px;
+}
+
+.card-title {
+ font-size: 14px;
+ font-weight: 600;
+ color: var(--theme_g1);
+ margin-bottom: 12px;
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+}
+
+/* grid --------------------------------------------------------------------- */
+
+.grid {
+ display: grid;
+ grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
+ gap: 20px;
+ margin-bottom: 24px;
+}
+
+/* metrics ------------------------------------------------------------------ */
+
+.metric-value {
+ font-size: 36px;
+ font-weight: 600;
+ color: var(--theme_bright);
+ line-height: 1;
+}
+
+.metric-label {
+ font-size: 12px;
+ color: var(--theme_g1);
+ margin-top: 4px;
+}
+
+/* section titles ----------------------------------------------------------- */
+
+.section-title {
+ font-size: 20px;
+ font-weight: 600;
+ margin-bottom: 16px;
+ color: var(--theme_bright);
+}
+
+/* html tables (compute pages) ---------------------------------------------- */
+
+table {
+ width: 100%;
+ border-collapse: collapse;
+ font-size: 13px;
+}
+
+th {
+ text-align: left;
+ color: var(--theme_g1);
+ padding: 8px 12px;
+ border-bottom: 1px solid var(--theme_g2);
+ font-weight: 600;
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+ font-size: 11px;
+}
+
+td {
+ padding: 8px 12px;
+ border-bottom: 1px solid var(--theme_border_subtle);
+ color: var(--theme_g0);
+}
+
+tr:last-child td {
+ border-bottom: none;
+}
+
+.total-row td {
+ border-top: 2px solid var(--theme_g2);
+ font-weight: 600;
+ color: var(--theme_bright);
+}
+
+/* status badges ------------------------------------------------------------ */
+
+.status-badge {
+ display: inline-block;
+ padding: 2px 8px;
+ border-radius: 4px;
+ font-size: 11px;
+ font-weight: 600;
+}
+
+.status-badge.active,
+.status-badge.success {
+ background: color-mix(in srgb, var(--theme_ok) 15%, transparent);
+ color: var(--theme_ok);
+}
+
+.status-badge.inactive {
+ background: color-mix(in srgb, var(--theme_g1) 15%, transparent);
+ color: var(--theme_g1);
+}
+
+.status-badge.failure {
+ background: color-mix(in srgb, var(--theme_fail) 15%, transparent);
+ color: var(--theme_fail);
+}
+
+/* health dots -------------------------------------------------------------- */
+
+.health-dot {
+ display: inline-block;
+ width: 10px;
+ height: 10px;
+ border-radius: 50%;
+ background: var(--theme_g1);
+}
+
+.health-green {
+ background: var(--theme_ok);
+}
+
+.health-yellow {
+ background: var(--theme_warn);
+}
+
+.health-red {
+ background: var(--theme_fail);
+}
+
+/* inline progress bar ------------------------------------------------------ */
+
+.progress-bar {
+ width: 100%;
+ height: 8px;
+ background: var(--theme_border_subtle);
+ border-radius: 4px;
+ overflow: hidden;
+ margin-top: 8px;
+}
+
+.progress-fill {
+ height: 100%;
+ background: var(--theme_p0);
+ transition: width 0.3s ease;
+}
+
+/* stats row (label + value pair) ------------------------------------------- */
+
+.stats-row {
+ display: flex;
+ justify-content: space-between;
+ margin-bottom: 12px;
+ padding: 8px 0;
+ border-bottom: 1px solid var(--theme_border_subtle);
+}
+
+.stats-row:last-child {
+ border-bottom: none;
+ margin-bottom: 0;
+}
+
+.stats-label {
+ color: var(--theme_g1);
+ font-size: 13px;
+}
+
+.stats-value {
+ color: var(--theme_bright);
+ font-weight: 600;
+ font-size: 13px;
+}
+
+/* detail tag (inline badge) ------------------------------------------------ */
+
+.detail-tag {
+ display: inline-block;
+ padding: 2px 8px;
+ border-radius: 4px;
+ background: var(--theme_border_subtle);
+ color: var(--theme_g0);
+ font-size: 11px;
+ margin: 2px 4px 2px 0;
+}
+
+/* timestamp ---------------------------------------------------------------- */
+
+.timestamp {
+ font-size: 12px;
+ color: var(--theme_faint);
+}
+
+/* inline error ------------------------------------------------------------- */
+
+.error {
+ color: var(--theme_fail);
+ padding: 12px;
+ background: var(--theme_er);
+ border-radius: 6px;
+ margin: 20px 0;
+ font-size: 13px;
+}
+
+/* empty state -------------------------------------------------------------- */
+
+.empty-state {
+ color: var(--theme_faint);
+ font-size: 13px;
+ padding: 20px 0;
+ text-align: center;
+}
+
+/* header layout ------------------------------------------------------------ */
+
+.header {
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ margin-bottom: 24px;
+}
+
+/* history tabs ------------------------------------------------------------- */
+
+.history-tabs {
+ display: flex;
+ gap: 4px;
+ background: var(--theme_g4);
+ border-radius: 6px;
+ padding: 2px;
+}
+
+.history-tab {
+ background: transparent;
+ color: var(--theme_g1);
+ font-size: 12px;
+ font-weight: 600;
+ padding: 4px 12px;
+ border-radius: 4px;
+ text-transform: uppercase;
+ letter-spacing: 0.5px;
+}
+
+.history-tab:hover {
+ color: var(--theme_g0);
+}
+
+.history-tab.active {
+ background: var(--theme_g2);
+ color: var(--theme_bright);
+}
diff --git a/src/zenserver/frontend/zipfs.cpp b/src/zenserver/frontend/zipfs.cpp
index f9c2bc8ff..42df0520f 100644
--- a/src/zenserver/frontend/zipfs.cpp
+++ b/src/zenserver/frontend/zipfs.cpp
@@ -149,13 +149,25 @@ ZipFs::ZipFs(IoBuffer&& Buffer)
IoBuffer
ZipFs::GetFile(const std::string_view& FileName) const
{
- FileMap::iterator Iter = m_Files.find(FileName);
- if (Iter == m_Files.end())
{
- return {};
+ RwLock::SharedLockScope _(m_FilesLock);
+
+ FileMap::const_iterator Iter = m_Files.find(FileName);
+ if (Iter == m_Files.end())
+ {
+ return {};
+ }
+
+ const FileItem& Item = Iter->second;
+ if (Item.GetSize() > 0)
+ {
+ return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize());
+ }
}
- FileItem& Item = Iter->second;
+ RwLock::ExclusiveLockScope _(m_FilesLock);
+
+ FileItem& Item = m_Files.find(FileName)->second;
if (Item.GetSize() > 0)
{
return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize());
diff --git a/src/zenserver/frontend/zipfs.h b/src/zenserver/frontend/zipfs.h
index 1fa7da451..645121693 100644
--- a/src/zenserver/frontend/zipfs.h
+++ b/src/zenserver/frontend/zipfs.h
@@ -3,23 +3,23 @@
#pragma once
#include <zencore/iobuffer.h>
+#include <zencore/thread.h>
#include <unordered_map>
namespace zen {
-//////////////////////////////////////////////////////////////////////////
class ZipFs
{
public:
- ZipFs() = default;
- ZipFs(IoBuffer&& Buffer);
+ explicit ZipFs(IoBuffer&& Buffer);
+
IoBuffer GetFile(const std::string_view& FileName) const;
- inline operator bool() const { return !m_Files.empty(); }
private:
using FileItem = MemoryView;
using FileMap = std::unordered_map<std::string_view, FileItem>;
+ mutable RwLock m_FilesLock;
FileMap mutable m_Files;
IoBuffer m_Buffer;
};
diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp
index 4d9da3a57..7b999ae20 100644
--- a/src/zenserver/hub/hubservice.cpp
+++ b/src/zenserver/hub/hubservice.cpp
@@ -4,10 +4,12 @@
#include "hydration.h"
+#include <zencore/assertfmt.h>
#include <zencore/compactbinarybuilder.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
+#include <zencore/process.h>
#include <zencore/scopeguard.h>
#include <zencore/system.h>
#include <zenutil/zenserverprocess.h>
@@ -150,7 +152,12 @@ struct StorageServerInstance
inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); }
+#if ZEN_PLATFORM_WINDOWS
+ void SetJobObject(JobObject* InJobObject) { m_JobObject = InJobObject; }
+#endif
+
private:
+ void WakeLocked();
RwLock m_Lock;
std::string m_ModuleId;
std::atomic<bool> m_IsProvisioned{false};
@@ -160,6 +167,9 @@ private:
std::filesystem::path m_TempDir;
std::filesystem::path m_HydrationPath;
ResourceMetrics m_ResourceMetrics;
+#if ZEN_PLATFORM_WINDOWS
+ JobObject* m_JobObject = nullptr;
+#endif
void SpawnServerProcess();
@@ -186,10 +196,13 @@ StorageServerInstance::~StorageServerInstance()
void
StorageServerInstance::SpawnServerProcess()
{
- ZEN_ASSERT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId);
+ ZEN_ASSERT_FORMAT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId);
m_ServerInstance.SetServerExecutablePath(GetRunningExecutablePath());
m_ServerInstance.SetDataDir(m_BaseDir);
+#if ZEN_PLATFORM_WINDOWS
+ m_ServerInstance.SetJobObject(m_JobObject);
+#endif
const uint16_t BasePort = m_ServerInstance.SpawnServerAndWaitUntilReady();
ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, BasePort);
@@ -211,7 +224,7 @@ StorageServerInstance::Provision()
if (m_IsHibernated)
{
- Wake();
+ WakeLocked();
}
else
{
@@ -294,9 +307,14 @@ StorageServerInstance::Hibernate()
void
StorageServerInstance::Wake()
{
- // Start server in-place using existing data
-
RwLock::ExclusiveLockScope _(m_Lock);
+ WakeLocked();
+}
+
+void
+StorageServerInstance::WakeLocked()
+{
+ // Start server in-place using existing data
if (!m_IsHibernated)
{
@@ -305,7 +323,7 @@ StorageServerInstance::Wake()
return;
}
- ZEN_ASSERT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId);
+ ZEN_ASSERT_FORMAT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId);
try
{
@@ -374,6 +392,21 @@ struct HttpHubService::Impl
// flexibility, and to allow running multiple hubs on the same host if
// necessary.
m_RunEnvironment.SetNextPortNumber(21000);
+
+#if ZEN_PLATFORM_WINDOWS
+ if (m_UseJobObject)
+ {
+ m_JobObject.Initialize();
+ if (m_JobObject.IsValid())
+ {
+ ZEN_INFO("Job object initialized for hub service child process management");
+ }
+ else
+ {
+ ZEN_WARN("Failed to initialize job object; child processes will not be auto-terminated on hub crash");
+ }
+ }
+#endif
}
void Cleanup()
@@ -416,6 +449,12 @@ struct HttpHubService::Impl
IsNewInstance = true;
auto NewInstance =
std::make_unique<StorageServerInstance>(m_RunEnvironment, ModuleId, m_FileHydrationPath, m_HydrationTempPath);
+#if ZEN_PLATFORM_WINDOWS
+ if (m_JobObject.IsValid())
+ {
+ NewInstance->SetJobObject(&m_JobObject);
+ }
+#endif
Instance = NewInstance.get();
m_Instances.emplace(std::string(ModuleId), std::move(NewInstance));
@@ -573,10 +612,15 @@ struct HttpHubService::Impl
inline int GetInstanceLimit() { return m_InstanceLimit; }
inline int GetMaxInstanceCount() { return m_MaxInstanceCount; }
+ bool m_UseJobObject = true;
+
private:
- ZenServerEnvironment m_RunEnvironment;
- std::filesystem::path m_FileHydrationPath;
- std::filesystem::path m_HydrationTempPath;
+ ZenServerEnvironment m_RunEnvironment;
+ std::filesystem::path m_FileHydrationPath;
+ std::filesystem::path m_HydrationTempPath;
+#if ZEN_PLATFORM_WINDOWS
+ JobObject m_JobObject;
+#endif
RwLock m_Lock;
std::unordered_map<std::string, std::unique_ptr<StorageServerInstance>> m_Instances;
std::unordered_set<std::string> m_DeprovisioningModules;
@@ -802,7 +846,7 @@ HttpHubService::HttpHubService(std::filesystem::path HubBaseDir, std::filesystem
Obj << "currentInstanceCount" << m_Impl->GetInstanceCount();
Obj << "maxInstanceCount" << m_Impl->GetMaxInstanceCount();
Obj << "instanceLimit" << m_Impl->GetInstanceLimit();
- Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
},
HttpVerb::kGet);
}
@@ -811,6 +855,12 @@ HttpHubService::~HttpHubService()
{
}
+void
+HttpHubService::SetUseJobObject(bool Enable)
+{
+ m_Impl->m_UseJobObject = Enable;
+}
+
const char*
HttpHubService::BaseUri() const
{
diff --git a/src/zenserver/hub/hubservice.h b/src/zenserver/hub/hubservice.h
index 1a5a8c57c..ef24bba69 100644
--- a/src/zenserver/hub/hubservice.h
+++ b/src/zenserver/hub/hubservice.h
@@ -28,6 +28,13 @@ public:
void SetNotificationEndpoint(std::string_view UpstreamNotificationEndpoint, std::string_view InstanceId);
+ /** Enable or disable the use of a Windows Job Object for child process management.
+ * When enabled, all spawned child processes are assigned to a job object with
+ * JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, ensuring children are terminated if the hub
+ * crashes or is force-killed. Must be called before Initialize(). No-op on non-Windows.
+ */
+ void SetUseJobObject(bool Enable);
+
private:
HttpRequestRouter m_Router;
diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
index 7a4ba951d..c6d2dc8d4 100644
--- a/src/zenserver/hub/zenhubserver.cpp
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -105,7 +105,7 @@ ZenHubServer::Initialize(const ZenHubServerConfig& ServerConfig, ZenServerState:
void
ZenHubServer::Cleanup()
{
- ZEN_TRACE_CPU("ZenStorageServer::Cleanup");
+ ZEN_TRACE_CPU("ZenHubServer::Cleanup");
ZEN_INFO(ZEN_APP_NAME " cleaning up");
try
{
@@ -115,6 +115,8 @@ ZenHubServer::Cleanup()
m_IoRunner.join();
}
+ ShutdownServices();
+
if (m_Http)
{
m_Http->Close();
@@ -143,6 +145,8 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig)
ZEN_INFO("instantiating hub service");
m_HubService = std::make_unique<HttpHubService>(ServerConfig.DataDir / "hub", ServerConfig.DataDir / "servers");
m_HubService->SetNotificationEndpoint(ServerConfig.UpstreamNotificationEndpoint, ServerConfig.InstanceId);
+
+ m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
}
void
@@ -159,6 +163,11 @@ ZenHubServer::RegisterServices(const ZenHubServerConfig& ServerConfig)
{
m_Http->RegisterService(*m_ApiService);
}
+
+ if (m_FrontendService)
+ {
+ m_Http->RegisterService(*m_FrontendService);
+ }
}
void
diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h
index ac14362f0..4c56fdce5 100644
--- a/src/zenserver/hub/zenhubserver.h
+++ b/src/zenserver/hub/zenhubserver.h
@@ -2,6 +2,7 @@
#pragma once
+#include "frontend/frontend.h"
#include "zenserver.h"
namespace cxxopts {
@@ -81,8 +82,9 @@ private:
std::filesystem::path m_ContentRoot;
bool m_DebugOptionForcedCrash = false;
- std::unique_ptr<HttpHubService> m_HubService;
- std::unique_ptr<HttpApiService> m_ApiService;
+ std::unique_ptr<HttpHubService> m_HubService;
+ std::unique_ptr<HttpApiService> m_ApiService;
+ std::unique_ptr<HttpFrontendService> m_FrontendService;
void InitializeState(const ZenHubServerConfig& ServerConfig);
void InitializeServices(const ZenHubServerConfig& ServerConfig);
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 3a58d1f4a..09ecc48e5 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -19,10 +19,13 @@
#include <zencore/thread.h>
#include <zencore/trace.h>
#include <zentelemetry/otlptrace.h>
-#include <zenutil/commandlineoptions.h>
+#include <zenutil/config/commandlineoptions.h>
#include <zenutil/service.h>
#include "diag/logging.h"
+
+#include "compute/computeserver.h"
+
#include "storage/storageconfig.h"
#include "storage/zenstorageserver.h"
@@ -38,7 +41,6 @@
// in some shared code into the executable
#if ZEN_WITH_TESTS
-# define ZEN_TEST_WITH_RUNNER 1
# include <zencore/testing.h>
#endif
@@ -61,11 +63,19 @@ namespace zen {
#if ZEN_PLATFORM_WINDOWS
-template<class T>
+/** Windows Service wrapper for Zen servers
+ *
+ * This class wraps a Zen server main entry point (the Main template parameter)
+ * into a Windows Service by implementing the WindowsService interface.
+ *
+ * The Main type needs to implement the virtual functions from the ZenServerMain
+ * base class, which provides the actual server logic.
+ */
+template<class Main>
class ZenWindowsService : public WindowsService
{
public:
- ZenWindowsService(typename T::Config& ServerOptions) : m_EntryPoint(ServerOptions) {}
+ ZenWindowsService(typename Main::Config& ServerOptions) : m_EntryPoint(ServerOptions) {}
ZenWindowsService(const ZenWindowsService&) = delete;
ZenWindowsService& operator=(const ZenWindowsService&) = delete;
@@ -73,7 +83,7 @@ public:
virtual int Run() override { return m_EntryPoint.Run(); }
private:
- T m_EntryPoint;
+ Main m_EntryPoint;
};
#endif // ZEN_PLATFORM_WINDOWS
@@ -84,6 +94,23 @@ private:
namespace zen {
+/** Application main entry point template
+ *
+ * This function handles common application startup tasks while allowing
+ * different server types to be plugged in via the Main template parameter.
+ *
+ * On Windows, this function also handles platform-specific service
+ * installation and uninstallation.
+ *
+ * The Main type needs to implement the virtual functions from the ZenServerMain
+ * base class, which provides the actual server logic.
+ *
+ * The Main type is also expected to provide the following members:
+ *
+ * typedef Config -- Server configuration type, derived from ZenServerConfig
+ * typedef Configurator -- Server configuration handler type, implements ZenServerConfiguratorBase
+ *
+ */
template<class Main>
int
AppMain(int argc, char* argv[])
@@ -219,7 +246,7 @@ test_main(int argc, char** argv)
# endif // ZEN_PLATFORM_WINDOWS
zen::logging::InitializeLogging();
- zen::logging::SetLogLevel(zen::logging::level::Debug);
+ zen::logging::SetLogLevel(zen::logging::Debug);
zen::MaximizeOpenFileCount();
@@ -239,16 +266,31 @@ main(int argc, char* argv[])
using namespace zen;
using namespace std::literals;
+ // note: doctest has locally (in thirdparty) been fixed to not cause shutdown
+ // crashes due to TLS destructors
+ //
+ // mimalloc on the other hand might still be causing issues, in which case
+ // we should work out either how to eliminate the mimalloc dependency or how
+ // to configure it in a way that doesn't cause shutdown issues
+
+#if 0
auto _ = zen::MakeGuard([] {
// Allow some time for worker threads to unravel, in an effort
- // to prevent shutdown races in TLS object destruction
+ // to prevent shutdown races in TLS object destruction, mainly due to
+ // threads which we don't directly control (Windows thread pool) and
+ // therefore can't join.
+ //
+ // This isn't a great solution, but for now it seems to help reduce
+ // shutdown crashes observed in some situations.
WaitForThreads(1000);
});
+#endif
enum
{
kHub,
kStore,
+ kCompute,
kTest
} ServerMode = kStore;
@@ -258,10 +300,14 @@ main(int argc, char* argv[])
{
ServerMode = kHub;
}
- else if (argv[1] == "store"sv)
+ else if ((argv[1] == "store"sv) || (argv[1] == "storage"sv))
{
ServerMode = kStore;
}
+ else if (argv[1] == "compute"sv)
+ {
+ ServerMode = kCompute;
+ }
else if (argv[1] == "test"sv)
{
ServerMode = kTest;
@@ -280,6 +326,13 @@ main(int argc, char* argv[])
break;
case kHub:
return AppMain<ZenHubServerMain>(argc, argv);
+ case kCompute:
+#if ZEN_WITH_COMPUTE_SERVICES
+ return AppMain<ZenComputeServerMain>(argc, argv);
+#else
+ fprintf(stderr, "compute services are not compiled in!\n");
+ exit(5);
+#endif
default:
case kStore:
return AppMain<ZenStorageServerMain>(argc, argv);
diff --git a/src/zenserver/sessions/httpsessions.cpp b/src/zenserver/sessions/httpsessions.cpp
new file mode 100644
index 000000000..05be3c814
--- /dev/null
+++ b/src/zenserver/sessions/httpsessions.cpp
@@ -0,0 +1,264 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "httpsessions.h"
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/trace.h>
+#include "sessions.h"
+
+namespace zen {
+using namespace std::literals;
+
+HttpSessionsService::HttpSessionsService(HttpStatusService& StatusService, HttpStatsService& StatsService, SessionsService& Sessions)
+: m_Log(logging::Get("sessions"))
+, m_StatusService(StatusService)
+, m_StatsService(StatsService)
+, m_Sessions(Sessions)
+{
+ Initialize();
+}
+
+HttpSessionsService::~HttpSessionsService()
+{
+ m_StatsService.UnregisterHandler("sessions", *this);
+ m_StatusService.UnregisterHandler("sessions", *this);
+}
+
+const char*
+HttpSessionsService::BaseUri() const
+{
+ return "/sessions/";
+}
+
+void
+HttpSessionsService::HandleRequest(HttpServerRequest& Request)
+{
+ metrics::OperationTiming::Scope $(m_HttpRequests);
+
+ if (m_Router.HandleRequest(Request) == false)
+ {
+ ZEN_WARN("No route found for {0}", Request.RelativeUri());
+ return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
+ }
+}
+
+CbObject
+HttpSessionsService::CollectStats()
+{
+ ZEN_TRACE_CPU("SessionsService::Stats");
+ CbObjectWriter Cbo;
+
+ EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+ Cbo.BeginObject("sessions");
+ {
+ Cbo << "readcount" << m_SessionsStats.SessionReadCount;
+ Cbo << "writecount" << m_SessionsStats.SessionWriteCount;
+ Cbo << "deletecount" << m_SessionsStats.SessionDeleteCount;
+ Cbo << "listcount" << m_SessionsStats.SessionListCount;
+ Cbo << "requestcount" << m_SessionsStats.RequestCount;
+ Cbo << "badrequestcount" << m_SessionsStats.BadRequestCount;
+ Cbo << "count" << m_Sessions.GetSessionCount();
+ }
+ Cbo.EndObject();
+
+ return Cbo.Save();
+}
+
+void
+HttpSessionsService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+ HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
+}
+
+void
+HttpSessionsService::HandleStatusRequest(HttpServerRequest& Request)
+{
+ ZEN_TRACE_CPU("HttpSessionsService::Status");
+ CbObjectWriter Cbo;
+ Cbo << "ok" << true;
+ Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
+HttpSessionsService::Initialize()
+{
+ using namespace std::literals;
+
+ ZEN_INFO("Initializing Sessions Service");
+
+ static constexpr AsciiSet ValidHexCharactersSet{"0123456789abcdefABCDEF"};
+
+ m_Router.AddMatcher("session_id", [](std::string_view Str) -> bool {
+ return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet);
+ });
+
+ m_Router.RegisterRoute(
+ "list",
+ [this](HttpRouterRequest& Req) { ListSessionsRequest(Req); },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "{session_id}",
+ [this](HttpRouterRequest& Req) { SessionRequest(Req); },
+ HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kPut | HttpVerb::kDelete);
+
+ m_Router.RegisterRoute(
+ "",
+ [this](HttpRouterRequest& Req) { ListSessionsRequest(Req); },
+ HttpVerb::kGet);
+
+ m_StatsService.RegisterHandler("sessions", *this);
+ m_StatusService.RegisterHandler("sessions", *this);
+}
+
+static void
+WriteSessionInfo(CbWriter& Writer, const SessionsService::SessionInfo& Info)
+{
+ Writer << "id" << Info.Id;
+ if (!Info.AppName.empty())
+ {
+ Writer << "appname" << Info.AppName;
+ }
+ if (Info.JobId != Oid::Zero)
+ {
+ Writer << "jobid" << Info.JobId;
+ }
+ Writer << "created_at" << Info.CreatedAt;
+ Writer << "updated_at" << Info.UpdatedAt;
+
+ if (Info.Metadata.GetSize() > 0)
+ {
+ Writer.BeginObject("metadata");
+ for (const CbField& Field : Info.Metadata)
+ {
+ Writer.AddField(Field);
+ }
+ Writer.EndObject();
+ }
+}
+
+void
+HttpSessionsService::ListSessionsRequest(HttpRouterRequest& Req)
+{
+ HttpServerRequest& ServerRequest = Req.ServerRequest();
+
+ m_SessionsStats.SessionListCount++;
+ m_SessionsStats.RequestCount++;
+
+ std::vector<Ref<SessionsService::Session>> Sessions = m_Sessions.GetSessions();
+
+ CbObjectWriter Response;
+ Response.BeginArray("sessions");
+ for (const Ref<SessionsService::Session>& Session : Sessions)
+ {
+ Response.BeginObject();
+ {
+ WriteSessionInfo(Response, Session->Info());
+ }
+ Response.EndObject();
+ }
+ Response.EndArray();
+
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save());
+}
+
+void
+HttpSessionsService::SessionRequest(HttpRouterRequest& Req)
+{
+ HttpServerRequest& ServerRequest = Req.ServerRequest();
+
+ const Oid SessionId = Oid::TryFromHexString(Req.GetCapture(1));
+ if (SessionId == Oid::Zero)
+ {
+ m_SessionsStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Invalid session id '{}'", Req.GetCapture(1)));
+ }
+
+ m_SessionsStats.RequestCount++;
+
+ switch (ServerRequest.RequestVerb())
+ {
+ case HttpVerb::kPost:
+ case HttpVerb::kPut:
+ {
+ IoBuffer Payload = ServerRequest.ReadPayload();
+ CbObject RequestObject;
+
+ if (Payload.GetSize() > 0)
+ {
+ if (CbValidateError ValidationResult = ValidateCompactBinary(Payload.GetView(), CbValidateMode::All);
+ ValidationResult != CbValidateError::None)
+ {
+ m_SessionsStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Invalid payload: {}", zen::ToString(ValidationResult)));
+ }
+ RequestObject = LoadCompactBinaryObject(Payload);
+ }
+
+ if (ServerRequest.RequestVerb() == HttpVerb::kPost)
+ {
+ std::string AppName(RequestObject["appname"sv].AsString());
+ Oid JobId = RequestObject["jobid"sv].AsObjectId();
+ CbObjectView MetadataView = RequestObject["metadata"sv].AsObjectView();
+
+ m_SessionsStats.SessionWriteCount++;
+ if (m_Sessions.RegisterSession(SessionId, std::move(AppName), JobId, MetadataView))
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", SessionId));
+ }
+ else
+ {
+ // Already exists - try update instead
+ if (m_Sessions.UpdateSession(SessionId, MetadataView))
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", SessionId));
+ }
+ return ServerRequest.WriteResponse(HttpResponseCode::InternalServerError);
+ }
+ }
+ else
+ {
+ // PUT - update only
+ m_SessionsStats.SessionWriteCount++;
+ if (m_Sessions.UpdateSession(SessionId, RequestObject["metadata"sv].AsObjectView()))
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", SessionId));
+ }
+ return ServerRequest.WriteResponse(HttpResponseCode::NotFound,
+ HttpContentType::kText,
+ fmt::format("Session '{}' not found", SessionId));
+ }
+ }
+ case HttpVerb::kGet:
+ {
+ m_SessionsStats.SessionReadCount++;
+ Ref<SessionsService::Session> Session = m_Sessions.GetSession(SessionId);
+ if (Session)
+ {
+ CbObjectWriter Response;
+ WriteSessionInfo(Response, Session->Info());
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save());
+ }
+ return ServerRequest.WriteResponse(HttpResponseCode::NotFound);
+ }
+ case HttpVerb::kDelete:
+ {
+ m_SessionsStats.SessionDeleteCount++;
+ if (m_Sessions.RemoveSession(SessionId))
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::OK);
+ }
+ return ServerRequest.WriteResponse(HttpResponseCode::NotFound);
+ }
+ }
+}
+
+} // namespace zen
diff --git a/src/zenserver/sessions/httpsessions.h b/src/zenserver/sessions/httpsessions.h
new file mode 100644
index 000000000..e07f3b59b
--- /dev/null
+++ b/src/zenserver/sessions/httpsessions.h
@@ -0,0 +1,55 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenhttp/httpserver.h>
+#include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
+#include <zentelemetry/stats.h>
+
+namespace zen {
+
+class SessionsService;
+
+class HttpSessionsService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
+{
+public:
+ HttpSessionsService(HttpStatusService& StatusService, HttpStatsService& StatsService, SessionsService& Sessions);
+ virtual ~HttpSessionsService();
+
+ virtual const char* BaseUri() const override;
+ virtual void HandleRequest(HttpServerRequest& Request) override;
+
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+
+private:
+ struct SessionsStats
+ {
+ std::atomic_uint64_t SessionReadCount{};
+ std::atomic_uint64_t SessionWriteCount{};
+ std::atomic_uint64_t SessionDeleteCount{};
+ std::atomic_uint64_t SessionListCount{};
+ std::atomic_uint64_t RequestCount{};
+ std::atomic_uint64_t BadRequestCount{};
+ };
+
+ inline LoggerRef Log() { return m_Log; }
+
+ LoggerRef m_Log;
+
+ void Initialize();
+
+ void ListSessionsRequest(HttpRouterRequest& Req);
+ void SessionRequest(HttpRouterRequest& Req);
+
+ HttpStatusService& m_StatusService;
+ HttpStatsService& m_StatsService;
+ HttpRequestRouter m_Router;
+ SessionsService& m_Sessions;
+ SessionsStats m_SessionsStats;
+ metrics::OperationTiming m_HttpRequests;
+};
+
+} // namespace zen
diff --git a/src/zenserver/sessions/sessions.cpp b/src/zenserver/sessions/sessions.cpp
new file mode 100644
index 000000000..f73aa40ff
--- /dev/null
+++ b/src/zenserver/sessions/sessions.cpp
@@ -0,0 +1,150 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "sessions.h"
+
+#include <zencore/basicfile.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+
+namespace zen {
+using namespace std::literals;
+
+class SessionLog : public TRefCounted<SessionLog>
+{
+public:
+ SessionLog(std::filesystem::path LogFilePath) { m_LogFile.Open(LogFilePath, BasicFile::Mode::kWrite); }
+
+private:
+ BasicFile m_LogFile;
+};
+
+class SessionLogStore
+{
+public:
+ SessionLogStore(std::filesystem::path StoragePath) : m_StoragePath(std::move(StoragePath)) {}
+
+ ~SessionLogStore() = default;
+
+ Ref<SessionLog> GetLogForSession(const Oid& SessionId)
+ {
+ // For now, just return a new log for each session. We can implement actual log storage and retrieval later.
+ return Ref(new SessionLog(m_StoragePath / (SessionId.ToString() + ".log")));
+ }
+
+ Ref<SessionLog> CreateLogForSession(const Oid& SessionId)
+ {
+ // For now, just return a new log for each session. We can implement actual log storage and retrieval later.
+ return Ref(new SessionLog(m_StoragePath / (SessionId.ToString() + ".log")));
+ }
+
+private:
+ std::filesystem::path m_StoragePath;
+};
+
+SessionsService::Session::Session(const SessionInfo& Info) : m_Info(Info)
+{
+}
+SessionsService::Session::~Session() = default;
+
+//////////////////////////////////////////////////////////////////////////
+
+SessionsService::SessionsService() : m_Log(logging::Get("sessions"))
+{
+}
+
+SessionsService::~SessionsService() = default;
+
+bool
+SessionsService::RegisterSession(const Oid& SessionId, std::string AppName, const Oid& JobId, CbObjectView Metadata)
+{
+ RwLock::ExclusiveLockScope Lock(m_Lock);
+
+ if (m_Sessions.contains(SessionId))
+ {
+ return false;
+ }
+
+ const DateTime Now = DateTime::Now();
+ m_Sessions.emplace(SessionId,
+ Ref(new Session(SessionInfo{.Id = SessionId,
+ .AppName = std::move(AppName),
+ .JobId = JobId,
+ .Metadata = CbObject::Clone(Metadata),
+ .CreatedAt = Now,
+ .UpdatedAt = Now})));
+
+ ZEN_INFO("Session {} registered (AppName: {}, JobId: {})", SessionId, AppName, JobId);
+ return true;
+}
+
+bool
+SessionsService::UpdateSession(const Oid& SessionId, CbObjectView Metadata)
+{
+ RwLock::ExclusiveLockScope Lock(m_Lock);
+
+ auto It = m_Sessions.find(SessionId);
+ if (It == m_Sessions.end())
+ {
+ return false;
+ }
+
+ It.value()->UpdateMetadata(Metadata);
+
+ const SessionInfo& Info = It.value()->Info();
+ ZEN_DEBUG("Session {} updated (AppName: {}, JobId: {})", SessionId, Info.AppName, Info.JobId);
+ return true;
+}
+
+Ref<SessionsService::Session>
+SessionsService::GetSession(const Oid& SessionId) const
+{
+ RwLock::SharedLockScope Lock(m_Lock);
+
+ auto It = m_Sessions.find(SessionId);
+ if (It == m_Sessions.end())
+ {
+ return {};
+ }
+
+ return It->second;
+}
+
+std::vector<Ref<SessionsService::Session>>
+SessionsService::GetSessions() const
+{
+ RwLock::SharedLockScope Lock(m_Lock);
+
+ std::vector<Ref<Session>> Result;
+ Result.reserve(m_Sessions.size());
+ for (const auto& [Id, SessionRef] : m_Sessions)
+ {
+ Result.push_back(SessionRef);
+ }
+ return Result;
+}
+
+bool
+SessionsService::RemoveSession(const Oid& SessionId)
+{
+ RwLock::ExclusiveLockScope Lock(m_Lock);
+
+ auto It = m_Sessions.find(SessionId);
+ if (It == m_Sessions.end())
+ {
+ return false;
+ }
+
+ ZEN_INFO("Session {} removed (AppName: {}, JobId: {})", SessionId, It.value()->Info().AppName, It.value()->Info().JobId);
+
+ m_Sessions.erase(It);
+ return true;
+}
+
+uint64_t
+SessionsService::GetSessionCount() const
+{
+ RwLock::SharedLockScope Lock(m_Lock);
+ return m_Sessions.size();
+}
+
+} // namespace zen
diff --git a/src/zenserver/sessions/sessions.h b/src/zenserver/sessions/sessions.h
new file mode 100644
index 000000000..db9704430
--- /dev/null
+++ b/src/zenserver/sessions/sessions.h
@@ -0,0 +1,83 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+#include <zencore/logbase.h>
+#include <zencore/thread.h>
+#include <zencore/uid.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace zen {
+
+class SessionLogStore;
+class SessionLog;
+
+/** Session tracker
+ *
+ * Acts as a log and session info concentrator when dealing with multiple
+ * servers and external processes acting as a group.
+ */
+
+class SessionsService
+{
+public:
+ struct SessionInfo
+ {
+ Oid Id;
+ std::string AppName;
+ Oid JobId;
+ CbObject Metadata;
+ DateTime CreatedAt;
+ DateTime UpdatedAt;
+ };
+
+ class Session : public TRefCounted<Session>
+ {
+ public:
+ Session(const SessionInfo& Info);
+ ~Session();
+
+ Session(Session&&) = delete;
+ Session& operator=(Session&&) = delete;
+
+ const SessionInfo& Info() const { return m_Info; }
+ void UpdateMetadata(CbObjectView Metadata)
+ {
+ // Should this be additive rather than replacing the whole thing? We'll see.
+ m_Info.Metadata = CbObject::Clone(Metadata);
+ m_Info.UpdatedAt = DateTime::Now();
+ }
+
+ private:
+ SessionInfo m_Info;
+ Ref<SessionLog> m_Log;
+ };
+
+ SessionsService();
+ ~SessionsService();
+
+ bool RegisterSession(const Oid& SessionId, std::string AppName, const Oid& JobId, CbObjectView Metadata);
+ bool UpdateSession(const Oid& SessionId, CbObjectView Metadata);
+ Ref<Session> GetSession(const Oid& SessionId) const;
+ std::vector<Ref<Session>> GetSessions() const;
+ bool RemoveSession(const Oid& SessionId);
+ uint64_t GetSessionCount() const;
+
+private:
+ LoggerRef& Log() { return m_Log; }
+
+ LoggerRef m_Log;
+ mutable RwLock m_Lock;
+ tsl::robin_map<Oid, Ref<Session>, Oid::Hasher> m_Sessions;
+ std::unique_ptr<SessionLogStore> m_SessionLogs;
+};
+
+} // namespace zen
diff --git a/src/zenserver/storage/admin/admin.cpp b/src/zenserver/storage/admin/admin.cpp
index 19155e02b..c9f999c69 100644
--- a/src/zenserver/storage/admin/admin.cpp
+++ b/src/zenserver/storage/admin/admin.cpp
@@ -716,7 +716,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
"logs",
[this](HttpRouterRequest& Req) {
CbObjectWriter Obj;
- auto LogLevel = logging::level::ToStringView(logging::GetLogLevel());
+ auto LogLevel = logging::ToStringView(logging::GetLogLevel());
Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size()));
Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath));
Obj.BeginObject("cache");
@@ -767,8 +767,8 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
}
if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false)
{
- logging::level::LogLevel NewLevel = logging::level::ParseLogLevelString(Param);
- std::string_view LogLevel = logging::level::ToStringView(NewLevel);
+ logging::LogLevel NewLevel = logging::ParseLogLevelString(Param);
+ std::string_view LogLevel = logging::ToStringView(NewLevel);
if (LogLevel != Param)
{
return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index f5ba30616..de9589078 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -71,7 +71,7 @@ HttpBuildStoreService::Initialize()
m_Router.RegisterRoute(
"{namespace}/{bucket}/{buildid}/blobs/{hash}",
[this](HttpRouterRequest& Req) { GetBlobRequest(Req); },
- HttpVerb::kGet);
+ HttpVerb::kGet | HttpVerb::kPost);
m_Router.RegisterRoute(
"{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata",
@@ -161,14 +161,57 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
HttpContentType::kText,
fmt::format("Invalid blob hash '{}'", Hash));
}
- zen::HttpRanges Ranges;
- bool HasRange = ServerRequest.TryGetRanges(Ranges);
- if (Ranges.size() > 1)
+
+ std::vector<std::pair<uint64_t, uint64_t>> OffsetAndLengthPairs;
+ if (ServerRequest.RequestVerb() == HttpVerb::kPost)
{
- // Only a single range is supported
- return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
- HttpContentType::kText,
- "Multiple ranges in blob request is not supported");
+ CbObject RangePayload = ServerRequest.ReadPayloadObject();
+ if (RangePayload)
+ {
+ CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView();
+ OffsetAndLengthPairs.reserve(RangesArray.Num());
+ for (CbFieldView FieldView : RangesArray)
+ {
+ CbObjectView RangeView = FieldView.AsObjectView();
+ uint64_t RangeOffset = RangeView["offset"sv].AsUInt64();
+ uint64_t RangeLength = RangeView["length"sv].AsUInt64();
+ OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength));
+ }
+ if (OffsetAndLengthPairs.size() > MaxRangeCountPerRequestSupported)
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Number of ranges ({}) for blob request exceeds maximum range count {}",
+ OffsetAndLengthPairs.size(),
+ MaxRangeCountPerRequestSupported));
+ }
+ }
+ if (OffsetAndLengthPairs.empty())
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ "Fetching blob without ranges must be done with the GET verb");
+ }
+ }
+ else
+ {
+ HttpRanges Ranges;
+ bool HasRange = ServerRequest.TryGetRanges(Ranges);
+ if (HasRange)
+ {
+ if (Ranges.size() > 1)
+ {
+ // Only a single http range is supported, we have limited support for http multirange responses
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Multiple ranges in blob request is only supported for {} accept type",
+ ToString(HttpContentType::kCbPackage)));
+ }
+ const HttpRange& FirstRange = Ranges.front();
+ OffsetAndLengthPairs.push_back(std::make_pair<uint64_t, uint64_t>(FirstRange.Start, FirstRange.End - FirstRange.Start + 1));
+ }
}
m_BuildStoreStats.BlobReadCount++;
@@ -179,24 +222,79 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
HttpContentType::kText,
fmt::format("Blob with hash '{}' could not be found", Hash));
}
- // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize());
m_BuildStoreStats.BlobHitCount++;
- if (HasRange)
+
+ if (OffsetAndLengthPairs.empty())
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
+ }
+
+ if (ServerRequest.AcceptContentType() == HttpContentType::kCbPackage)
{
- const HttpRange& Range = Ranges.front();
- const uint64_t BlobSize = Blob.GetSize();
- const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0;
- const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize);
- if (Range.Start + RangeSize > BlobSize)
+ const uint64_t BlobSize = Blob.GetSize();
+
+ CbPackage ResponsePackage;
+ std::vector<IoBuffer> RangeBuffers;
+ CbObjectWriter Writer;
+ Writer.BeginArray("ranges"sv);
+ for (const std::pair<uint64_t, uint64_t>& Range : OffsetAndLengthPairs)
{
- return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+ const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0;
+ const uint64_t RangeSize = Min(Range.second, MaxBlobSize);
+ Writer.BeginObject();
+ {
+ if (Range.first + RangeSize <= BlobSize)
+ {
+ RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize));
+ Writer.AddInteger("offset"sv, Range.first);
+ Writer.AddInteger("length"sv, RangeSize);
+ }
+ else
+ {
+ Writer.AddInteger("offset"sv, Range.first);
+ Writer.AddInteger("length"sv, 0);
+ }
+ }
+ Writer.EndObject();
}
- Blob = IoBuffer(Blob, Range.Start, RangeSize);
- return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
+ Writer.EndArray();
+
+ CompositeBuffer Ranges(RangeBuffers);
+ CbAttachment PayloadAttachment(std::move(Ranges), BlobHash);
+ Writer.AddAttachment("payload", PayloadAttachment);
+
+ CbObject HeaderObject = Writer.Save();
+
+ ResponsePackage.AddAttachment(PayloadAttachment);
+ ResponsePackage.SetObject(HeaderObject);
+
+ CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage);
+ uint64_t ResponseSize = RpcResponseBuffer.GetSize();
+ ZEN_UNUSED(ResponseSize);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
}
else
{
- return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
+ if (OffsetAndLengthPairs.size() != 1)
+ {
+ // Only a single http range is supported, we have limited support for http multirange responses
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Multiple ranges in blob request is only supported for {} accept type", ToString(HttpContentType::kCbPackage)));
+ }
+
+ const std::pair<uint64_t, uint64_t>& OffsetAndLength = OffsetAndLengthPairs.front();
+ const uint64_t BlobSize = Blob.GetSize();
+ const uint64_t MaxBlobSize = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0;
+ const uint64_t RangeSize = Min(OffsetAndLength.second, MaxBlobSize);
+ if (OffsetAndLength.first + RangeSize > BlobSize)
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+ }
+ Blob = IoBuffer(Blob, OffsetAndLength.first, RangeSize);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
}
}
@@ -507,8 +605,8 @@ HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req)
return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject);
}
-void
-HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
+CbObject
+HttpBuildStoreService::CollectStats()
{
ZEN_TRACE_CPU("HttpBuildStoreService::Stats");
@@ -562,7 +660,13 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
}
Cbo.EndObject();
- return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ return Cbo.Save();
+}
+
+void
+HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
+{
+ Request.WriteResponse(HttpResponseCode::OK, CollectStats());
}
void
@@ -571,6 +675,11 @@ HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request)
ZEN_TRACE_CPU("HttpBuildStoreService::Status");
CbObjectWriter Cbo;
Cbo << "ok" << true;
+ Cbo.BeginObject("capabilities");
+ {
+ Cbo << "maxrangecountperrequest" << MaxRangeCountPerRequestSupported;
+ }
+ Cbo.EndObject(); // capabilities
Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
}
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h
index e10986411..2a09b71cf 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.h
+++ b/src/zenserver/storage/buildstore/httpbuildstore.h
@@ -22,8 +22,9 @@ public:
virtual const char* BaseUri() const override;
virtual void HandleRequest(zen::HttpServerRequest& Request) override;
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
- virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct BuildStoreStats
@@ -45,6 +46,8 @@ private:
inline LoggerRef Log() { return m_Log; }
+ static constexpr uint32_t MaxRangeCountPerRequestSupported = 256u;
+
LoggerRef m_Log;
void PutBlobRequest(HttpRouterRequest& Req);
diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp
index 72f29d14e..06b8f6c27 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.cpp
+++ b/src/zenserver/storage/cache/httpstructuredcache.cpp
@@ -654,7 +654,7 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque
auto NewEnd = std::unique(AllAttachments.begin(), AllAttachments.end());
AllAttachments.erase(NewEnd, AllAttachments.end());
- uint64_t AttachmentsSize = 0;
+ std::atomic<uint64_t> AttachmentsSize = 0;
m_CidStore.IterateChunks(
AllAttachments,
@@ -746,7 +746,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
ResponseWriter << "Size" << ValuesSize;
ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size();
- uint64_t AttachmentsSize = 0;
+ std::atomic<uint64_t> AttachmentsSize = 0;
WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background);
@@ -1827,8 +1827,8 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::st
}
}
-void
-HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
+CbObject
+HttpStructuredCacheService::CollectStats()
{
ZEN_MEMSCOPE(GetCacheHttpTag());
@@ -1858,13 +1858,132 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
const CidStoreSize CidSize = m_CidStore.TotalSize();
const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
+ Cbo.BeginObject("cache");
+ {
+ Cbo << "badrequestcount" << BadRequestCount;
+ Cbo.BeginObject("rpc");
+ Cbo << "count" << RpcRequests;
+ Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests;
+ Cbo.BeginObject("records");
+ Cbo << "count" << RpcRecordRequests;
+ Cbo << "ops" << RpcRecordBatchRequests;
+ Cbo.EndObject();
+ Cbo.BeginObject("values");
+ Cbo << "count" << RpcValueRequests;
+ Cbo << "ops" << RpcValueBatchRequests;
+ Cbo.EndObject();
+ Cbo.BeginObject("chunks");
+ Cbo << "count" << RpcChunkRequests;
+ Cbo << "ops" << RpcChunkBatchRequests;
+ Cbo.EndObject();
+ Cbo.EndObject();
+
+ Cbo.BeginObject("size");
+ {
+ Cbo << "disk" << CacheSize.DiskSize;
+ Cbo << "memory" << CacheSize.MemorySize;
+ }
+ Cbo.EndObject();
+
+ Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
+ Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
+
+ if (m_UpstreamCache.IsActive())
+ {
+ Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+ Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+ Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+ Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
+ }
+
+ Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
+
+ {
+ ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats();
+ Cbo.BeginObject("store");
+ Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount
+ << "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount;
+ const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount;
+ Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0);
+ EmitSnapshot("read", StoreStatsData.GetOps, Cbo);
+ EmitSnapshot("write", StoreStatsData.PutOps, Cbo);
+ Cbo.EndObject();
+ }
+ }
+ Cbo.EndObject();
+
+ if (m_UpstreamCache.IsActive())
+ {
+ EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
+ Cbo.BeginObject("upstream");
+ {
+ m_UpstreamCache.GetStatus(Cbo);
+ }
+ Cbo.EndObject();
+ }
+
+ Cbo.BeginObject("cid");
+ {
+ Cbo.BeginObject("size");
+ {
+ Cbo << "tiny" << CidSize.TinySize;
+ Cbo << "small" << CidSize.SmallSize;
+ Cbo << "large" << CidSize.LargeSize;
+ Cbo << "total" << CidSize.TotalSize;
+ }
+ Cbo.EndObject();
+ }
+ Cbo.EndObject();
+
+ return Cbo.Save();
+}
+
+void
+HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
+{
+ ZEN_MEMSCOPE(GetCacheHttpTag());
+
bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true";
bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true";
- CidStoreStats CidStoreStats = {};
+ if (!ShowCidStoreStats && !ShowCacheStoreStats)
+ {
+ Request.WriteResponse(HttpResponseCode::OK, CollectStats());
+ return;
+ }
+
+ // Full stats with optional detailed store/cid breakdowns
+
+ CbObjectWriter Cbo;
+
+ EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+ const uint64_t HitCount = m_CacheStats.HitCount;
+ const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
+ const uint64_t MissCount = m_CacheStats.MissCount;
+ const uint64_t WriteCount = m_CacheStats.WriteCount;
+ const uint64_t BadRequestCount = m_CacheStats.BadRequestCount;
+ struct CidStoreStats StoreStats = m_CidStore.Stats();
+ const uint64_t ChunkHitCount = StoreStats.HitCount;
+ const uint64_t ChunkMissCount = StoreStats.MissCount;
+ const uint64_t ChunkWriteCount = StoreStats.WriteCount;
+ const uint64_t TotalCount = HitCount + MissCount;
+
+ const uint64_t RpcRequests = m_CacheStats.RpcRequests;
+ const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests;
+ const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests;
+ const uint64_t RpcValueRequests = m_CacheStats.RpcValueRequests;
+ const uint64_t RpcValueBatchRequests = m_CacheStats.RpcValueBatchRequests;
+ const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests;
+ const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests;
+
+ const CidStoreSize CidSize = m_CidStore.TotalSize();
+ const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
+
+ CidStoreStats DetailedCidStoreStats = {};
if (ShowCidStoreStats)
{
- CidStoreStats = m_CidStore.Stats();
+ DetailedCidStoreStats = m_CidStore.Stats();
}
ZenCacheStore::CacheStoreStats CacheStoreStats = {};
if (ShowCacheStoreStats)
@@ -2002,8 +2121,8 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
}
Cbo.EndObject();
}
- Cbo.EndObject();
}
+ Cbo.EndObject();
if (m_UpstreamCache.IsActive())
{
@@ -2029,10 +2148,10 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
if (ShowCidStoreStats)
{
Cbo.BeginObject("store");
- Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount;
- EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo);
- EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo);
- // EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo);
+ Cbo << "hits" << DetailedCidStoreStats.HitCount << "misses" << DetailedCidStoreStats.MissCount << "writes"
+ << DetailedCidStoreStats.WriteCount;
+ EmitSnapshot("read", DetailedCidStoreStats.FindChunkOps, Cbo);
+ EmitSnapshot("write", DetailedCidStoreStats.AddChunkOps, Cbo);
Cbo.EndObject();
}
}
diff --git a/src/zenserver/storage/cache/httpstructuredcache.h b/src/zenserver/storage/cache/httpstructuredcache.h
index 5a795c215..d462415d4 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.h
+++ b/src/zenserver/storage/cache/httpstructuredcache.h
@@ -102,11 +102,12 @@ private:
void HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace);
void HandleDetailsRequest(HttpServerRequest& Request);
- void HandleCacheRequest(HttpServerRequest& Request);
- void HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view Namespace);
- void HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket);
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
- virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+ void HandleCacheRequest(HttpServerRequest& Request);
+ void HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view Namespace);
+ void HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket);
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
bool AreDiskWritesAllowed() const;
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index fe32fa15b..836d84292 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -13,7 +13,12 @@
#include <zencore/scopeguard.h>
#include <zencore/stream.h>
#include <zencore/trace.h>
+#include <zenhttp/httpclientauth.h>
#include <zenhttp/packageformat.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildstorageutil.h>
+#include <zenremotestore/jupiter/jupiterhost.h>
+#include <zenremotestore/operationlogoutput.h>
#include <zenremotestore/projectstore/buildsremoteprojectstore.h>
#include <zenremotestore/projectstore/fileremoteprojectstore.h>
#include <zenremotestore/projectstore/jupiterremoteprojectstore.h>
@@ -244,6 +249,22 @@ namespace {
{
std::shared_ptr<RemoteProjectStore> Store;
std::string Description;
+ double LatencySec = -1.0;
+ uint64_t MaxRangeCountPerRequest = 1;
+
+ struct Cache
+ {
+ std::unique_ptr<HttpClient> Http;
+ std::unique_ptr<BuildStorageCache> Cache;
+ Oid BuildsId = Oid::Zero;
+ std::string Description;
+ double LatencySec = -1.0;
+ uint64_t MaxRangeCountPerRequest = 1;
+ BuildStorageCache::Statistics Stats;
+ bool Populate = false;
+ };
+
+ std::unique_ptr<Cache> OptionalCache;
};
CreateRemoteStoreResult CreateRemoteStore(LoggerRef InLog,
@@ -260,7 +281,7 @@ namespace {
using namespace std::literals;
- std::shared_ptr<RemoteProjectStore> RemoteStore;
+ CreateRemoteStoreResult Result;
if (CbObjectView File = Params["file"sv].AsObjectView(); File)
{
@@ -285,7 +306,9 @@ namespace {
std::string(OptionalBaseName),
ForceDisableBlocks,
ForceEnableTempBlocks};
- RemoteStore = CreateFileRemoteStore(Log(), Options);
+ Result.Store = CreateFileRemoteStore(Log(), Options);
+ Result.LatencySec = 0.5 / 1000.0; // 0.5 ms
+ Result.MaxRangeCountPerRequest = 1024u;
}
if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud)
@@ -363,21 +386,32 @@ namespace {
bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false);
bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false);
- JupiterRemoteStoreOptions Options = {
- RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
- Url,
- std::string(Namespace),
- std::string(Bucket),
- Key,
- BaseKey,
- std::string(OpenIdProvider),
- AccessToken,
- AuthManager,
- OidcExePath,
- ForceDisableBlocks,
- ForceDisableTempBlocks,
- AssumeHttp2};
- RemoteStore = CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+ if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(Url, AssumeHttp2, /*Verbose*/ false); TestResult.Success)
+ {
+ Result.LatencySec = TestResult.LatencySeconds;
+ Result.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest;
+
+ JupiterRemoteStoreOptions Options = {
+ RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
+ Url,
+ std::string(Namespace),
+ std::string(Bucket),
+ Key,
+ BaseKey,
+ std::string(OpenIdProvider),
+ AccessToken,
+ AuthManager,
+ OidcExePath,
+ ForceDisableBlocks,
+ ForceDisableTempBlocks,
+ AssumeHttp2};
+ Result.Store =
+ CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true);
+ }
+ else
+ {
+ return {nullptr, fmt::format("Unable to connect to jupiter host '{}'", Url)};
+ }
}
if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen)
@@ -393,12 +427,13 @@ namespace {
{
return {nullptr, "Missing oplog"};
}
+
ZenRemoteStoreOptions Options = {
RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
std::string(Url),
std::string(Project),
std::string(Oplog)};
- RemoteStore = CreateZenRemoteStore(Log(), Options, TempFilePath);
+ Result.Store = CreateZenRemoteStore(Log(), Options, TempFilePath);
}
if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds)
@@ -471,11 +506,76 @@ namespace {
MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView();
IoBuffer MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize());
+ auto EnsureHttps = [](const std::string& Host, std::string_view PreferredProtocol) {
+ if (!Host.empty() && Host.find("://"sv) == std::string::npos)
+ {
+ // Assume https URL
+ return fmt::format("{}://{}"sv, PreferredProtocol, Host);
+ }
+ return Host;
+ };
+
+ Host = EnsureHttps(Host, "https");
+ OverrideHost = EnsureHttps(OverrideHost, "https");
+ ZenHost = EnsureHttps(ZenHost, "http");
+
+ std::function<HttpClientAccessToken()> TokenProvider;
+ if (!OpenIdProvider.empty())
+ {
+ TokenProvider = httpclientauth::CreateFromOpenIdProvider(AuthManager, OpenIdProvider);
+ }
+ else if (!AccessToken.empty())
+ {
+ TokenProvider = httpclientauth::CreateFromStaticToken(AccessToken);
+ }
+ else if (!OidcExePath.empty())
+ {
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(OidcExePath,
+ Host.empty() ? OverrideHost : Host,
+ /*Quiet*/ false,
+ /*Unattended*/ false,
+ /*Hidden*/ true);
+ TokenProviderMaybe)
+ {
+ TokenProvider = TokenProviderMaybe.value();
+ }
+ }
+
+ if (!TokenProvider)
+ {
+ TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(AuthManager);
+ }
+
+ BuildStorageResolveResult ResolveResult;
+ {
+ HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
+ .AccessTokenProvider = TokenProvider,
+ .AssumeHttp2 = AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 2};
+
+ std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(Log()));
+
+ try
+ {
+ ResolveResult = ResolveBuildStorage(*Output,
+ ClientSettings,
+ Host,
+ OverrideHost,
+ ZenHost,
+ ZenCacheResolveMode::Discovery,
+ /*Verbose*/ false);
+ }
+ catch (const std::exception& Ex)
+ {
+ return {nullptr, fmt::format("Failed resolving storage host and cache. Reason: '{}'", Ex.what())};
+ }
+ }
+ Result.LatencySec = ResolveResult.Cloud.LatencySec;
+ Result.MaxRangeCountPerRequest = ResolveResult.Cloud.Caps.MaxRangeCountPerRequest;
+
BuildsRemoteStoreOptions Options = {
RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize},
- Host,
- OverrideHost,
- ZenHost,
std::string(Namespace),
std::string(Bucket),
BuildId,
@@ -485,25 +585,43 @@ namespace {
OidcExePath,
ForceDisableBlocks,
ForceDisableTempBlocks,
- AssumeHttp2,
- PopulateCache,
MetaData,
MaximumInMemoryDownloadSize};
- RemoteStore = CreateJupiterBuildsRemoteStore(Log(),
- Options,
- TempFilePath,
- /*Quiet*/ false,
- /*Unattended*/ false,
- /*Hidden*/ true,
- GetTinyWorkerPool(EWorkloadType::Background));
+ Result.Store = CreateJupiterBuildsRemoteStore(Log(), ResolveResult, std::move(TokenProvider), Options, TempFilePath);
+
+ if (!ResolveResult.Cache.Address.empty())
+ {
+ Result.OptionalCache = std::make_unique<CreateRemoteStoreResult::Cache>();
+
+ HttpClientSettings CacheClientSettings{.LogCategory = "httpcacheclient",
+ .ConnectTimeout = std::chrono::milliseconds{3000},
+ .Timeout = std::chrono::milliseconds{30000},
+ .AssumeHttp2 = ResolveResult.Cache.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 0,
+ .MaximumInMemoryDownloadSize = MaximumInMemoryDownloadSize};
+
+ Result.OptionalCache->Http = std::make_unique<HttpClient>(ResolveResult.Cache.Address, CacheClientSettings);
+ Result.OptionalCache->Cache = CreateZenBuildStorageCache(*Result.OptionalCache->Http,
+ Result.OptionalCache->Stats,
+ Namespace,
+ Bucket,
+ TempFilePath,
+ GetTinyWorkerPool(EWorkloadType::Background));
+ Result.OptionalCache->BuildsId = BuildId;
+ Result.OptionalCache->LatencySec = ResolveResult.Cache.LatencySec;
+ Result.OptionalCache->MaxRangeCountPerRequest = ResolveResult.Cache.Caps.MaxRangeCountPerRequest;
+ Result.OptionalCache->Populate = PopulateCache;
+ Result.OptionalCache->Description =
+ fmt::format("[zenserver] {} namespace {} bucket {}", ResolveResult.Cache.Address, Namespace, Bucket);
+ }
}
-
- if (!RemoteStore)
+ if (!Result.Store)
{
return {nullptr, "Unknown remote store type"};
}
- return {std::move(RemoteStore), ""};
+ return Result;
}
std::pair<HttpResponseCode, std::string> ConvertResult(const RemoteProjectStore::Result& Result)
@@ -714,8 +832,8 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
}
}
-void
-HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+CbObject
+HttpProjectService::CollectStats()
{
ZEN_TRACE_CPU("ProjectService::Stats");
@@ -781,7 +899,13 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
}
Cbo.EndObject();
- return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ return Cbo.Save();
+}
+
+void
+HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+ HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
}
void
@@ -2373,15 +2497,19 @@ HttpProjectService::HandleOplogSaveRequest(HttpRouterRequest& Req)
tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
auto HasAttachment = [this](const IoHash& RawHash) { return m_CidStore.ContainsChunk(RawHash); };
- auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) {
+ auto OnNeedBlock = [&AttachmentsLock, &Attachments](ThinChunkBlockDescription&& ThinBlockDescription,
+ std::vector<uint32_t>&& NeededChunkIndexes) {
RwLock::ExclusiveLockScope _(AttachmentsLock);
- if (BlockHash != IoHash::Zero)
+ if (ThinBlockDescription.BlockHash != IoHash::Zero)
{
- Attachments.insert(BlockHash);
+ Attachments.insert(ThinBlockDescription.BlockHash);
}
else
{
- Attachments.insert(ChunkHashes.begin(), ChunkHashes.end());
+ for (uint32_t ChunkIndex : NeededChunkIndexes)
+ {
+ Attachments.insert(ThinBlockDescription.ChunkRawHashes[ChunkIndex]);
+ }
}
};
auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) {
@@ -2687,36 +2815,39 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
bool CleanOplog = Params["clean"].AsBool(false);
bool BoostWorkerCount = Params["boostworkercount"].AsBool(false);
bool BoostWorkerMemory = Params["boostworkermemory"sv].AsBool(false);
-
- CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Log(),
- Params,
- m_AuthMgr,
- MaxBlockSize,
- MaxChunkEmbedSize,
- GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
- Oplog->TempPath());
-
- if (RemoteStoreResult.Store == nullptr)
+ EPartialBlockRequestMode PartialBlockRequestMode =
+ PartialBlockRequestModeFromString(Params["partialblockrequestmode"sv].AsString("true"));
+
+ std::shared_ptr<CreateRemoteStoreResult> RemoteStoreResult =
+ std::make_shared<CreateRemoteStoreResult>(CreateRemoteStore(Log(),
+ Params,
+ m_AuthMgr,
+ MaxBlockSize,
+ MaxChunkEmbedSize,
+ GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory),
+ Oplog->TempPath()));
+
+ if (RemoteStoreResult->Store == nullptr)
{
- return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description);
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult->Description);
}
- std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store);
- RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
JobId JobId = m_JobQueue.QueueJob(
fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()),
[this,
- ChunkStore = &m_CidStore,
- ActualRemoteStore = std::move(RemoteStore),
+ RemoteStoreResult = std::move(RemoteStoreResult),
Oplog,
Force,
IgnoreMissingAttachments,
CleanOplog,
+ PartialBlockRequestMode,
BoostWorkerCount](JobContext& Context) {
- Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}",
- Oplog->GetOuterProjectIdentifier(),
- Oplog->OplogId(),
- ActualRemoteStore->GetInfo().Description));
+ Context.ReportMessage(
+ fmt::format("Loading oplog '{}/{}'\n Host: {}\n Cache: {}",
+ Oplog->GetOuterProjectIdentifier(),
+ Oplog->OplogId(),
+ RemoteStoreResult->Store->GetInfo().Description,
+ RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Description : "<none>"));
Ref<TransferThreadWorkers> Workers = GetThreadWorkers(BoostWorkerCount, /*SingleThreaded*/ false);
@@ -2724,16 +2855,26 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
WorkerThreadPool& NetworkWorkerPool = Workers->GetNetworkPool();
Context.ReportMessage(fmt::format("{}", Workers->GetWorkersInfo()));
-
- RemoteProjectStore::Result Result = LoadOplog(m_CidStore,
- *ActualRemoteStore,
- *Oplog,
- NetworkWorkerPool,
- WorkerPool,
- Force,
- IgnoreMissingAttachments,
- CleanOplog,
- &Context);
+ RemoteProjectStore::Result Result = LoadOplog(LoadOplogContext{
+ .ChunkStore = m_CidStore,
+ .RemoteStore = *RemoteStoreResult->Store,
+ .OptionalCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Cache.get() : nullptr,
+ .CacheBuildId = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->BuildsId : Oid::Zero,
+ .OptionalCacheStats = RemoteStoreResult->OptionalCache ? &RemoteStoreResult->OptionalCache->Stats : nullptr,
+ .Oplog = *Oplog,
+ .NetworkWorkerPool = NetworkWorkerPool,
+ .WorkerPool = WorkerPool,
+ .ForceDownload = Force,
+ .IgnoreMissingAttachments = IgnoreMissingAttachments,
+ .CleanOplog = CleanOplog,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .PopulateCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Populate : false,
+ .StoreLatencySec = RemoteStoreResult->LatencySec,
+ .StoreMaxRangeCountPerRequest = RemoteStoreResult->MaxRangeCountPerRequest,
+ .CacheLatencySec = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->LatencySec : -1.0,
+ .CacheMaxRangeCountPerRequest =
+ RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->MaxRangeCountPerRequest : 0,
+ .OptionalJobContext = &Context});
auto Response = ConvertResult(Result);
ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second);
if (!IsHttpSuccessCode(Response.first))
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h
index 1d71329b1..a1f649ed6 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.h
+++ b/src/zenserver/storage/projectstore/httpprojectstore.h
@@ -51,8 +51,9 @@ public:
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& Request) override;
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
- virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct ProjectStats
diff --git a/src/zenserver/storage/storageconfig.cpp b/src/zenserver/storage/storageconfig.cpp
index 99d0f89d7..ad1fb88ea 100644
--- a/src/zenserver/storage/storageconfig.cpp
+++ b/src/zenserver/storage/storageconfig.cpp
@@ -804,6 +804,7 @@ ZenStorageServerCmdLineOptions::AddCacheOptions(cxxopts::Options& options, ZenSt
cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"),
"");
+ options.add_option("compute", "", "lie-cpus", "Lie to upstream about CPU capabilities", cxxopts::value<int>(ServerOptions.LieCpu), "");
options.add_option("cache",
"",
"cache-bucket-maxblocksize",
diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h
index bc2dc78c9..d935ed8b3 100644
--- a/src/zenserver/storage/storageconfig.h
+++ b/src/zenserver/storage/storageconfig.h
@@ -1,4 +1,5 @@
// Copyright Epic Games, Inc. All Rights Reserved.
+#pragma once
#include "config/config.h"
@@ -156,6 +157,7 @@ struct ZenStorageServerConfig : public ZenServerConfig
ZenWorkspacesConfig WorksSpacesConfig;
std::filesystem::path PluginsConfigFile; // Path to plugins config file
bool ObjectStoreEnabled = false;
+ bool ComputeEnabled = true;
std::string ScrubOptions;
bool RestrictContentTypes = false;
};
diff --git a/src/zenserver/storage/workspaces/httpworkspaces.cpp b/src/zenserver/storage/workspaces/httpworkspaces.cpp
index dc4cc7e69..785dd62f0 100644
--- a/src/zenserver/storage/workspaces/httpworkspaces.cpp
+++ b/src/zenserver/storage/workspaces/httpworkspaces.cpp
@@ -110,8 +110,8 @@ HttpWorkspacesService::HandleRequest(HttpServerRequest& Request)
}
}
-void
-HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
+CbObject
+HttpWorkspacesService::CollectStats()
{
ZEN_TRACE_CPU("WorkspacesService::Stats");
CbObjectWriter Cbo;
@@ -150,7 +150,13 @@ HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
}
Cbo.EndObject();
- return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+ return Cbo.Save();
+}
+
+void
+HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
+{
+ HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats());
}
void
diff --git a/src/zenserver/storage/workspaces/httpworkspaces.h b/src/zenserver/storage/workspaces/httpworkspaces.h
index 888a34b4d..7c5ddeff1 100644
--- a/src/zenserver/storage/workspaces/httpworkspaces.h
+++ b/src/zenserver/storage/workspaces/httpworkspaces.h
@@ -29,8 +29,9 @@ public:
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& Request) override;
- virtual void HandleStatsRequest(HttpServerRequest& Request) override;
- virtual void HandleStatusRequest(HttpServerRequest& Request) override;
+ virtual CbObject CollectStats() override;
+ virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct WorkspacesStats
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index ea05bd155..f43bb9987 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -33,6 +33,7 @@
#include <zenutil/service.h>
#include <zenutil/workerpools.h>
#include <zenutil/zenserverprocess.h>
+#include "../sessions/sessions.h"
#if ZEN_PLATFORM_WINDOWS
# include <zencore/windows.h>
@@ -133,7 +134,6 @@ void
ZenStorageServer::RegisterServices()
{
m_Http->RegisterService(*m_AuthService);
- m_Http->RegisterService(m_StatsService);
m_Http->RegisterService(m_TestService); // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics
#if ZEN_WITH_TESTS
@@ -160,6 +160,11 @@ ZenStorageServer::RegisterServices()
m_Http->RegisterService(*m_HttpWorkspacesService);
}
+ if (m_HttpSessionsService)
+ {
+ m_Http->RegisterService(*m_HttpSessionsService);
+ }
+
m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
if (m_FrontendService)
@@ -182,6 +187,18 @@ ZenStorageServer::RegisterServices()
#endif // ZEN_WITH_VFS
m_Http->RegisterService(*m_AdminService);
+
+ if (m_ApiService)
+ {
+ m_Http->RegisterService(*m_ApiService);
+ }
+
+#if ZEN_WITH_COMPUTE_SERVICES
+ if (m_HttpComputeService)
+ {
+ m_Http->RegisterService(*m_HttpComputeService);
+ }
+#endif
}
void
@@ -227,6 +244,11 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
*m_Workspaces));
}
+ {
+ m_SessionsService = std::make_unique<SessionsService>();
+ m_HttpSessionsService = std::make_unique<HttpSessionsService>(m_StatusService, m_StatsService, *m_SessionsService);
+ }
+
if (ServerOptions.BuildStoreConfig.Enabled)
{
CidStoreConfiguration BuildCidConfig;
@@ -273,6 +295,16 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
m_BuildStoreService = std::make_unique<HttpBuildStoreService>(m_StatusService, m_StatsService, *m_BuildStore);
}
+#if ZEN_WITH_COMPUTE_SERVICES
+ if (ServerOptions.ComputeEnabled)
+ {
+ ZEN_OTEL_SPAN("InitializeComputeService");
+
+ m_HttpComputeService =
+ std::make_unique<compute::HttpComputeService>(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions");
+ }
+#endif
+
#if ZEN_WITH_VFS
m_VfsServiceImpl = std::make_unique<VfsServiceImpl>();
m_VfsServiceImpl->AddService(Ref<ProjectStore>(m_ProjectStore));
@@ -305,13 +337,15 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
.AttachmentPassCount = ServerOptions.GcConfig.AttachmentPassCount};
m_GcScheduler.Initialize(GcConfig);
+ m_ApiService = std::make_unique<HttpApiService>(*m_Http);
+
// Create and register admin interface last to make sure all is properly initialized
m_AdminService = std::make_unique<HttpAdminService>(
m_GcScheduler,
*m_JobQueue,
m_CacheStore.Get(),
[this]() { Flush(); },
- HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile,
+ HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.LoggingConfig.AbsLogFile,
.HttpLogPath = ServerOptions.DataDir / "logs" / "http.log",
.CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"},
ServerOptions);
@@ -689,6 +723,15 @@ ZenStorageServer::Run()
ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId());
+ if (m_FrontendService)
+ {
+ ZEN_INFO("frontend link: {}", m_Http->GetServiceUri(m_FrontendService.get()));
+ }
+ else
+ {
+ ZEN_INFO("frontend service disabled");
+ }
+
#if ZEN_PLATFORM_WINDOWS
if (zen::windows::IsRunningOnWine())
{
@@ -796,6 +839,8 @@ ZenStorageServer::Cleanup()
m_IoRunner.join();
}
+ ShutdownServices();
+
if (m_Http)
{
m_Http->Close();
@@ -811,6 +856,10 @@ ZenStorageServer::Cleanup()
Flush();
+#if ZEN_WITH_COMPUTE_SERVICES
+ m_HttpComputeService.reset();
+#endif
+
m_AdminService.reset();
m_VfsService.reset();
m_VfsServiceImpl.reset();
@@ -826,6 +875,8 @@ ZenStorageServer::Cleanup()
m_UpstreamCache.reset();
m_CacheStore = {};
+ m_HttpSessionsService.reset();
+ m_SessionsService.reset();
m_HttpWorkspacesService.reset();
m_Workspaces.reset();
m_HttpProjectService.reset();
diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h
index 5ccb587d6..d625f869c 100644
--- a/src/zenserver/storage/zenstorageserver.h
+++ b/src/zenserver/storage/zenstorageserver.h
@@ -6,11 +6,13 @@
#include <zenhttp/auth/authmgr.h>
#include <zenhttp/auth/authservice.h>
+#include <zenhttp/httpapiservice.h>
#include <zenhttp/httptest.h>
#include <zenstore/cache/structuredcachestore.h>
#include <zenstore/gc.h>
#include <zenstore/projectstore.h>
+#include "../sessions/httpsessions.h"
#include "admin/admin.h"
#include "buildstore/httpbuildstore.h"
#include "cache/httpstructuredcache.h"
@@ -23,6 +25,10 @@
#include "vfs/vfsservice.h"
#include "workspaces/httpworkspaces.h"
+#if ZEN_WITH_COMPUTE_SERVICES
+# include <zencompute/httpcomputeservice.h>
+#endif
+
namespace zen {
class ZenStorageServer : public ZenServerBase
@@ -34,11 +40,6 @@ public:
ZenStorageServer();
~ZenStorageServer();
- void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; }
- void SetTestMode(bool State) { m_TestMode = State; }
- void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; }
- void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; }
-
int Initialize(const ZenStorageServerConfig& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry);
void Run();
void Cleanup();
@@ -48,14 +49,9 @@ private:
void InitializeStructuredCache(const ZenStorageServerConfig& ServerOptions);
void Flush();
- bool m_IsDedicatedMode = false;
- bool m_TestMode = false;
- bool m_DebugOptionForcedCrash = false;
- std::string m_StartupScrubOptions;
- CbObject m_RootManifest;
- std::filesystem::path m_DataRoot;
- std::filesystem::path m_ContentRoot;
- asio::steady_timer m_StateMarkerTimer{m_IoContext};
+ std::string m_StartupScrubOptions;
+ CbObject m_RootManifest;
+ asio::steady_timer m_StateMarkerTimer{m_IoContext};
void EnqueueStateMarkerTimer();
void CheckStateMarker();
@@ -67,7 +63,6 @@ private:
void InitializeServices(const ZenStorageServerConfig& ServerOptions);
void RegisterServices();
- HttpStatsService m_StatsService;
std::unique_ptr<JobQueue> m_JobQueue;
GcManager m_GcManager;
GcScheduler m_GcScheduler{m_GcManager};
@@ -87,6 +82,8 @@ private:
std::unique_ptr<HttpProjectService> m_HttpProjectService;
std::unique_ptr<Workspaces> m_Workspaces;
std::unique_ptr<HttpWorkspacesService> m_HttpWorkspacesService;
+ std::unique_ptr<SessionsService> m_SessionsService;
+ std::unique_ptr<HttpSessionsService> m_HttpSessionsService;
std::unique_ptr<UpstreamCache> m_UpstreamCache;
std::unique_ptr<HttpUpstreamService> m_UpstreamService;
std::unique_ptr<HttpStructuredCacheService> m_StructuredCacheService;
@@ -95,6 +92,11 @@ private:
std::unique_ptr<HttpBuildStoreService> m_BuildStoreService;
std::unique_ptr<VfsService> m_VfsService;
std::unique_ptr<HttpAdminService> m_AdminService;
+ std::unique_ptr<HttpApiService> m_ApiService;
+
+#if ZEN_WITH_COMPUTE_SERVICES
+ std::unique_ptr<compute::HttpComputeService> m_HttpComputeService;
+#endif
};
struct ZenStorageServerConfigurator;
diff --git a/src/zenserver/trace/tracerecorder.cpp b/src/zenserver/trace/tracerecorder.cpp
new file mode 100644
index 000000000..5dec20e18
--- /dev/null
+++ b/src/zenserver/trace/tracerecorder.cpp
@@ -0,0 +1,565 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "tracerecorder.h"
+
+#include <zencore/basicfile.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/uid.h>
+
+#include <asio.hpp>
+
+#include <atomic>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+namespace zen {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TraceSession : public std::enable_shared_from_this<TraceSession>
+{
+ TraceSession(asio::ip::tcp::socket&& Socket, const std::filesystem::path& OutputDir)
+ : m_Socket(std::move(Socket))
+ , m_OutputDir(OutputDir)
+ , m_SessionId(Oid::NewOid())
+ {
+ try
+ {
+ m_RemoteAddress = m_Socket.remote_endpoint().address().to_string();
+ }
+ catch (...)
+ {
+ m_RemoteAddress = "unknown";
+ }
+
+ ZEN_INFO("Trace session {} started from {}", m_SessionId, m_RemoteAddress);
+ }
+
+ ~TraceSession()
+ {
+ if (m_TraceFile.IsOpen())
+ {
+ m_TraceFile.Close();
+ }
+
+ ZEN_INFO("Trace session {} ended, {} bytes recorded to '{}'", m_SessionId, m_TotalBytesRecorded, m_TraceFilePath);
+ }
+
+ void Start() { ReadPreambleHeader(); }
+
+ bool IsActive() const { return m_Socket.is_open(); }
+
+ TraceSessionInfo GetInfo() const
+ {
+ TraceSessionInfo Info;
+ Info.SessionGuid = m_SessionGuid;
+ Info.TraceGuid = m_TraceGuid;
+ Info.ControlPort = m_ControlPort;
+ Info.TransportVersion = m_TransportVersion;
+ Info.ProtocolVersion = m_ProtocolVersion;
+ Info.RemoteAddress = m_RemoteAddress;
+ Info.BytesRecorded = m_TotalBytesRecorded;
+ Info.TraceFilePath = m_TraceFilePath;
+ return Info;
+ }
+
+private:
+ // Preamble format:
+ // [magic: 4 bytes][metadata_size: 2 bytes][metadata fields: variable][version: 2 bytes]
+ //
+ // Magic bytes: [0]=version_char ('2'-'9'), [1]='C', [2]='R', [3]='T'
+ //
+ // Metadata fields (repeated):
+ // [size: 1 byte][id: 1 byte][data: <size> bytes]
+ // Field 0: ControlPort (uint16)
+ // Field 1: SessionGuid (16 bytes)
+ // Field 2: TraceGuid (16 bytes)
+ //
+ // Version: [transport: 1 byte][protocol: 1 byte]
+
+ static constexpr size_t kMagicSize = 4;
+ static constexpr size_t kMetadataSizeFieldSize = 2;
+ static constexpr size_t kPreambleHeaderSize = kMagicSize + kMetadataSizeFieldSize;
+ static constexpr size_t kVersionSize = 2;
+ static constexpr size_t kPreambleBufferSize = 256;
+ static constexpr size_t kReadBufferSize = 64 * 1024;
+
+ void ReadPreambleHeader()
+ {
+ auto Self = shared_from_this();
+
+ // Read the first 6 bytes: 4 magic + 2 metadata size
+ asio::async_read(m_Socket,
+ asio::buffer(m_PreambleBuffer, kPreambleHeaderSize),
+ [this, Self](const asio::error_code& Ec, std::size_t /*BytesRead*/) {
+ if (Ec)
+ {
+ HandleReadError("preamble header", Ec);
+ return;
+ }
+
+ if (!ValidateMagic())
+ {
+ ZEN_WARN("Trace session {}: invalid trace magic header", m_SessionId);
+ CloseSocket();
+ return;
+ }
+
+ ReadPreambleMetadata();
+ });
+ }
+
+ bool ValidateMagic()
+ {
+ const uint8_t* Cursor = m_PreambleBuffer;
+
+ // Validate magic: bytes are version, 'C', 'R', 'T'
+ if (Cursor[3] != 'T' || Cursor[2] != 'R' || Cursor[1] != 'C')
+ {
+ return false;
+ }
+
+ if (Cursor[0] < '2' || Cursor[0] > '9')
+ {
+ return false;
+ }
+
+ // Extract the metadata fields size (does not include the trailing version bytes)
+ std::memcpy(&m_MetadataFieldsSize, Cursor + kMagicSize, sizeof(m_MetadataFieldsSize));
+
+ if (m_MetadataFieldsSize + kVersionSize > kPreambleBufferSize - kPreambleHeaderSize)
+ {
+ return false;
+ }
+
+ return true;
+ }
+
+ void ReadPreambleMetadata()
+ {
+ auto Self = shared_from_this();
+ size_t ReadSize = m_MetadataFieldsSize + kVersionSize;
+
+ // Read metadata fields + 2 version bytes
+ asio::async_read(m_Socket,
+ asio::buffer(m_PreambleBuffer + kPreambleHeaderSize, ReadSize),
+ [this, Self](const asio::error_code& Ec, std::size_t /*BytesRead*/) {
+ if (Ec)
+ {
+ HandleReadError("preamble metadata", Ec);
+ return;
+ }
+
+ if (!ParseMetadata())
+ {
+ ZEN_WARN("Trace session {}: malformed trace metadata", m_SessionId);
+ CloseSocket();
+ return;
+ }
+
+ if (!CreateTraceFile())
+ {
+ CloseSocket();
+ return;
+ }
+
+ // Write the full preamble to the trace file so it remains a valid .utrace
+ size_t PreambleSize = kPreambleHeaderSize + m_MetadataFieldsSize + kVersionSize;
+ std::error_code WriteEc;
+ m_TraceFile.Write(m_PreambleBuffer, PreambleSize, 0, WriteEc);
+
+ if (WriteEc)
+ {
+ ZEN_ERROR("Trace session {}: failed to write preamble: {}", m_SessionId, WriteEc.message());
+ CloseSocket();
+ return;
+ }
+
+ m_TotalBytesRecorded = PreambleSize;
+
+ ZEN_INFO("Trace session {}: metadata - TransportV{} ProtocolV{} ControlPort:{} SessionGuid:{} TraceGuid:{}",
+ m_SessionId,
+ m_TransportVersion,
+ m_ProtocolVersion,
+ m_ControlPort,
+ m_SessionGuid,
+ m_TraceGuid);
+
+ // Begin streaming trace data to disk
+ ReadMore();
+ });
+ }
+
+ bool ParseMetadata()
+ {
+ const uint8_t* Cursor = m_PreambleBuffer + kPreambleHeaderSize;
+ int32_t Remaining = static_cast<int32_t>(m_MetadataFieldsSize);
+
+ while (Remaining >= 2)
+ {
+ uint8_t FieldSize = Cursor[0];
+ uint8_t FieldId = Cursor[1];
+ Cursor += 2;
+ Remaining -= 2;
+
+ if (Remaining < FieldSize)
+ {
+ return false;
+ }
+
+ switch (FieldId)
+ {
+ case 0: // ControlPort
+ if (FieldSize >= sizeof(uint16_t))
+ {
+ std::memcpy(&m_ControlPort, Cursor, sizeof(uint16_t));
+ }
+ break;
+ case 1: // SessionGuid
+ if (FieldSize >= sizeof(Guid))
+ {
+ std::memcpy(&m_SessionGuid, Cursor, sizeof(Guid));
+ }
+ break;
+ case 2: // TraceGuid
+ if (FieldSize >= sizeof(Guid))
+ {
+ std::memcpy(&m_TraceGuid, Cursor, sizeof(Guid));
+ }
+ break;
+ }
+
+ Cursor += FieldSize;
+ Remaining -= FieldSize;
+ }
+
+ // Metadata should be fully consumed
+ if (Remaining != 0)
+ {
+ return false;
+ }
+
+ // Version bytes follow immediately after the metadata fields
+ const uint8_t* VersionPtr = m_PreambleBuffer + kPreambleHeaderSize + m_MetadataFieldsSize;
+ m_TransportVersion = VersionPtr[0];
+ m_ProtocolVersion = VersionPtr[1];
+
+ return true;
+ }
+
+ bool CreateTraceFile()
+ {
+ m_TraceFilePath = m_OutputDir / fmt::format("{}.utrace", m_SessionId);
+
+ try
+ {
+ m_TraceFile.Open(m_TraceFilePath, BasicFile::Mode::kTruncate);
+ ZEN_INFO("Trace session {} writing to '{}'", m_SessionId, m_TraceFilePath);
+ return true;
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("Trace session {}: failed to create trace file '{}': {}", m_SessionId, m_TraceFilePath, Ex.what());
+ return false;
+ }
+ }
+
+ void ReadMore()
+ {
+ auto Self = shared_from_this();
+
+ m_Socket.async_read_some(asio::buffer(m_ReadBuffer, kReadBufferSize),
+ [this, Self](const asio::error_code& Ec, std::size_t BytesRead) {
+ if (!Ec)
+ {
+ if (BytesRead > 0 && m_TraceFile.IsOpen())
+ {
+ std::error_code WriteEc;
+ const uint64_t FileOffset = m_TotalBytesRecorded;
+ m_TraceFile.Write(m_ReadBuffer, BytesRead, FileOffset, WriteEc);
+
+ if (WriteEc)
+ {
+ ZEN_ERROR("Trace session {}: write error: {}", m_SessionId, WriteEc.message());
+ CloseSocket();
+ return;
+ }
+
+ m_TotalBytesRecorded += BytesRead;
+ }
+
+ ReadMore();
+ }
+ else if (Ec == asio::error::eof)
+ {
+ ZEN_DEBUG("Trace session {} connection closed by peer", m_SessionId);
+ CloseSocket();
+ }
+ else if (Ec == asio::error::operation_aborted)
+ {
+ ZEN_DEBUG("Trace session {} operation aborted", m_SessionId);
+ }
+ else
+ {
+ ZEN_WARN("Trace session {} read error: {}", m_SessionId, Ec.message());
+ CloseSocket();
+ }
+ });
+ }
+
+ void HandleReadError(const char* Phase, const asio::error_code& Ec)
+ {
+ if (Ec == asio::error::eof)
+ {
+ ZEN_DEBUG("Trace session {}: connection closed during {}", m_SessionId, Phase);
+ }
+ else if (Ec == asio::error::operation_aborted)
+ {
+ ZEN_DEBUG("Trace session {}: operation aborted during {}", m_SessionId, Phase);
+ }
+ else
+ {
+ ZEN_WARN("Trace session {}: error during {}: {}", m_SessionId, Phase, Ec.message());
+ }
+
+ CloseSocket();
+ }
+
+ void CloseSocket()
+ {
+ std::error_code Ec;
+ m_Socket.close(Ec);
+
+ if (m_TraceFile.IsOpen())
+ {
+ m_TraceFile.Close();
+ }
+ }
+
+ asio::ip::tcp::socket m_Socket;
+ std::filesystem::path m_OutputDir;
+ std::filesystem::path m_TraceFilePath;
+ BasicFile m_TraceFile;
+ Oid m_SessionId;
+ std::string m_RemoteAddress;
+
+ // Preamble parsing
+ uint8_t m_PreambleBuffer[kPreambleBufferSize] = {};
+ uint16_t m_MetadataFieldsSize = 0;
+
+ // Extracted metadata
+ Guid m_SessionGuid{};
+ Guid m_TraceGuid{};
+ uint16_t m_ControlPort = 0;
+ uint8_t m_TransportVersion = 0;
+ uint8_t m_ProtocolVersion = 0;
+
+ // Streaming
+ uint8_t m_ReadBuffer[kReadBufferSize];
+ uint64_t m_TotalBytesRecorded = 0;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct TraceRecorder::Impl
+{
+ Impl() : m_IoContext(), m_Acceptor(m_IoContext) {}
+
+ ~Impl() { Shutdown(); }
+
+ void Initialize(uint16_t InPort, const std::filesystem::path& OutputDir)
+ {
+ std::lock_guard<std::mutex> Lock(m_Mutex);
+
+ if (m_IsRunning)
+ {
+ ZEN_WARN("TraceRecorder already initialized");
+ return;
+ }
+
+ m_OutputDir = OutputDir;
+
+ try
+ {
+ // Create output directory if it doesn't exist
+ CreateDirectories(m_OutputDir);
+
+ // Configure acceptor
+ m_Acceptor.open(asio::ip::tcp::v4());
+ m_Acceptor.set_option(asio::socket_base::reuse_address(true));
+ m_Acceptor.bind(asio::ip::tcp::endpoint(asio::ip::tcp::v4(), InPort));
+ m_Acceptor.listen();
+
+ m_Port = m_Acceptor.local_endpoint().port();
+
+ ZEN_INFO("TraceRecorder listening on port {}, output directory: '{}'", m_Port, m_OutputDir);
+
+ m_IsRunning = true;
+
+ // Start accepting connections
+ StartAccept();
+
+ // Start IO thread
+ m_IoThread = std::thread([this]() {
+ try
+ {
+ m_IoContext.run();
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("TraceRecorder IO thread exception: {}", Ex.what());
+ }
+ });
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_ERROR("Failed to initialize TraceRecorder: {}", Ex.what());
+ m_IsRunning = false;
+ throw;
+ }
+ }
+
+ void Shutdown()
+ {
+ std::lock_guard<std::mutex> Lock(m_Mutex);
+
+ if (!m_IsRunning)
+ {
+ return;
+ }
+
+ ZEN_INFO("TraceRecorder shutting down");
+
+ m_IsRunning = false;
+
+ std::error_code Ec;
+ m_Acceptor.close(Ec);
+
+ m_IoContext.stop();
+
+ if (m_IoThread.joinable())
+ {
+ m_IoThread.join();
+ }
+
+ {
+ std::lock_guard<std::mutex> SessionLock(m_SessionsMutex);
+ m_Sessions.clear();
+ }
+
+ ZEN_INFO("TraceRecorder shutdown complete");
+ }
+
+ bool IsRunning() const { return m_IsRunning; }
+
+ uint16_t GetPort() const { return m_Port; }
+
+ std::vector<TraceSessionInfo> GetActiveSessions() const
+ {
+ std::lock_guard<std::mutex> Lock(m_SessionsMutex);
+
+ std::vector<TraceSessionInfo> Result;
+ for (const auto& WeakSession : m_Sessions)
+ {
+ if (auto Session = WeakSession.lock())
+ {
+ if (Session->IsActive())
+ {
+ Result.push_back(Session->GetInfo());
+ }
+ }
+ }
+ return Result;
+ }
+
+private:
+ void StartAccept()
+ {
+ auto Socket = std::make_shared<asio::ip::tcp::socket>(m_IoContext);
+
+ m_Acceptor.async_accept(*Socket, [this, Socket](const asio::error_code& Ec) {
+ if (!Ec)
+ {
+ auto Session = std::make_shared<TraceSession>(std::move(*Socket), m_OutputDir);
+
+ {
+ std::lock_guard<std::mutex> Lock(m_SessionsMutex);
+
+ // Prune expired sessions while adding the new one
+ std::erase_if(m_Sessions, [](const std::weak_ptr<TraceSession>& Wp) { return Wp.expired(); });
+ m_Sessions.push_back(Session);
+ }
+
+ Session->Start();
+ }
+ else if (Ec != asio::error::operation_aborted)
+ {
+ ZEN_WARN("Accept error: {}", Ec.message());
+ }
+
+ // Continue accepting if still running
+ if (m_IsRunning)
+ {
+ StartAccept();
+ }
+ });
+ }
+
+ asio::io_context m_IoContext;
+ asio::ip::tcp::acceptor m_Acceptor;
+ std::thread m_IoThread;
+ std::filesystem::path m_OutputDir;
+ std::mutex m_Mutex;
+ std::atomic<bool> m_IsRunning{false};
+ uint16_t m_Port = 0;
+
+ mutable std::mutex m_SessionsMutex;
+ std::vector<std::weak_ptr<TraceSession>> m_Sessions;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+TraceRecorder::TraceRecorder() : m_Impl(std::make_unique<Impl>())
+{
+}
+
+TraceRecorder::~TraceRecorder()
+{
+ Shutdown();
+}
+
+void
+TraceRecorder::Initialize(uint16_t InPort, const std::filesystem::path& OutputDir)
+{
+ m_Impl->Initialize(InPort, OutputDir);
+}
+
+void
+TraceRecorder::Shutdown()
+{
+ m_Impl->Shutdown();
+}
+
+bool
+TraceRecorder::IsRunning() const
+{
+ return m_Impl->IsRunning();
+}
+
+uint16_t
+TraceRecorder::GetPort() const
+{
+ return m_Impl->GetPort();
+}
+
+std::vector<TraceSessionInfo>
+TraceRecorder::GetActiveSessions() const
+{
+ return m_Impl->GetActiveSessions();
+}
+
+} // namespace zen
diff --git a/src/zenserver/trace/tracerecorder.h b/src/zenserver/trace/tracerecorder.h
new file mode 100644
index 000000000..48857aec8
--- /dev/null
+++ b/src/zenserver/trace/tracerecorder.h
@@ -0,0 +1,46 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/guid.h>
+#include <zencore/zencore.h>
+
+#include <filesystem>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace zen {
+
+struct TraceSessionInfo
+{
+ Guid SessionGuid{};
+ Guid TraceGuid{};
+ uint16_t ControlPort = 0;
+ uint8_t TransportVersion = 0;
+ uint8_t ProtocolVersion = 0;
+ std::string RemoteAddress;
+ uint64_t BytesRecorded = 0;
+ std::filesystem::path TraceFilePath;
+};
+
+class TraceRecorder
+{
+public:
+ TraceRecorder();
+ ~TraceRecorder();
+
+ void Initialize(uint16_t InPort, const std::filesystem::path& OutputDir);
+ void Shutdown();
+
+ bool IsRunning() const;
+ uint16_t GetPort() const;
+
+ std::vector<TraceSessionInfo> GetActiveSessions() const;
+
+private:
+ struct Impl;
+ std::unique_ptr<Impl> m_Impl;
+};
+
+} // namespace zen \ No newline at end of file
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index 6ee80dc62..f2ed17f05 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -2,7 +2,11 @@
target("zenserver")
set_kind("binary")
+ if enable_unity then
+ add_rules("c++.unity_build", {batchsize = 4})
+ end
add_deps("zencore",
+ "zencompute",
"zenhttp",
"zennet",
"zenremotestore",
@@ -15,6 +19,12 @@ target("zenserver")
add_files("**.cpp")
add_files("frontend/*.zip")
add_files("zenserver.cpp", {unity_ignored = true })
+
+ if is_plat("linux") and not (get_config("toolchain") or ""):find("clang") then
+ -- GCC false positives in deeply inlined code (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100137)
+ add_files("storage/projectstore/httpprojectstore.cpp", {force = {cxxflags = "-Wno-stringop-overflow"} })
+ add_files("storage/storageconfig.cpp", {force = {cxxflags = "-Wno-array-bounds"} })
+ end
add_includedirs(".")
set_symbols("debug")
@@ -23,6 +33,8 @@ target("zenserver")
add_packages("json11")
add_packages("lua")
add_packages("consul")
+ add_packages("oidctoken")
+ add_packages("nomad")
if has_config("zenmimalloc") then
add_packages("mimalloc")
@@ -32,6 +44,14 @@ target("zenserver")
add_packages("sentry-native")
end
+ if has_config("zenhorde") then
+ add_deps("zenhorde")
+ end
+
+ if has_config("zennomad") then
+ add_deps("zennomad")
+ end
+
if is_mode("release") then
set_optimize("fastest")
end
@@ -141,4 +161,24 @@ target("zenserver")
end
copy_if_newer(path.join(installdir, "bin", consul_bin), path.join(target:targetdir(), consul_bin), consul_bin)
end
+
+ local oidctoken_pkg = target:pkg("oidctoken")
+ if oidctoken_pkg then
+ local installdir = oidctoken_pkg:installdir()
+ local oidctoken_bin = "OidcToken"
+ if is_plat("windows") then
+ oidctoken_bin = "OidcToken.exe"
+ end
+ copy_if_newer(path.join(installdir, "bin", oidctoken_bin), path.join(target:targetdir(), oidctoken_bin), oidctoken_bin)
+ end
+
+ local nomad_pkg = target:pkg("nomad")
+ if nomad_pkg then
+ local installdir = nomad_pkg:installdir()
+ local nomad_bin = "nomad"
+ if is_plat("windows") then
+ nomad_bin = "nomad.exe"
+ end
+ copy_if_newer(path.join(installdir, "bin", nomad_bin), path.join(target:targetdir(), nomad_bin), nomad_bin)
+ end
end)
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 2bafeeaa1..bb6b02d21 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -18,11 +18,13 @@
#include <zencore/sentryintegration.h>
#include <zencore/session.h>
#include <zencore/string.h>
+#include <zencore/system.h>
#include <zencore/thread.h>
#include <zencore/timer.h>
#include <zencore/trace.h>
#include <zencore/workthreadpool.h>
#include <zenhttp/httpserver.h>
+#include <zenhttp/security/passwordsecurityfilter.h>
#include <zentelemetry/otlptrace.h>
#include <zenutil/service.h>
#include <zenutil/workerpools.h>
@@ -44,6 +46,20 @@ ZEN_THIRD_PARTY_INCLUDES_END
//////////////////////////////////////////////////////////////////////////
+#ifndef ZEN_WITH_COMPUTE_SERVICES
+# define ZEN_WITH_COMPUTE_SERVICES 0
+#endif
+
+#ifndef ZEN_WITH_HORDE
+# define ZEN_WITH_HORDE 0
+#endif
+
+#ifndef ZEN_WITH_NOMAD
+# define ZEN_WITH_NOMAD 0
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
#include "config/config.h"
#include "diag/logging.h"
@@ -142,8 +158,18 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState::
ZEN_INFO("Effective concurrency: {} (hw: {})", GetHardwareConcurrency(), std::thread::hardware_concurrency());
+ InitializeSecuritySettings(ServerOptions);
+
+ if (ServerOptions.LieCpu)
+ {
+ SetCpuCountForReporting(ServerOptions.LieCpu);
+
+ ZEN_INFO("Reporting concurrency: {}", ServerOptions.LieCpu);
+ }
+
m_StatusService.RegisterHandler("status", *this);
m_Http->RegisterService(m_StatusService);
+ m_Http->RegisterService(m_StatsService);
m_StatsReporter.Initialize(ServerOptions.StatsConfig);
if (ServerOptions.StatsConfig.Enabled)
@@ -151,10 +177,37 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState::
EnqueueStatsReportingTimer();
}
- m_HealthService.SetHealthInfo({.DataRoot = ServerOptions.DataDir,
- .AbsLogPath = ServerOptions.AbsLogFile,
- .HttpServerClass = std::string(ServerOptions.HttpConfig.ServerClass),
- .BuildVersion = std::string(ZEN_CFG_VERSION_BUILD_STRING_FULL)});
+ // clang-format off
+ HealthServiceInfo HealthInfo {
+ .DataRoot = ServerOptions.DataDir,
+ .AbsLogPath = ServerOptions.LoggingConfig.AbsLogFile,
+ .HttpServerClass = std::string(ServerOptions.HttpConfig.ServerClass),
+ .BuildVersion = std::string(ZEN_CFG_VERSION_BUILD_STRING_FULL),
+ .Port = EffectiveBasePort,
+ .Pid = GetCurrentProcessId(),
+ .IsDedicated = ServerOptions.IsDedicated,
+ .StartTimeMs = std::chrono::duration_cast<std::chrono::milliseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count(),
+ .BuildOptions = {
+ {"ZEN_ADDRESS_SANITIZER", ZEN_ADDRESS_SANITIZER != 0},
+ {"ZEN_USE_SENTRY", ZEN_USE_SENTRY != 0},
+ {"ZEN_WITH_TESTS", ZEN_WITH_TESTS != 0},
+ {"ZEN_USE_MIMALLOC", ZEN_USE_MIMALLOC != 0},
+ {"ZEN_USE_RPMALLOC", ZEN_USE_RPMALLOC != 0},
+ {"ZEN_WITH_HTTPSYS", ZEN_WITH_HTTPSYS != 0},
+ {"ZEN_WITH_MEMTRACK", ZEN_WITH_MEMTRACK != 0},
+ {"ZEN_WITH_TRACE", ZEN_WITH_TRACE != 0},
+ {"ZEN_WITH_COMPUTE_SERVICES", ZEN_WITH_COMPUTE_SERVICES != 0},
+ {"ZEN_WITH_HORDE", ZEN_WITH_HORDE != 0},
+ {"ZEN_WITH_NOMAD", ZEN_WITH_NOMAD != 0},
+ },
+ .RuntimeConfig = BuildSettingsList(ServerOptions),
+ };
+ // clang-format on
+
+ HealthInfo.RuntimeConfig.emplace(HealthInfo.RuntimeConfig.begin() + 2, "EffectivePort"sv, fmt::to_string(EffectiveBasePort));
+
+ m_HealthService.SetHealthInfo(std::move(HealthInfo));
LogSettingsSummary(ServerOptions);
@@ -164,12 +217,23 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState::
void
ZenServerBase::Finalize()
{
+ m_StatsService.RegisterHandler("http", *m_Http);
+
+ m_Http->SetDefaultRedirect("/dashboard/");
+
// Register health service last so if we return "OK" for health it means all services have been properly initialized
m_Http->RegisterService(m_HealthService);
}
void
+ZenServerBase::ShutdownServices()
+{
+ m_StatsService.UnregisterHandler("http", *m_Http);
+ m_StatsService.Shutdown();
+}
+
+void
ZenServerBase::GetBuildOptions(StringBuilderBase& OutOptions, char Separator) const
{
ZEN_MEMSCOPE(GetZenserverTag());
@@ -375,46 +439,65 @@ ZenServerBase::CheckSigInt()
void
ZenServerBase::HandleStatusRequest(HttpServerRequest& Request)
{
+ auto Metrics = m_MetricsTracker.Query();
+
CbObjectWriter Cbo;
Cbo << "ok" << true;
Cbo << "state" << ToString(m_CurrentState);
+ Cbo << "hostname" << GetMachineName();
+ Cbo << "cpuUsagePercent" << Metrics.CpuUsagePercent;
Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
}
-void
-ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig)
+std::vector<std::pair<std::string_view, std::string>>
+ZenServerBase::BuildSettingsList(const ZenServerConfig& ServerConfig)
{
// clang-format off
- std::list<std::pair<std::string_view, std::string>> Settings = {
- {"DataDir"sv, ServerConfig.DataDir.string()},
- {"AbsLogFile"sv, ServerConfig.AbsLogFile.string()},
- {"SystemRootDir"sv, ServerConfig.SystemRootDir.string()},
- {"ContentDir"sv, ServerConfig.ContentDir.string()},
+ std::vector<std::pair<std::string_view, std::string>> Settings = {
+ {"SystemRootDir"sv, fmt::format("{}", ServerConfig.SystemRootDir)},
+ {"ContentDir"sv, fmt::format("{}", ServerConfig.ContentDir)},
{"BasePort"sv, fmt::to_string(ServerConfig.BasePort)},
+ {"CoreLimit"sv, fmt::to_string(ServerConfig.CoreLimit)},
{"IsDebug"sv, fmt::to_string(ServerConfig.IsDebug)},
{"IsCleanStart"sv, fmt::to_string(ServerConfig.IsCleanStart)},
{"IsPowerCycle"sv, fmt::to_string(ServerConfig.IsPowerCycle)},
{"IsTest"sv, fmt::to_string(ServerConfig.IsTest)},
{"Detach"sv, fmt::to_string(ServerConfig.Detach)},
- {"NoConsoleOutput"sv, fmt::to_string(ServerConfig.NoConsoleOutput)},
- {"QuietConsole"sv, fmt::to_string(ServerConfig.QuietConsole)},
- {"CoreLimit"sv, fmt::to_string(ServerConfig.CoreLimit)},
- {"IsDedicated"sv, fmt::to_string(ServerConfig.IsDedicated)},
- {"ShouldCrash"sv, fmt::to_string(ServerConfig.ShouldCrash)},
+ {"NoConsoleOutput"sv, fmt::to_string(ServerConfig.LoggingConfig.NoConsoleOutput)},
+ {"QuietConsole"sv, fmt::to_string(ServerConfig.LoggingConfig.QuietConsole)},
{"ChildId"sv, ServerConfig.ChildId},
- {"LogId"sv, ServerConfig.LogId},
+ {"LogId"sv, ServerConfig.LoggingConfig.LogId},
{"Sentry DSN"sv, ServerConfig.SentryConfig.Dsn.empty() ? "not set" : ServerConfig.SentryConfig.Dsn},
{"Sentry Environment"sv, ServerConfig.SentryConfig.Environment},
{"Statsd Enabled"sv, fmt::to_string(ServerConfig.StatsConfig.Enabled)},
+ {"SecurityConfigPath"sv, fmt::format("{}", ServerConfig.SecurityConfigPath)},
};
// clang-format on
if (ServerConfig.StatsConfig.Enabled)
{
- Settings.emplace_back("Statsd Host", ServerConfig.StatsConfig.StatsdHost);
- Settings.emplace_back("Statsd Port", fmt::to_string(ServerConfig.StatsConfig.StatsdPort));
+ Settings.emplace_back("Statsd Host"sv, ServerConfig.StatsConfig.StatsdHost);
+ Settings.emplace_back("Statsd Port"sv, fmt::to_string(ServerConfig.StatsConfig.StatsdPort));
}
+ return Settings;
+}
+
+void
+ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig)
+{
+ auto Settings = BuildSettingsList(ServerConfig);
+
+ // Log-only entries not needed in RuntimeConfig
+ // clang-format off
+ Settings.insert(Settings.begin(), {
+ {"DataDir"sv, fmt::format("{}", ServerConfig.DataDir)},
+ {"AbsLogFile"sv, fmt::format("{}", ServerConfig.LoggingConfig.AbsLogFile)},
+ });
+ // clang-format on
+ Settings.emplace_back("IsDedicated"sv, fmt::to_string(ServerConfig.IsDedicated));
+ Settings.emplace_back("ShouldCrash"sv, fmt::to_string(ServerConfig.ShouldCrash));
+
size_t MaxWidth = 0;
for (const auto& Setting : Settings)
{
@@ -432,6 +515,44 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig)
}
}
+void
+ZenServerBase::InitializeSecuritySettings(const ZenServerConfig& ServerOptions)
+{
+ ZEN_ASSERT(m_Http);
+
+ if (!ServerOptions.SecurityConfigPath.empty())
+ {
+ IoBuffer SecurityJson = ReadFile(ServerOptions.SecurityConfigPath).Flatten();
+ std::string_view Json(reinterpret_cast<const char*>(SecurityJson.GetData()), SecurityJson.GetSize());
+ std::string JsonError;
+ CbObject SecurityConfig = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+ if (!JsonError.empty())
+ {
+ throw std::runtime_error(
+ fmt::format("Invalid security configuration file at {}. '{}'", ServerOptions.SecurityConfigPath, JsonError));
+ }
+
+ CbObjectView HttpRootFilterConfig = SecurityConfig["http"sv].AsObjectView()["root"sv].AsObjectView()["filter"sv].AsObjectView();
+ if (HttpRootFilterConfig)
+ {
+ std::string_view FilterType = HttpRootFilterConfig["type"sv].AsString();
+ if (FilterType == PasswordHttpFilter::TypeName)
+ {
+ PasswordHttpFilter::Configuration Config =
+ PasswordHttpFilter::ReadConfiguration(HttpRootFilterConfig["config"].AsObjectView());
+ m_HttpRequestFilter = std::make_unique<PasswordHttpFilter>(Config);
+ m_Http->SetHttpRequestFilter(m_HttpRequestFilter.get());
+ }
+ else
+ {
+ throw std::runtime_error(fmt::format("Security configuration file at {} references unknown http root filter type '{}'",
+ ServerOptions.SecurityConfigPath,
+ FilterType));
+ }
+ }
+ }
+}
+
//////////////////////////////////////////////////////////////////////////
ZenServerMain::ZenServerMain(ZenServerConfig& ServerOptions) : m_ServerOptions(ServerOptions)
@@ -467,7 +588,7 @@ ZenServerMain::Run()
ZEN_OTEL_SPAN("SentryInit");
std::string SentryDatabasePath = (m_ServerOptions.DataDir / ".sentry-native").string();
- std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string();
+ std::string SentryAttachmentPath = m_ServerOptions.LoggingConfig.AbsLogFile.string();
Sentry.Initialize({.DatabasePath = SentryDatabasePath,
.AttachmentsPath = SentryAttachmentPath,
@@ -567,6 +688,8 @@ ZenServerMain::Run()
{
ZEN_INFO(ZEN_APP_NAME " unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message());
Sleep(500);
+
+ m_LockFile.Create(LockFilePath, MakeLockData(false), Ec);
if (Ec)
{
ZEN_WARN(ZEN_APP_NAME " exiting, unable to grab lock at '{}' (reason: '{}')", LockFilePath, Ec.message());
@@ -622,6 +745,10 @@ ZenServerMain::Run()
RequestApplicationExit(1);
}
+#if ZEN_USE_SENTRY
+ Sentry.Close();
+#endif
+
ShutdownServerLogging();
ReportServiceStatus(ServiceStatus::Stopped);
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index ab7122fcc..c06093f0d 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -3,11 +3,13 @@
#pragma once
#include <zencore/basicfile.h>
+#include <zencore/system.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstats.h>
#include <zenhttp/httpstatus.h>
#include <zenutil/zenserverprocess.h>
+#include <atomic>
#include <memory>
#include <string_view>
#include "config/config.h"
@@ -43,11 +45,18 @@ public:
void SetIsReadyFunc(std::function<void()>&& IsReadyFunc) { m_IsReadyFunc = std::move(IsReadyFunc); }
+ void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; }
+ void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; }
+ void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; }
+ void SetTestMode(bool State) { m_TestMode = State; }
+
protected:
int Initialize(const ZenServerConfig& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry);
void Finalize();
+ void ShutdownServices();
void GetBuildOptions(StringBuilderBase& OutOptions, char Separator = ',') const;
- void LogSettingsSummary(const ZenServerConfig& ServerConfig);
+ static std::vector<std::pair<std::string_view, std::string>> BuildSettingsList(const ZenServerConfig& ServerConfig);
+ void LogSettingsSummary(const ZenServerConfig& ServerConfig);
protected:
NamedMutex m_ServerMutex;
@@ -55,6 +64,10 @@ protected:
bool m_UseSentry = false;
bool m_IsPowerCycle = false;
+ bool m_IsDedicatedMode = false;
+ bool m_TestMode = false;
+ bool m_DebugOptionForcedCrash = false;
+
std::thread m_IoRunner;
asio::io_context m_IoContext;
void EnsureIoRunner();
@@ -64,17 +77,26 @@ protected:
kInitializing,
kRunning,
kShuttingDown
- } m_CurrentState = kInitializing;
+ };
+ std::atomic<ServerState> m_CurrentState = kInitializing;
- inline void SetNewState(ServerState NewState) { m_CurrentState = NewState; }
+ inline void SetNewState(ServerState NewState) { m_CurrentState.store(NewState, std::memory_order_relaxed); }
static std::string_view ToString(ServerState Value);
std::function<void()> m_IsReadyFunc;
void OnReady();
- Ref<HttpServer> m_Http;
- HttpHealthService m_HealthService;
- HttpStatusService m_StatusService;
+ std::filesystem::path m_DataRoot; // Root directory for server state
+ std::filesystem::path m_ContentRoot; // Root directory for frontend content
+
+ Ref<HttpServer> m_Http;
+
+ std::unique_ptr<IHttpRequestFilter> m_HttpRequestFilter;
+
+ HttpHealthService m_HealthService;
+ HttpStatsService m_StatsService{m_IoContext};
+ HttpStatusService m_StatusService;
+ SystemMetricsTracker m_MetricsTracker;
// Stats reporting
@@ -107,8 +129,10 @@ protected:
// IHttpStatusProvider
virtual void HandleStatusRequest(HttpServerRequest& Request) override;
-};
+private:
+ void InitializeSecuritySettings(const ZenServerConfig& ServerOptions);
+};
class ZenServerMain
{
public:
diff --git a/src/zenserver/zenserver.rc b/src/zenserver/zenserver.rc
index e0003ea8f..f353bd9cc 100644
--- a/src/zenserver/zenserver.rc
+++ b/src/zenserver/zenserver.rc
@@ -28,7 +28,7 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
// Icon with lowest ID value placed first to ensure application icon
// remains consistent on all systems.
-IDI_ICON1 ICON "..\\UnrealEngine.ico"
+IDI_ICON1 ICON "..\\zen.ico"
#endif // English (United States) resources
/////////////////////////////////////////////////////////////////////////////