From 2a445406e09328cb4cf320300f2678997d6775b7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 23 Mar 2026 14:19:57 +0100 Subject: Dashboard refresh (logs, storage, network, object store, docs) (#835) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR adds a session management service, several new dashboard pages, and a number of infrastructure improvements. ### Sessions Service - `SessionsServiceClient` in `zenutil` announces sessions to a remote zenserver with a 15s heartbeat (POST/PUT/DELETE lifecycle) - Storage server registers itself with its own local sessions service on startup - Session mode attribute coupled to server mode (Compute, Proxy, Hub, etc.) - Ended sessions tracked with `ended_at` timestamp; status filtering (Active/Ended/All) - `--sessions-url` config option for remote session announcement - In-process log sink (`InProcSessionLogSink`) forwards server log output to the server's own session, visible in the dashboard ### Session Log Viewer - POST/GET endpoints for session logs (`/sessions/{id}/log`) supporting raw text and structured JSON/CbObject with batch `entries` array - In-memory log storage per session (capped at 10k entries) with cursor-based pagination for efficient incremental fetching - Log panel in the sessions dashboard with incremental DOM updates, auto-scroll (Follow toggle), newest-first toggle, text filter, and log-level coloring - Auto-selects the server's own session on page load ### TCP Log Streaming - `LogStreamListener` and `TcpLogStreamSink` for log delivery over TCP - Sequence numbers on each message with drop detection and synthetic "dropped" notice on gaps - Gathered buffer writes to reduce syscall overhead when flushing batches - Tests covering basic delivery, multi-line splitting, drop detection, and sequencing ### New Dashboard Pages - **Sessions**: master-detail layout with selectable rows, metadata panel, live WebSocket updates, paging, abbreviated date formatting, and "this" pill for the local session - **Object Store**: summary stats tiles and bucket table with click-to-expand inline object listing (`GET /obj/`) - **Storage**: per-volume disk usage breakdown (`GET /admin/storage`), Garbage Collection status section (next-run countdown, last-run stats), and GC History table with paginated rows and expandable detail panels - **Network**: overview tiles, per-service request table, proxy connections, and live WebSocket updates; distinct client IPs and session counts via HyperLogLog ### Documentation Page - In-dashboard Docs page with sidebar navigation, markdown rendering (via `marked`), Mermaid diagram support (theme-aware), collapsible sections, text filtering with highlighting, and cross-document linking - New user-facing docs: `overview.md` (with architecture and per-mode diagrams), `sessions.md`, `cache.md`, `projects.md`; updated `compute.md` - Dev docs moved to `docs/dev/` ### Infrastructure & Bug Fixes - **Deflate compression** for the embedded frontend zip (~3.4MB → ~950KB); zlib inflate support added to `ZipFs` with cached decompressed buffers - **Local IP addresses**: `GetLocalIpAddresses()` (Windows via `GetAdaptersAddresses`, Linux/Mac via `getifaddrs`); surfaced in `/status/status`, `/health/info`, and the dashboard banner - **Dashboard nav**: unified into `zen-nav` web component with `MutationObserver` for dynamically added links, CSS `::part()` to merge banner/nav border radii, and prefix-based active link detection - Stats broadcast refactored from manual JSON string concatenation to `CbObjectWriter`; `CbObject`-to-JS conversion improved for `TimeSpan`, `DateTime`, and large integers - Stats WebSocket boilerplate consolidated into `ZenPage.connect_stats_ws()` --- src/zenserver/storage/admin/admin.cpp | 133 ++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) (limited to 'src/zenserver/storage/admin/admin.cpp') diff --git a/src/zenserver/storage/admin/admin.cpp b/src/zenserver/storage/admin/admin.cpp index 6e78a6179..f1c2daea4 100644 --- a/src/zenserver/storage/admin/admin.cpp +++ b/src/zenserver/storage/admin/admin.cpp @@ -790,6 +790,139 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView()); }, HttpVerb::kPost); + m_Router.RegisterRoute( + "storage", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + + // Collect known storage directories + struct StorageDir + { + std::string_view Name; + std::filesystem::path Path; + }; + + std::vector Dirs; + const std::filesystem::path& DataDir = m_ServerOptions.DataDir; + Dirs.push_back({"cache"sv, DataDir / "cache"}); + Dirs.push_back({"cas"sv, DataDir / "cas"}); + Dirs.push_back({"projects"sv, DataDir / "projects"}); + Dirs.push_back({"builds"sv, DataDir / "builds"}); + Dirs.push_back({"builds_cas"sv, DataDir / "builds_cas"}); + Dirs.push_back({"obj"sv, DataDir / "obj"}); + + // Group directories by volume (identified by total capacity) + struct VolumeInfo + { + DiskSpace Space; + std::vector Directories; + }; + + // Use canonical path to identify volumes. Directories on the same volume + // will report the same total capacity from DiskSpaceInfo. + std::vector Volumes; + auto FindOrAddVolume = [&](DiskSpace Space) -> VolumeInfo& { + for (VolumeInfo& V : Volumes) + { + if (V.Space.Total == Space.Total && V.Space.Free == Space.Free) + { + return V; + } + } + Volumes.push_back({Space, {}}); + return Volumes.back(); + }; + + for (StorageDir& Dir : Dirs) + { + if (!IsDir(Dir.Path)) + { + continue; + } + DiskSpace Space; + if (DiskSpaceInfo(Dir.Path, Space)) + { + FindOrAddVolume(Space).Directories.push_back(&Dir); + } + } + + Obj.BeginArray("volumes"sv); + for (const VolumeInfo& Vol : Volumes) + { + Obj.BeginObject(); + Obj << "total"sv << Vol.Space.Total; + Obj << "free"sv << Vol.Space.Free; + Obj << "used"sv << (Vol.Space.Total - Vol.Space.Free); + + Obj.BeginArray("directories"sv); + for (const StorageDir* Dir : Vol.Directories) + { + Obj.BeginObject(); + Obj << "name"sv << Dir->Name; + Obj << "path"sv << Dir->Path.string(); + + DirStats Stats = GetStatsForDirectory(Dir->Path); + Obj << "bytes"sv << Stats.ByteCount; + Obj << "files"sv << Stats.FileCount; + Obj.EndObject(); + } + Obj.EndArray(); + Obj.EndObject(); + } + Obj.EndArray(); + + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "gclog", + [this](HttpRouterRequest& Req) { + const GcSchedulerState State = m_GcScheduler.GetState(); + const std::filesystem::path Path = State.Config.RootDirectory / "gc.log"; + + CbObjectWriter Response; + Response.BeginArray("entries"sv); + + try + { + if (IsFile(Path)) + { + IoBuffer FileData = ReadFile(Path).Flatten(); + + // The log file contains concatenated named CBO object fields. + // Each field is a complete entry: [type+name header][object payload]. + // We wrap each one in a CbObject and add it to the response array. + const uint8_t* Ptr = static_cast(FileData.GetData()); + const uint8_t* End = Ptr + FileData.GetSize(); + + while (Ptr < End) + { + CbFieldView Field(Ptr); + uint64_t FieldSize = Field.GetSize(); + if (FieldSize == 0 || Ptr + FieldSize > End) + { + break; + } + + // Wrap the named field as an object and add it + CbObjectView ObjView = Field.AsObjectView(); + CbObject Entry = CbObject::Clone(ObjView); + Response.AddObject(Entry); + Ptr += FieldSize; + } + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("failed to read gc log '{}': {}", Path, Ex.what()); + } + + Response.EndArray(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save()); + }, + HttpVerb::kGet); + m_Router.RegisterRoute( "flush", [this](HttpRouterRequest& Req) { -- cgit v1.2.3