// Copyright Epic Games, Inc. All Rights Reserved. "use strict"; import { ZenPage } from "./page.js" import { Fetcher } from "../util/fetcher.js" import { Friendly } from "../util/friendly.js" import { Modal } from "../util/modal.js" import { Table, Toolbar } from "../util/widgets.js" //////////////////////////////////////////////////////////////////////////////// export class Page extends ZenPage { generate_crumbs() {} async main() { this.set_title("cache"); // Cache Service Stats const stats_section = this._collapsible_section("Cache Service Stats"); stats_section.tag().classify("dropall").text("raw yaml \u2192").on_click(() => { window.open("/stats/z$.yaml?cidstorestats=true&cachestorestats=true", "_blank"); }); this._stats_grid = stats_section.tag().classify("grid").classify("stats-tiles"); this._details_host = stats_section; this._details_container = null; this._selected_category = null; const stats = await new Fetcher().resource("stats", "z$").json(); if (stats) { this._render_stats(stats); } this.connect_stats_ws((all_stats) => { const stats = all_stats["z$"]; if (stats) { this._render_stats(stats); } }); // Cache Namespaces var section = this._collapsible_section("Cache Namespaces"); section.tag().classify("dropall").text("drop-all").on_click(() => this.drop_all()); var columns = [ "namespace", "dir", "buckets", "entries", "size disk", "size mem", "actions", ]; var zcache_info = await new Fetcher().resource("/z$/").json(); this._cache_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_AlignNumeric); for (const namespace of zcache_info["Namespaces"] || []) { new Fetcher().resource(`/z$/${namespace}/`).json().then((data) => { const row = this._cache_table.add_row( "", data["Configuration"]["RootDir"], data["Buckets"].length, data["EntryCount"], Friendly.bytes(data["StorageSize"].DiskSize), Friendly.bytes(data["StorageSize"].MemorySize) ); var cell = row.get_cell(0); cell.tag().text(namespace).on_click(() => this.view_namespace(namespace)); cell = row.get_cell(-1); const action_tb = new Toolbar(cell, true); action_tb.left().add("view").on_click(() => this.view_namespace(namespace)); action_tb.left().add("drop").on_click(() => this.drop_namespace(namespace)); row.attr("zs_name", namespace); }); } // Namespace detail area (inside namespaces section so it collapses together) this._namespace_host = section; this._namespace_container = null; this._selected_namespace = null; // Restore namespace from URL if present const ns_param = this.get_param("namespace"); if (ns_param) { this.view_namespace(ns_param); } } _collapsible_section(name) { const section = this.add_section(name); const container = section._parent.inner(); const heading = container.firstElementChild; heading.style.cursor = "pointer"; heading.style.userSelect = "none"; const indicator = document.createElement("span"); indicator.textContent = " \u25BC"; indicator.style.fontSize = "0.7em"; heading.appendChild(indicator); let collapsed = false; heading.addEventListener("click", (e) => { if (e.target !== heading && e.target !== indicator) { return; } collapsed = !collapsed; indicator.textContent = collapsed ? " \u25B6" : " \u25BC"; let sibling = heading.nextElementSibling; while (sibling) { sibling.style.display = collapsed ? "none" : ""; sibling = sibling.nextElementSibling; } }); return section; } _render_stats(stats) { const safe = (obj, path) => path.split(".").reduce((a, b) => a && a[b], obj); const grid = this._stats_grid; this._last_stats = stats; grid.inner().innerHTML = ""; // Store I/O tile { const store = safe(stats, "cache.store"); if (store) { const tile = grid.tag().classify("card").classify("stats-tile").classify("stats-tile-detailed"); if (this._selected_category === "store") tile.classify("stats-tile-selected"); tile.on_click(() => this._select_category("store")); tile.tag().classify("card-title").text("Store I/O"); const columns = tile.tag().classify("tile-columns"); const left = columns.tag().classify("tile-metrics"); const storeHits = store.hits || 0; const storeMisses = store.misses || 0; const storeTotal = storeHits + storeMisses; const storeRatio = storeTotal > 0 ? ((storeHits / storeTotal) * 100).toFixed(1) + "%" : "-"; this._metric(left, storeRatio, "store hit ratio", true); this._metric(left, Friendly.sep(storeHits), "hits"); this._metric(left, Friendly.sep(storeMisses), "misses"); this._metric(left, Friendly.sep(store.writes || 0), "writes"); this._metric(left, Friendly.sep(store.rejected_reads || 0), "rejected reads"); this._metric(left, Friendly.sep(store.rejected_writes || 0), "rejected writes"); const right = columns.tag().classify("tile-metrics"); const readRateMean = safe(store, "read.bytes.rate_mean") || 0; const readRate1 = safe(store, "read.bytes.rate_1") || 0; const readRate5 = safe(store, "read.bytes.rate_5") || 0; const writeRateMean = safe(store, "write.bytes.rate_mean") || 0; const writeRate1 = safe(store, "write.bytes.rate_1") || 0; const writeRate5 = safe(store, "write.bytes.rate_5") || 0; this._metric(right, Friendly.bytes(readRateMean) + "/s", "read rate (mean)", true); this._metric(right, Friendly.bytes(readRate1) + "/s", "read rate (1m)"); this._metric(right, Friendly.bytes(readRate5) + "/s", "read rate (5m)"); this._metric(right, Friendly.bytes(writeRateMean) + "/s", "write rate (mean)"); this._metric(right, Friendly.bytes(writeRate1) + "/s", "write rate (1m)"); this._metric(right, Friendly.bytes(writeRate5) + "/s", "write rate (5m)"); } } // Hit/Miss tile { const tile = grid.tag().classify("card").classify("stats-tile"); tile.tag().classify("card-title").text("Hit Ratio"); const columns = tile.tag().classify("tile-columns"); const left = columns.tag().classify("tile-metrics"); const hits = safe(stats, "cache.hits") || 0; const misses = safe(stats, "cache.misses") || 0; const writes = safe(stats, "cache.writes") || 0; const total = hits + misses; const ratio = total > 0 ? ((hits / total) * 100).toFixed(1) + "%" : "-"; this._metric(left, ratio, "hit ratio", true); this._metric(left, Friendly.sep(hits), "hits"); this._metric(left, Friendly.sep(misses), "misses"); this._metric(left, Friendly.sep(writes), "writes"); const right = columns.tag().classify("tile-metrics"); const cidHits = safe(stats, "cache.cidhits") || 0; const cidMisses = safe(stats, "cache.cidmisses") || 0; const cidWrites = safe(stats, "cache.cidwrites") || 0; const cidTotal = cidHits + cidMisses; const cidRatio = cidTotal > 0 ? ((cidHits / cidTotal) * 100).toFixed(1) + "%" : "-"; this._metric(right, cidRatio, "cid hit ratio", true); this._metric(right, Friendly.sep(cidHits), "cid hits"); this._metric(right, Friendly.sep(cidMisses), "cid misses"); this._metric(right, Friendly.sep(cidWrites), "cid writes"); } // HTTP Requests tile { const req = safe(stats, "requests"); if (req) { const tile = grid.tag().classify("card").classify("stats-tile"); tile.tag().classify("card-title").text("HTTP Requests"); const columns = tile.tag().classify("tile-columns"); const left = columns.tag().classify("tile-metrics"); const reqData = req.requests || req; this._metric(left, Friendly.sep(reqData.count || 0), "total requests", true); if (reqData.rate_mean > 0) { this._metric(left, Friendly.sep(reqData.rate_mean, 1) + "/s", "req/sec (mean)"); } if (reqData.rate_1 > 0) { this._metric(left, Friendly.sep(reqData.rate_1, 1) + "/s", "req/sec (1m)"); } if (reqData.rate_5 > 0) { this._metric(left, Friendly.sep(reqData.rate_5, 1) + "/s", "req/sec (5m)"); } if (reqData.rate_15 > 0) { this._metric(left, Friendly.sep(reqData.rate_15, 1) + "/s", "req/sec (15m)"); } const badRequests = safe(stats, "cache.badrequestcount") || 0; this._metric(left, Friendly.sep(badRequests), "bad requests"); const right = columns.tag().classify("tile-metrics"); this._metric(right, Friendly.duration(reqData.t_avg || 0), "avg latency", true); if (reqData.t_p75) { this._metric(right, Friendly.duration(reqData.t_p75), "p75"); } if (reqData.t_p95) { this._metric(right, Friendly.duration(reqData.t_p95), "p95"); } if (reqData.t_p99) { this._metric(right, Friendly.duration(reqData.t_p99), "p99"); } if (reqData.t_p999) { this._metric(right, Friendly.duration(reqData.t_p999), "p999"); } if (reqData.t_max) { this._metric(right, Friendly.duration(reqData.t_max), "max"); } } } // RPC tile { const rpc = safe(stats, "cache.rpc"); if (rpc) { const tile = grid.tag().classify("card").classify("stats-tile"); tile.tag().classify("card-title").text("RPC"); const columns = tile.tag().classify("tile-columns"); const left = columns.tag().classify("tile-metrics"); this._metric(left, Friendly.sep(rpc.count || 0), "rpc calls", true); this._metric(left, Friendly.sep(rpc.ops || 0), "batch ops"); const right = columns.tag().classify("tile-metrics"); if (rpc.records) { this._metric(right, Friendly.sep(rpc.records.count || 0), "record calls"); this._metric(right, Friendly.sep(rpc.records.ops || 0), "record ops"); } if (rpc.values) { this._metric(right, Friendly.sep(rpc.values.count || 0), "value calls"); this._metric(right, Friendly.sep(rpc.values.ops || 0), "value ops"); } if (rpc.chunks) { this._metric(right, Friendly.sep(rpc.chunks.count || 0), "chunk calls"); this._metric(right, Friendly.sep(rpc.chunks.ops || 0), "chunk ops"); } } } // Storage tile { const tile = grid.tag().classify("card").classify("stats-tile").classify("stats-tile-detailed"); if (this._selected_category === "storage") tile.classify("stats-tile-selected"); tile.on_click(() => this._select_category("storage")); tile.tag().classify("card-title").text("Storage"); const columns = tile.tag().classify("tile-columns"); const left = columns.tag().classify("tile-metrics"); this._metric(left, safe(stats, "cache.size.disk") != null ? Friendly.bytes(safe(stats, "cache.size.disk")) : "-", "cache disk", true); this._metric(left, safe(stats, "cache.size.memory") != null ? Friendly.bytes(safe(stats, "cache.size.memory")) : "-", "cache memory"); const right = columns.tag().classify("tile-metrics"); this._metric(right, safe(stats, "cid.size.total") != null ? Friendly.bytes(safe(stats, "cid.size.total")) : "-", "cid total", true); this._metric(right, safe(stats, "cid.size.tiny") != null ? Friendly.bytes(safe(stats, "cid.size.tiny")) : "-", "cid tiny"); this._metric(right, safe(stats, "cid.size.small") != null ? Friendly.bytes(safe(stats, "cid.size.small")) : "-", "cid small"); this._metric(right, safe(stats, "cid.size.large") != null ? Friendly.bytes(safe(stats, "cid.size.large")) : "-", "cid large"); } // Upstream tile (only if upstream is active) { const upstream = safe(stats, "upstream"); if (upstream) { const tile = grid.tag().classify("card").classify("stats-tile"); tile.tag().classify("card-title").text("Upstream"); const body = tile.tag().classify("tile-metrics"); const upstreamHits = safe(stats, "cache.upstream_hits") || 0; this._metric(body, Friendly.sep(upstreamHits), "upstream hits", true); if (upstream.url) { this._metric(body, upstream.url, "endpoint"); } } } } _metric(parent, value, label, hero = false) { const m = parent.tag().classify("tile-metric"); if (hero) { m.classify("tile-metric-hero"); } m.tag().classify("metric-value").text(value); m.tag().classify("metric-label").text(label); } async _select_category(category) { // Toggle off if already selected if (this._selected_category === category) { this._selected_category = null; this._clear_details(); this._render_stats(this._last_stats); return; } this._selected_category = category; this._render_stats(this._last_stats); // Fetch detailed stats const detailed = await new Fetcher() .resource("stats", "z$") .param("cachestorestats", "true") .param("cidstorestats", "true") .json(); if (!detailed || this._selected_category !== category) { return; } this._clear_details(); const safe = (obj, path) => path.split(".").reduce((a, b) => a && a[b], obj); if (category === "store") { this._render_store_details(detailed, safe); } else if (category === "storage") { this._render_storage_details(detailed, safe); } } _clear_details() { if (this._details_container) { this._details_container.inner().remove(); this._details_container = null; } } _render_store_details(stats, safe) { const namespaces = safe(stats, "cache.store.namespaces") || []; if (namespaces.length === 0) { return; } const container = this._details_host.tag(); this._details_container = container; const columns = [ "namespace", "bucket", "hits", "misses", "writes", "hit ratio", "read count", "read bandwidth", "write count", "write bandwidth", ]; const table = new Table(container, columns, Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric); for (const ns of namespaces) { const nsHits = ns.hits || 0; const nsMisses = ns.misses || 0; const nsTotal = nsHits + nsMisses; const nsRatio = nsTotal > 0 ? ((nsHits / nsTotal) * 100).toFixed(1) + "%" : "-"; const readCount = safe(ns, "read.request.count") || 0; const readBytes = safe(ns, "read.bytes.count") || 0; const writeCount = safe(ns, "write.request.count") || 0; const writeBytes = safe(ns, "write.bytes.count") || 0; table.add_row( ns.namespace, "", Friendly.sep(nsHits), Friendly.sep(nsMisses), Friendly.sep(ns.writes || 0), nsRatio, Friendly.sep(readCount), Friendly.bytes(readBytes), Friendly.sep(writeCount), Friendly.bytes(writeBytes), ); if (ns.buckets && ns.buckets.length > 0) { for (const bucket of ns.buckets) { const bHits = bucket.hits || 0; const bMisses = bucket.misses || 0; const bTotal = bHits + bMisses; const bRatio = bTotal > 0 ? ((bHits / bTotal) * 100).toFixed(1) + "%" : "-"; const bReadCount = safe(bucket, "read.request.count") || 0; const bReadBytes = safe(bucket, "read.bytes.count") || 0; const bWriteCount = safe(bucket, "write.request.count") || 0; const bWriteBytes = safe(bucket, "write.bytes.count") || 0; table.add_row( ns.namespace, bucket.bucket, Friendly.sep(bHits), Friendly.sep(bMisses), Friendly.sep(bucket.writes || 0), bRatio, Friendly.sep(bReadCount), Friendly.bytes(bReadBytes), Friendly.sep(bWriteCount), Friendly.bytes(bWriteBytes), ); } } } } _render_storage_details(stats, safe) { const namespaces = safe(stats, "cache.store.namespaces") || []; if (namespaces.length === 0) { return; } const container = this._details_host.tag(); this._details_container = container; const columns = [ "namespace", "bucket", "disk", "memory", ]; const table = new Table(container, columns, Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric); for (const ns of namespaces) { const diskSize = safe(ns, "size.disk") || 0; const memSize = safe(ns, "size.memory") || 0; table.add_row( ns.namespace, "", Friendly.bytes(diskSize), Friendly.bytes(memSize), ); if (ns.buckets && ns.buckets.length > 0) { for (const bucket of ns.buckets) { const bDisk = safe(bucket, "size.disk") || 0; const bMem = safe(bucket, "size.memory") || 0; table.add_row( ns.namespace, bucket.bucket, Friendly.bytes(bDisk), Friendly.bytes(bMem), ); } } } } async view_namespace(namespace) { // Toggle off if already selected if (this._selected_namespace === namespace) { this._selected_namespace = null; this._clear_namespace(); this._clear_param("namespace"); return; } this._selected_namespace = namespace; this._clear_namespace(); this.set_param("namespace", namespace); const info = await new Fetcher().resource(`/z$/${namespace}/`).json(); if (this._selected_namespace !== namespace) { return; } const section = this._namespace_host.add_section(namespace); this._namespace_container = section; // Buckets table const bucket_section = section.add_section("Buckets"); const bucket_columns = ["name", "disk", "memory", "entries", "actions"]; const bucket_table = bucket_section.add_widget( Table, bucket_columns, Table.Flag_FitLeft|Table.Flag_PackRight|Table.Flag_Sortable|Table.Flag_AlignNumeric ); // Right-align header for numeric columns (skip # and name) const header = bucket_table._element.firstElementChild; for (let i = 2; i < header.children.length - 1; i++) { header.children[i].style.textAlign = "right"; } let totalDisk = 0, totalMem = 0, totalEntries = 0; const total_row = bucket_table.add_row("TOTAL"); total_row.get_cell(0).style("fontWeight", "bold"); total_row.get_cell(1).style("textAlign", "right").style("fontWeight", "bold"); total_row.get_cell(2).style("textAlign", "right").style("fontWeight", "bold"); total_row.get_cell(3).style("textAlign", "right").style("fontWeight", "bold"); for (const bucket of info["Buckets"]) { const row = bucket_table.add_row(bucket); new Fetcher().resource(`/z$/${namespace}/${bucket}`).json().then((data) => { row.get_cell(1).text(Friendly.bytes(data["StorageSize"]["DiskSize"])).style("textAlign", "right"); row.get_cell(2).text(Friendly.bytes(data["StorageSize"]["MemorySize"])).style("textAlign", "right"); row.get_cell(3).text(Friendly.sep(data["DiskEntryCount"])).style("textAlign", "right"); const cell = row.get_cell(-1); const action_tb = new Toolbar(cell, true); action_tb.left().add("drop").on_click(() => this.drop_bucket(namespace, bucket)); totalDisk += data["StorageSize"]["DiskSize"]; totalMem += data["StorageSize"]["MemorySize"]; totalEntries += data["DiskEntryCount"]; total_row.get_cell(1).text(Friendly.bytes(totalDisk)).style("textAlign", "right").style("fontWeight", "bold"); total_row.get_cell(2).text(Friendly.bytes(totalMem)).style("textAlign", "right").style("fontWeight", "bold"); total_row.get_cell(3).text(Friendly.sep(totalEntries)).style("textAlign", "right").style("fontWeight", "bold"); }); } } _clear_param(name) { this._params.delete(name); const url = new URL(window.location); url.searchParams.delete(name); history.replaceState(null, "", url); } _clear_namespace() { if (this._namespace_container) { this._namespace_container._parent.inner().remove(); this._namespace_container = null; } } drop_bucket(namespace, bucket) { const drop = async () => { await new Fetcher().resource("z$", namespace, bucket).delete(); // Refresh the namespace view this._selected_namespace = null; this._clear_namespace(); this.view_namespace(namespace); }; new Modal() .title("Confirmation") .message(`Drop bucket '${bucket}'?`) .option("Yes", () => drop()) .option("No"); } drop_namespace(namespace) { const drop = async () => { await new Fetcher().resource("z$", namespace).delete(); this.reload(); }; new Modal() .title("Confirmation") .message(`Drop cache namespace '${namespace}'?`) .option("Yes", () => drop()) .option("No"); } async drop_all() { const drop = async () => { for (const row of this._cache_table) { const namespace = row.attr("zs_name"); await new Fetcher().resource("z$", namespace).delete(); } this.reload(); }; new Modal() .title("Confirmation") .message("Drop every cache namespace?") .option("Yes", () => drop()) .option("No"); } }