Self-hosted dashboard: Searchable oplog and links between oplog entry dependencies (#213)v5.5.12-pre0

* Consistent use of semicolons

* Added fallback if oplog entry assumptions do not hold

* 'marker' and 'expired' cells were incorrectly friendly

* Two spaces when there should only be one

* Robustness against .text(undefined) calls

* A single step into JavaScript modules

* Turned Fetcher into a module

* Friendly into a module

* Specialise Cbo field name comparison as TextDecoder() is very slow

* Prefer is_named() over get_name()

* Incorrect logic checking if a server reply was okay

* Try and make sure it's always numbers that flow through Friendly

* Added a progress bar component

* Swap key and package hash columns

* CbObject cloning

* Dark and light themes depending on browser settings

* Adjust styling of input boxes

* Add theme swatches to test page

* Turns out one can nest CSS selectors

* Separate swatch for links/actions

* Generate theme by lerping intermediate colours

* Clearer progress bar

* Chromium was complaining about label-less input elements

* Promise-based cache using an IndexedDb

* WebWorker for generating map of package ids to names

* Indexer class for building, loading, and saving map of ids to names

* Added links to oplog entries of an entry's dependencies

* This doesn't need to be decorated as async any longer

* Implemented oplog searching

* View and drop make no sense on package data payloads

* Rudimentary search result truncation

* Updated changelog

* Updated HTML zip archive

1 files changed, 178 insertions, 127 deletions

diff --git a/src/zenserver/frontend/html/zen.js b/src/zenserver/frontend/html/zen.js
index 4d3895d1f..ffeaeb4ee 100644
--- a/src/zenserver/frontend/html/zen.js
+++ b/src/zenserver/frontend/html/zen.js
@@ -2,27 +2,9 @@
 
 "use strict";
 
-////////////////////////////////////////////////////////////////////////////////
-class Friendly
-{
-	static sep(value, prec=0)
-	{
-		return value.toLocaleString("en", {
-			style: "decimal",
-			minimumFractionDigits : prec,
-			maximumFractionDigits : prec,
-		});
-	}
-
-	static k(x)		{ return Friendly.sep((x + 999)  / Math.pow(10, 3), 0) + "K"; }
-	static m(x)		{ return Friendly.sep( x         / Math.pow(10, 6), 1) + "M"; }
-	static g(x)		{ return Friendly.sep( x         / Math.pow(10, 6), 2) + "G"; }
-	static kib(x)	{ return Friendly.sep((x + 1023) / (1 << 10), 0) + " KiB"; }
-	static mib(x)	{ return Friendly.sep( x         / (1 << 20), 1) + " MiB"; }
-	static gib(x)	{ return Friendly.sep( x         / (1 << 30), 2) + " GiB"; }
-}
-
-
+import { Fetcher } from "./util/fetcher.js"
+import { Friendly } from "./util/friendly.js"
+import { create_indexer } from "./indexer/indexer.js"
 
 ////////////////////////////////////////////////////////////////////////////////
 class ComponentBase
@@ -70,7 +52,7 @@ class ComponentDom extends ComponentBase
 
 	text(value)
 	{
-		value = value.toString();
+		value = (value == undefined) ? "undefined" : value.toString();
 		this._element.innerHTML = (value != "") ? value : "";
 		return this;
 	}
@@ -162,7 +144,7 @@ class TableCell extends Component
 	constructor(element, row)
 	{
 		super(element);
-		this._row = row
+		this._row = row;
 	}
 
 	get_table()		{ return this.get_row().get_table(); }
@@ -254,7 +236,7 @@ class Table extends Component
 			cells.push("");
 
 		var ret = [];
-		var row = this.tag()
+		var row = this.tag();
 		row = new TableRow(row, this, index, ret);
 		for (const cell of cells)
 		{
@@ -265,7 +247,7 @@ class Table extends Component
 		if (this._index >= 0)
 			ret.shift();
 
-		return row
+		return row;
 	}
 
 	add_row(...args)
@@ -333,7 +315,7 @@ class PropTable extends Table
 	filter(...needles)
 	{
 		for (var row of this)
-			row.retag("div")
+			row.retag("div");
 
 		if (needles.length == 0)
 			return;
@@ -365,8 +347,8 @@ class Modal
 		bg.on("click", () => this._root.destroy());
 
 		const rect = this._root.tag();
-		this._title = rect.tag().classify("zen_modal_title")
-		this._content = rect.tag().classify("zen_modal_message")
+		this._title = rect.tag().classify("zen_modal_title");
+		this._content = rect.tag().classify("zen_modal_message");
 		this._buttons = rect.tag().classify("zen_modal_buttons");
 	}
 
@@ -412,8 +394,8 @@ class Toolbar extends Component
 		if (inline)
 			root.classify("zen_toolbar_inline");
 
-		this._left = new Toolbar.Side(root.tag())
-		this._right = new Toolbar.Side(root.tag())
+		this._left = new Toolbar.Side(root.tag());
+		this._right = new Toolbar.Side(root.tag());
 	}
 
 	left()		{ return this._left; }
@@ -423,95 +405,44 @@ class Toolbar extends Component
 
 
 ////////////////////////////////////////////////////////////////////////////////
-class Sectormatron extends Component
+class ProgressBar extends Component
 {
-	constructor(parent, depth=1)
+	constructor(parent)
 	{
-		super(parent);
-		this._depth = depth;
+		const root = parent.tag().classify("zen_progressbar");
+		super(root);
+		this._label = root.tag();
+		root.tag(); // bg
+		this._bar = root.tag();
 	}
 
-	add_section(name)
+	set_progress(what, count=0, end=1)
 	{
-		var node = this.tag();
-		if (this._depth == 1)
-			node.classify("zen_sector");
-
-		node.tag("h" + this._depth).text(name);
-		return new Sectormatron(node, this._depth + 1);
+		const percent = (((count * 100) / end) | 0).toString() + "%";
+		this._bar.style("width", percent);
+		this._label.text(`${what}... ${count}/${end} (${percent})`);
 	}
 }
 
 
 
 ////////////////////////////////////////////////////////////////////////////////
-class Fetcher
+class Sectormatron extends Component
 {
-	constructor()
-	{
-		this._resource = "";
-		this._query = {};
-	}
-
-	resource(...parts)
-	{
-		var value = parts.join("/");
-		if (!value.startsWith("/"))
-			value= "/" + value;
-		this._resource = value;
-		return this;
-	}
-
-	param(name, value)
-	{
-		this._query[name] = value;
-		return this;
-	}
-
-	async json()
-	{
-		const response = await this._get("application/json");
-		return response ? (await response.json()) : {};
-	}
-
-	async cbo()
-	{
-		const response = await this._get("application/x-ue-cb");
-		if (!response)
-			return null;
-
-		const buffer = await response.arrayBuffer();
-		const data = new Uint8Array(buffer);
-		return new CbObject(data);
-	}
-
-	async delete()
-	{
-		const resource = this._build_uri();
-		const response = await fetch(resource, { "method" : "DELETE" });
-	}
-
-	_build_uri()
+	constructor(parent, depth=1)
 	{
-		var suffix = "";
-		for (var key in this._query)
-		{
-			suffix += suffix ? "&" : "?";
-			suffix += key + "=" + this._query[key];
-		}
-		return this._resource + suffix;
+		super(parent);
+		this._depth = depth;
 	}
 
-	async _get(accept="*")
+	add_section(name)
 	{
-		const resource = this._build_uri();
-		const response = await fetch(resource, {
-			"method" : "GET",
-			"headers" : { "Accept": accept },
-		});
+		var node = this.tag();
+		if (this._depth == 1)
+			node.classify("zen_sector");
 
-		if (response.status >= 200 || response.status <= 299)
-			return response;
+		node.tag("h" + this._depth).text(name);
+		return new Sectormatron(node, this._depth + 1);
 	}
 }
 
@@ -555,7 +486,7 @@ class Page
 
 		const url = new URL(window.location);
 		for (var [key, xfer] of this._params)
-			url.searchParams.set(key, xfer)
+			url.searchParams.set(key, xfer);
 		history.replaceState(null, "", url);
 
 		return value;
@@ -651,7 +582,7 @@ class ZenPage extends Page
 ////////////////////////////////////////////////////////////////////////////////
 class Entry extends ZenPage
 {
-	async main()
+	main()
 	{
 		this.set_title("oplog entry");
 
@@ -659,11 +590,33 @@ class Entry extends ZenPage
 		const oplog = this.get_param("oplog");
 		const opkey = this.get_param("opkey");
 
-		var entry = new Fetcher()
+		this._entry = new Fetcher()
 			.resource("prj", project, "oplog", oplog, "entries")
 			.param("opkey", opkey)
-			.cbo()
-		entry = await entry;
+			.cbo();
+
+		this.load_indexer(project, oplog, () => this._build_page());
+	}
+
+	async load_indexer(project, oplog, loaded_cb)
+	{
+		if (this._indexer != undefined)
+			return loaded_cb();
+
+		const progress_bar = new ProgressBar(this._parent);
+		progress_bar.set_progress("indexing");
+		const indexer = create_indexer(project, oplog, (...args) => {
+			progress_bar.set_progress(...args);
+		});
+		this._indexer = await indexer;
+		progress_bar.destroy();
+
+		loaded_cb();
+	}
+
+	async _build_page()
+	{
+		var entry = await this._entry;
 		entry = entry.as_object().find("entry").as_object();
 
 		const name = entry.find("key").as_value();
@@ -675,18 +628,23 @@ class Entry extends ZenPage
 			if (tree == undefined)
 				tree = this._convert_legacy_to_tree(entry);
 
+			if (tree == undefined)
+				return this._display_unsupported(section, entry);
+
 			delete tree["$id"];
 
 			const sub_section = section.add_section("deps");
 			for (const dep_name in tree)
 			{
 				const dep_section = sub_section.add_section(dep_name);
-				const table = new Table(dep_section, ["id", "name"], Table.Flag_FitLeft);
+				const table = new Table(dep_section, ["name", "id"], Table.Flag_PackRight);
 				for (const dep_id of tree[dep_name])
 				{
-					const cell_values = [dep_id.toString(16)];
+					const cell_values = ["", dep_id.toString(16).padStart(16, "0")];
 					const row = table.add_row(...cell_values);
-					row.get_cell(0).on_click(() => void(0));
+
+					var opkey = this._indexer.lookup_id(dep_id);
+					row.get_cell(0).text(opkey).on_click((k) => this.view_opkey(k), opkey);
 				}
 			}
 		}
@@ -707,8 +665,8 @@ class Entry extends ZenPage
 					var file_name;
 					for (const field of item.as_object())
 					{
-						if (field.get_name() == "data") io_hash = field.as_value();
-						else if (field.get_name() == "filename") file_name = field.as_value();
+						if (field.is_named("data")) io_hash = field.as_value();
+						else if (field.is_named("filename")) file_name = field.as_value();
 					}
 
 					if (io_hash instanceof Uint8Array)
@@ -732,26 +690,35 @@ class Entry extends ZenPage
 					action_tb.left().add("copy-hash").on_click(async (v) => {
 						await navigator.clipboard.writeText(v);
 					}, io_hash);
-					action_tb.left().add("view");
-					action_tb.left().add("drop");
 				}
 			}
 		}
 
 		// props
 		{
-			const object =  entry.to_js_object();
+			const object = entry.to_js_object();
 			var sub_section = section.add_section("props");
 			new PropTable(sub_section).add_object(object);
 		}
 	}
 
+	_display_unsupported(section, entry)
+	{
+		const object = entry.to_js_object();
+		const text = JSON.stringify(object, null, "  ");
+		section.tag("pre").text(text);
+	}
+
 	_convert_legacy_to_tree(entry)
 	{
+		const pkg_data = entry.find("packagedata");
+		if (pkg_data == undefined)
+			return
+
 		const tree = {};
 
 		var id = 0n;
-		for (var item of entry.find("packagedata").as_array())
+		for (var item of pkg_data.as_array())
 		{
 			var pkg_id = item.as_object().find("id");
 			if (pkg_id == undefined)
@@ -771,7 +738,7 @@ class Entry extends ZenPage
 
 		for (const field of pkgst_entry)
 		{
-			const field_name = field.get_name()
+			const field_name = field.get_name();
 			if (!field_name.endsWith("importedpackageids"))
 				continue;
 
@@ -786,6 +753,13 @@ class Entry extends ZenPage
 
 		return tree;
 	}
+
+	view_opkey(opkey)
+	{
+		const params = this._params;
+		params.set("opkey", opkey);
+		window.location.search = params;
+	}
 }
 
 
@@ -807,16 +781,30 @@ class Oplog extends ZenPage
 		const project = this.get_param("project");
 		const oplog = this.get_param("oplog");
 
+		this._indexer = this._create_indexer(project, oplog);
+
 		this.set_title("oplog - " + oplog);
 
 		var section = this.add_section(project + " - " + oplog);
 
-		this._build_nav(section)
+		this._build_nav(section);
 
 		this._entry_table = new Table(section, ["key"]);
 		await this._build_table();
 	}
 
+	async _create_indexer(project, oplog)
+	{
+		const progress_bar = new ProgressBar(this._parent);
+		progress_bar.set_progress("indexing");
+		var indexer = create_indexer(project, oplog, (...args) => {
+			progress_bar.set_progress(...args);
+		});
+		indexer = await indexer;
+		progress_bar.destroy();
+		return indexer;
+	}
+
 	_build_nav(section)
 	{
 		var nav = new Toolbar(section);
@@ -832,8 +820,8 @@ class Oplog extends ZenPage
 			nav.left().add(count).on_click(handler, count);
 		}
 
-		nav.right().add("search:", "label");
-		nav.right().add("", "input").attr("disabled", "");
+		var search_input = nav.right().add("search:", "label").tag("input")
+		search_input.on("change", (x) => this._search(x.inner().value), search_input);
 	}
 
 	async _build_table()
@@ -851,7 +839,7 @@ class Oplog extends ZenPage
 
 		entries = (await entries)["entries"];
 		if (entries == undefined)
-			return
+			return;
 
 		for (const entry of entries)
 		{
@@ -878,6 +866,40 @@ class Oplog extends ZenPage
 		this._index_start = Math.max(0, index);
 		this._build_table();
 	}
+
+	async _search(needle)
+	{
+		needle = needle.trim();
+		if (needle.length < 3)
+			return;
+
+		this._entry_table.clear(this._index_start);
+
+		const project = this.get_param("project");
+		const oplog = this.get_param("oplog");
+
+		const indexer = await this._indexer;
+
+		var added = 0;
+		const truncate_at = this.get_param("searchmax") || 250;
+		for (var name of indexer.search(needle))
+		{
+			var row = this._entry_table.add_row(name);
+
+			row.get_cell(0).link("", {
+				"page" : "entry",
+				"project" : project,
+				"oplog" : oplog,
+				"opkey" : name,
+			});
+
+			if (++added >= truncate_at)
+			{
+				this._entry_table.add_row("...truncated");
+				break;
+			}
+		}
+	}
 }
 
 
@@ -934,10 +956,10 @@ class Project extends ZenPage
 			action_tb.left().add("drop").on_click((x) => this.drop_oplog(x), name);
 
 			info = await info;
-			row.get_cell(1).text(Friendly.sep(info["markerpath"]));
+			row.get_cell(1).text(info["markerpath"]);
 			row.get_cell(2).text(Friendly.kib(info["totalsize"]));
 			row.get_cell(3).text(Friendly.sep(info["opcount"]));
-			row.get_cell(4).text(Friendly.sep(info["expired"]));
+			row.get_cell(4).text(info["expired"]);
 		}
 	}
 
@@ -954,7 +976,7 @@ class Project extends ZenPage
 			.title("Confirmation")
 			.message(`Drop oplog '${oplog_id}'?`)
 			.option("Yes", () => drop())
-			.option("No")
+			.option("No");
 	}
 }
 
@@ -973,10 +995,10 @@ class Test extends ZenPage
 			};
 			return function(a=5, b=10) {
 				const co = "aeioubcdfghjklmnpqrstvwxyz";
-				var ret = ""
+				var ret = "";
 				for (var i = 0, n = r(a,b); i < n; ++i)
 					ret += co[r(0, co.length)];
-				return ret
+				return ret;
 			};
 		})();
 		var gen_para = function(a=5, b=10, s=" ") {
@@ -988,6 +1010,23 @@ class Test extends ZenPage
 
 		this.set_title("test");
 
+		// swatches
+		const swatches = this._parent.tag()
+			.style("position", "absolute")
+			.style("top", "3.5em")
+			.style("left", "3.5em")
+		for (var suffix of ["g0", "g1", "g2", "g3", "g4",
+							"p0", "p1", "p2", "p3", "p4",
+							"ln", "er"])
+		{
+			var swatch = swatches.tag()
+				.style("float", "left")
+				.style("width", "2em")
+				.style("height", "2em")
+				.style("background-color", `var(--theme_${suffix})`)
+				.text(suffix);
+		}
+
 		// section
 		var section0 = this.add_section("section");
 		var section1 = section0.add_section("sub-section");
@@ -1068,6 +1107,16 @@ class Test extends ZenPage
 		toolbar.right().add("tbitem2").on_click(tb_item_clicked, 33, -55);
 		toolbar.right().add("tbitem3").on_click(tb_item_clicked, 44, -88);
 
+		// progress bar
+		const progress_bar = new ProgressBar(this._parent);
+		setInterval(function() {
+			var count = 0
+			return () => {
+				count = (count + 1) % 100;
+				progress_bar.set_progress("testing", count, 100);
+			};
+		}(), 49.3);
+
 		// error
 		throw Error("deliberate error");
 	}
@@ -1236,7 +1285,7 @@ class Stat extends ZenPage
 		tb_right.add("-none-").on_click((x) => this.update_filter(""));
 		for (var preset of ["read.", "write.", ".request", ".bytes"])
 			tb_right.add(preset).on_click((x) => this.update_filter(x), preset);
-		this._filter_input = tb_right.add("", "input")
+		this._filter_input = tb_right.add("", "label").tag("input");
 		this._filter_input.on("change", (x) => this.update_filter(x.inner().value), this._filter_input);
 
 		this._table = new PropTable(section);
@@ -1356,3 +1405,5 @@ async function main()
 
 	impl.main();
 }
+
+main();