Dashboard: display package data sizes in oplog entry and tree views. (#232)

* Wrong divisor for friendly giga-values

* We want Explorer style for kilo/kibi units; round up

* var -> const - zero idea if this matters

* Include sum of an entry's package data sizes in index

* Method to enurate all properties of a loaded oplog index

* Include bulkdata size in an oplog index

* Found a space that was missing

* Show package data sizes when viewing an oplog entry

* Navigating a component tree would error out at the end of the chain

* Parameterise friendly rounding

* Added size and rawsize columns to oplog tree view

* Sort of parameterised indexer's worker count and page size

* Right-align size columns on entry view page

* Updated frontend .zip archive

* A changelog update

8 files changed, 168 insertions, 42 deletions

diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index 30d67ea51..5f53d9f38 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
diff --git a/src/zenserver/frontend/html/indexer/indexer.js b/src/zenserver/frontend/html/indexer/indexer.js
index 5bbb7c352..4412e3a57 100644
--- a/src/zenserver/frontend/html/indexer/indexer.js
+++ b/src/zenserver/frontend/html/indexer/indexer.js
@@ -55,6 +55,13 @@ class Indexer
 			for (const [_, name] of page)
 				yield name;
 	}
+
+	*enum_all()
+	{
+		for (const page of this._pages)
+			for (const [_, name, size, raw_size] of page)
+				yield [name, size|0, raw_size|0];
+	}
 }
 
 
@@ -90,14 +97,13 @@ async function save(progress_cb, oplog_info, pages)
 }
 
 ////////////////////////////////////////////////////////////////////////////////
-async function build(progress_cb, oplog_info)
+async function build(progress_cb, oplog_info, max_workers=6, page_size=48 << 10)
 {
 	const project_id = oplog_info["project"];
 	const oplog = oplog_info["id"];
 	const init_msg = Message.create(Message.Init, project_id, oplog);
 
-	const worker_n = Math.min(navigator.hardwareConcurrency / 2,  6);
-	const page_size = 48 << 10;
+	const worker_n = Math.min(navigator.hardwareConcurrency / 2,  max_workers);
 	const stride = page_size * worker_n;
 	const end = oplog_info["opcount"];
 	var entry_count = 0;
diff --git a/src/zenserver/frontend/html/indexer/worker.js b/src/zenserver/frontend/html/indexer/worker.js
index b8183cc6f..25c8d7671 100644
--- a/src/zenserver/frontend/html/indexer/worker.js
+++ b/src/zenserver/frontend/html/indexer/worker.js
@@ -31,7 +31,7 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride)
 			.resource(uri)
 			.param("start", index)
 			.param("count", page_size)
-			.param("fieldfilter", "packagedata,key")
+			.param("fieldfilter", "packagedata,bulkdata,key")
 			.cbo()
 
 		const entry_count = Math.min(page_size, -(index - end));
@@ -66,34 +66,63 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride)
 
 			var key = undefined;
 			var pkg_data = undefined;
+			var bulk_data = undefined;
 			for (const field of entry)
 			{
-				if (field.is_named("key")) key = field;
+				if (field.is_named("key"))				key = field;
 				else if (field.is_named("packagedata")) pkg_data = field;
+				else if (field.is_named("bulkdata"))	bulk_data = field;
 			}
 			if (key == undefined || pkg_data == undefined)
 				continue;
 
 			var id = 0n;
-			for (var item of pkg_data.as_array())
+			var size = 0;
+			var raw_size = 0;
+			for (const item of pkg_data.as_array())
 			{
-				var pkg_id = item.as_object().find("id");
-				if (pkg_id == undefined)
+				var found = 0, pkg_id;
+				for (const field of item.as_object())
+				{
+					if (!id && field.is_named("id"))	pkg_id = field.as_value();
+					else if (field.is_named("size"))	size += field.as_value();
+					else if (field.is_named("rawsize"))	raw_size += field.as_value();
+					else continue;
+					if (found++ >= 3)
+						break;
+				}
+
+				if (pkg_id === undefined)
 					continue;
 
-				pkg_id = pkg_id.as_value().subarray(0, 8);
+				pkg_id = pkg_id.subarray(0, 8);
 				for (var i = 7; i >= 0; --i)
 				{
 					id <<= 8n;
 					id  |= BigInt(pkg_id[i]);
 				}
-				break;
+			}
+
+			if (bulk_data)
+			{
+				for (const item of bulk_data.as_array())
+				{
+					var found = 0;
+					for (const field of item.as_object())
+					{
+						if (field.is_named("size"))			size += field.as_value();
+						else if (field.is_named("rawsize"))	raw_size += field.as_value();
+						else continue;
+						if (found++ >= 2)
+							break;
+					}
+				}
 			}
 
 			if (id == 0)
 				continue;
 
-			result[count] = [id, key.as_value()];
+			result[count] = [id, key.as_value(), size, raw_size];
 			count++;
 		}
 
diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js
index b166d0a6f..a0e3ee915 100644
--- a/src/zenserver/frontend/html/pages/entry.js
+++ b/src/zenserver/frontend/html/pages/entry.js
@@ -4,6 +4,7 @@
 
 import { ZenPage } from "./page.js"
 import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
 import { Table, PropTable, Toolbar, ProgressBar } from "../util/widgets.js"
 import { create_indexer } from "../indexer/indexer.js"
 
@@ -84,7 +85,11 @@ export class Page extends ZenPage
 		// data
 		{
 			const sub_section = section.add_section("data");
-			const table = sub_section.add_widget(Table, ["name", "actions"], Table.Flag_PackRight);
+			const table = sub_section.add_widget(
+				Table,
+				["name", "size", "rawsize", "actions"], Table.Flag_PackRight
+			);
+			table.id("datatable");
 			for (const field_name of ["packagedata", "bulkdata"])
 			{
 				var pkg_data = entry.find(field_name);
@@ -93,12 +98,13 @@ export class Page extends ZenPage
 
 				for (const item of pkg_data.as_array())
 				{
-					var io_hash;
-					var file_name;
+					var io_hash, size, raw_size, file_name;
 					for (const field of item.as_object())
 					{
-						if (field.is_named("data")) io_hash = field.as_value();
-						else if (field.is_named("filename")) file_name = field.as_value();
+						if (field.is_named("data"))				io_hash = field.as_value();
+						else if (field.is_named("filename"))	file_name = field.as_value();
+						else if (field.is_named("size"))		size = field.as_value();
+						else if (field.is_named("rawsize"))		raw_size = field.as_value();
 					}
 
 					if (io_hash instanceof Uint8Array)
@@ -109,7 +115,10 @@ export class Page extends ZenPage
 						io_hash = ret;
 					}
 
-					const row = table.add_row(file_name);
+					size = (size !== undefined) ? Friendly.kib(size) : "";
+					raw_size = (raw_size !== undefined) ? Friendly.kib(raw_size) : "";
+
+					const row = table.add_row(file_name, size, raw_size);
 
 					const project = this.get_param("project");
 					const oplog = this.get_param("oplog");
diff --git a/src/zenserver/frontend/html/pages/tree.js b/src/zenserver/frontend/html/pages/tree.js
index c3cea0eb5..23ff3e819 100644
--- a/src/zenserver/frontend/html/pages/tree.js
+++ b/src/zenserver/frontend/html/pages/tree.js
@@ -3,6 +3,7 @@
 "use strict";
 
 import { ZenPage } from "./page.js"
+import { Friendly } from "../util/friendly.js"
 import { ProgressBar } from "../util/widgets.js"
 import { create_indexer } from "../indexer/indexer.js"
 
@@ -13,17 +14,31 @@ export class Page extends ZenPage
 	{
 		const project = this.get_param("project");
 		const oplog = this.get_param("oplog");
+		const sort_by = this.get_param("sort", -1);
 
 		this._indexer = this._load_indexer(project, oplog);
 
 		this.set_title("tree - " + oplog);
 		const section = this.add_section(project + " - " + oplog);
 
+		this._create_tree(section);
+		this._expand(this._root);
+		this._activate_sort_header(-1, sort_by);
+	}
+
+	_create_tree(section)
+	{
 		const list = section.tag().id("tree_root").tag("ul");
 		const root = list.tag("li");
 		root.attr("part", "/");
-		root.tag().text("/");
-		this._expand(root);
+		const header = root.tag();
+		header.tag().text("/");
+		this._sort_headers = [
+			header.tag().text("size").on_click(() => this._change_sort(0)),
+			header.tag().text("rawsize").on_click(() => this._change_sort(1)),
+			header.tag().text("count").on_click(() => this._change_sort(2)),
+		];
+		this._root = root;
 	}
 
 	async _load_indexer(project, oplog)
@@ -47,12 +62,11 @@ export class Page extends ZenPage
 			if (!item.is("li")) continue;
 			prefix = item.attr("part") + prefix;
 		}
-		console.log(prefix);
 
 		const indexer = await this._indexer;
 
 		const new_nodes = new Object();
-		for (var name of indexer.enum_names())
+		for (var [name, size, raw_size] of indexer.enum_all())
 		{
 			if (!name.startsWith(prefix))
 				continue;
@@ -62,19 +76,25 @@ export class Page extends ZenPage
 			if (slash != -1)
 				name = name.substr(0, slash + 1);
 
-			if (new_nodes[name] === undefined)
-				new_nodes[name] = 1;
+			if (new_nodes[name] !== undefined)
+			{
+				new_nodes[name][0] += size;
+				new_nodes[name][1] += raw_size;
+				new_nodes[name][2] += 1;
+			}
 			else
-				new_nodes[name] += 1;
+				new_nodes[name] = [size, raw_size, 1];
 		}
 
-		const by_count = this.get_param("bycount", 0)|0;
+		var sort_by = this.get_param("sort", -1)|0;
+		sort_by = Math.min(Math.max(sort_by, -1), 3);
+
 		const sorted_keys = Object.keys(new_nodes).sort((l, r) => {
 			const is_node_l = l.endsWith("/");
 			const any_nodes = is_node_l + r.endsWith("/");
-			if (any_nodes == 0)	return r < l;
 			if (any_nodes == 1)	return is_node_l ? -1 : 1;
-			return by_count ? (new_nodes[r] - new_nodes[l]) : (r < l);
+			if (sort_by >= 0) return (new_nodes[r][sort_by] - new_nodes[l][sort_by]);
+			return r < l;
 		})
 
 		const list = node.tag("ul");
@@ -83,10 +103,18 @@ export class Page extends ZenPage
 			const item = list.tag("li").attr("part", name);
 			const info = item.tag();
 			const label = info.tag().text(name);
+
+			for (var i = 0; i < 2; ++i)
+			{
+				const size = Friendly.kib(new_nodes[name][i]);
+				info.tag().text(size);
+			}
+
 			if (name.endsWith("/"))
 			{
+				const count = Friendly.sep(new_nodes[name][2]);
+				info.tag().text(count);
 				label.on_click((x) => this.expand_collapse(x), item);
-				info.tag().text(new_nodes[name]);
 				continue;
 			}
 
@@ -115,4 +143,35 @@ export class Page extends ZenPage
 			return this._expand(node);
 		return this._collapse(node);
 	}
+
+	_activate_sort_header(current, next)
+	{
+		const impl = (index, is_on) => {
+			if (index >= 0 && index < this._sort_headers.length)
+				this._sort_headers[index].attr("active", is_on ? "" : null);
+		};
+		impl(current, false);
+		impl(next, true);
+	}
+
+	_change_sort(sort_by)
+	{
+		const current = this.get_param("sort");
+		if (current == sort_by)
+			sort_by = -1;
+
+		this._activate_sort_header(current, sort_by);
+
+		this.set_param("sort", sort_by);
+
+		for (var node = this._root.first_child(); node;)
+		{
+			const next = node.next_sibling();
+			if (node.is("ul"))
+				node.destroy();
+			node = next;
+		}
+
+		this._expand(this._root);
+	}
 }
diff --git a/src/zenserver/frontend/html/util/component.js b/src/zenserver/frontend/html/util/component.js
index 39a9f2fe6..205aa038e 100644
--- a/src/zenserver/frontend/html/util/component.js
+++ b/src/zenserver/frontend/html/util/component.js
@@ -20,17 +20,20 @@ class ComponentBase
 
 	parent()
 	{
-		return this.new_component(this._element.parentElement);
+		const e = this._element.parentElement;
+		return e ? this.new_component(e) : null;
 	}
 
 	first_child()
 	{
-		return this.new_component(this._element.firstElementChild);
+		const e = this._element.firstElementChild;
+		return e ? this.new_component(e) : null;
 	}
 
 	next_sibling()
 	{
-		return this.new_component(this._element.nextElementSibling);
+		const e = this._element.nextElementSibling;
+		return e ? this.new_component(e) : null;
 	}
 
 	destroy()
diff --git a/src/zenserver/frontend/html/util/friendly.js b/src/zenserver/frontend/html/util/friendly.js
index 6eee3a5b8..b27721964 100644
--- a/src/zenserver/frontend/html/util/friendly.js
+++ b/src/zenserver/frontend/html/util/friendly.js
@@ -14,10 +14,10 @@ export class Friendly
 		});
 	}
 
-	static k(x)		{ return Friendly.sep((x + 999)  / Math.pow(10, 3), 0) + "K"; }
-	static m(x)		{ return Friendly.sep( x         / Math.pow(10, 6), 1) + "M"; }
-	static g(x)		{ return Friendly.sep( x         / Math.pow(10, 6), 2) + "G"; }
-	static kib(x)	{ return Friendly.sep((x + 1023) / (1 << 10), 0) + " KiB"; }
-	static mib(x)	{ return Friendly.sep( x         / (1 << 20), 1) + " MiB"; }
-	static gib(x)	{ return Friendly.sep( x         / (1 << 30), 2) + " GiB"; }
+	static k(x, p=0)	{ return Friendly.sep((x + 999)  / Math.pow(10, 3)|0, p) + "K"; }
+	static m(x, p=1)	{ return Friendly.sep( x         / Math.pow(10, 6),   p) + "M"; }
+	static g(x, p=2)	{ return Friendly.sep( x         / Math.pow(10, 9),   p) + "G"; }
+	static kib(x, p=0)	{ return Friendly.sep((x + 1023) / (1 << 10)|0, p) + " KiB"; }
+	static mib(x, p=1)	{ return Friendly.sep( x         / (1 << 20),   p) + " MiB"; }
+	static gib(x, p=2)	{ return Friendly.sep( x         / (1 << 30),   p) + " GiB"; }
 }
diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css
index 2463a6b5d..c9dc0d83e 100644
--- a/src/zenserver/frontend/html/zen.css
+++ b/src/zenserver/frontend/html/zen.css
@@ -405,17 +405,26 @@ a {
 	min-width:		15%;
 }
 
+/* entry -------------------------------------------------------------------- */
+
+#entry {
+	#datatable > div {
+		> div:nth-child(3), > div:nth-child(4) {
+			text-align: right;
+		}
+	}
+}
+
 /* tree --------------------------------------------------------------------- */
 
 #tree {
-	#tree_root {
-		margin-left:			5em;
-		margin-right:			10em;
+	#tree_root > ul {
+		margin-left:			0em;
 	}
 	ul {
 		list-style-type:		none;
 		padding-left:			0;
-		margin-left:			2em;
+		margin-left:			1em;
 	}
 	li > div {
 		display:				flex;
@@ -423,9 +432,20 @@ a {
 		padding-left:			0.3em;
 		padding-right:			0.3em;
 	}
-	li > div > div:last-child {
+	li > div > div[active] {
+		text-transform:			uppercase;
+	}
+	li > div > div:nth-last-child(3) {
 		margin-left:			auto;
 	}
+	li > div > div:nth-last-child(-n + 3) {
+		font-size:				0.8em;
+		width:					10em;
+		text-align:				right;
+	}
+	li > div > div:nth-last-child(1) {
+		width:					6em;
+	}
 	li > div:hover {
 		background-color:		var(--theme_p4);
 		border-bottom:			1px solid var(--theme_g2);