Self-hosted dashboard: Searchable oplog and links between oplog entry dependencies (#213)v5.5.12-pre0

* Consistent use of semicolons

* Added fallback if oplog entry assumptions do not hold

* 'marker' and 'expired' cells were incorrectly friendly

* Two spaces when there should only be one

* Robustness against .text(undefined) calls

* A single step into JavaScript modules

* Turned Fetcher into a module

* Friendly into a module

* Specialise Cbo field name comparison as TextDecoder() is very slow

* Prefer is_named() over get_name()

* Incorrect logic checking if a server reply was okay

* Try and make sure it's always numbers that flow through Friendly

* Added a progress bar component

* Swap key and package hash columns

* CbObject cloning

* Dark and light themes depending on browser settings

* Adjust styling of input boxes

* Add theme swatches to test page

* Turns out one can nest CSS selectors

* Separate swatch for links/actions

* Generate theme by lerping intermediate colours

* Clearer progress bar

* Chromium was complaining about label-less input elements

* Promise-based cache using an IndexedDb

* WebWorker for generating map of package ids to names

* Indexer class for building, loading, and saving map of ids to names

* Added links to oplog entries of an entry's dependencies

* This doesn't need to be decorated as async any longer

* Implemented oplog searching

* View and drop make no sense on package data payloads

* Rudimentary search result truncation

* Updated changelog

* Updated HTML zip archive

3 files changed, 389 insertions, 0 deletions

diff --git a/src/zenserver/frontend/html/indexer/cache.js b/src/zenserver/frontend/html/indexer/cache.js
new file mode 100644
index 000000000..390aa948d
--- /dev/null
+++ b/src/zenserver/frontend/html/indexer/cache.js
@@ -0,0 +1,65 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+////////////////////////////////////////////////////////////////////////////////
+export class Cache
+{
+	constructor(db_name, ...store_names)
+	{
+		this._db_name = db_name;
+		this._store_names = store_names;
+		this._version = 1;
+		this._db = this._open();
+	}
+
+	put(store_name, key, value)
+	{
+		const executor = async (resolve, reject) => {
+			const db = await this._db;
+			const transaction = db.transaction(store_name, "readwrite");
+			const store = transaction.objectStore(store_name);
+			const request = store.put(value, key);
+			request.onerror = (evt) => reject(Error("put transaction error"));
+			request.onsuccess = (evt) => resolve(true);
+		};
+		return new Promise(executor);
+	}
+
+	get(store_name, key)
+	{
+		const executor = async (resolve, reject) => {
+			const db = await this._db;
+			const transaction = db.transaction(store_name, "readonly");
+			const store = transaction.objectStore(store_name);
+			const request = store.get(key);
+			request.onerror = (evt) => reject(Error("get transaction error"));
+			request.onsuccess = (evt) => {
+				if (request.result)
+					resolve(request.result);
+				else
+					resolve(false);
+			};
+		};
+		return new Promise(executor);
+	}
+
+	_open()
+	{
+		const executor = (resolve, reject) => {
+			const request = indexedDB.open(this._db_name, this._version);
+			request.onerror = (evt) => reject(Error("Failed to open IndexedDb"));
+			request.onsuccess = (evt) => resolve(evt.target.result);
+			request.onupgradeneeded = (evt) => {
+				const db = evt.target.result;
+
+				for (const store_name of db.objectStoreNames)
+					db.deleteObjectStore(store_name)
+
+				for (const store_name of this._store_names)
+					db.createObjectStore(store_name);
+			};
+		};
+		return new Promise(executor);
+	}
+}
diff --git a/src/zenserver/frontend/html/indexer/indexer.js b/src/zenserver/frontend/html/indexer/indexer.js
new file mode 100644
index 000000000..8e5003edf
--- /dev/null
+++ b/src/zenserver/frontend/html/indexer/indexer.js
@@ -0,0 +1,193 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { Cache } from "./cache.js"
+import { Message } from "./worker.js"
+import { Fetcher } from "../util/fetcher.js"
+
+////////////////////////////////////////////////////////////////////////////////
+class Indexer
+{
+	constructor(pages)
+	{
+		this._pages = pages;
+	}
+
+	lookup_id(entry_id)
+	{
+		const bin_search = function(page) {
+			var l = 0;
+			var r = page.length;
+			while (l < r)
+			{
+				const mid = l + ((r - l) >> 1);
+				const d = entry_id - page[mid][0];
+				if (d < 0n)			r = mid;
+				else if (d > 0n)	l = mid + 1;
+				else				return mid;
+			}
+
+			return -1;
+		};
+
+		for (const page of this._pages)
+		{
+			const index = bin_search(page);
+			if (index >= 0)
+				return page[index][1];
+		}
+
+		return "";
+	}
+
+	*search(needle)
+	{
+		for (const page of this._pages)
+			for (const [_, name] of page)
+				if (name.indexOf(needle) >= 0)
+					yield name;
+	}
+}
+
+
+
+////////////////////////////////////////////////////////////////////////////////
+async function save(progress_cb, oplog_info, pages)
+{
+	const project_id = oplog_info["project"];
+	const cache = new Cache(project_id, "pages");
+
+	const page_count = pages.length;
+	const puts = new Array(page_count);
+	for (var i = 0; i < page_count; ++i)
+		puts[i] = cache.put("pages", i, pages[i]);
+
+	var okay = true
+	for (var i = 0; i < page_count; ++i)
+	{
+		okay &= await puts[i];
+		progress_cb("saving", i + 1, page_count);
+	}
+	if (!okay)
+		return false;
+
+	cache.put("pages", "$", {
+		"page_count"	: pages.length,
+		"total_size"	: oplog_info["totalsize"],
+		"op_count"		: oplog_info["opcount"],
+		"timestamp"		: (Date.now() / 1000) | 0,
+	});
+
+	return true
+}
+
+////////////////////////////////////////////////////////////////////////////////
+async function build(progress_cb, oplog_info)
+{
+	const project_id = oplog_info["project"];
+	const oplog = oplog_info["id"];
+	const init_msg = Message.create(Message.Init, project_id, oplog);
+
+	const worker_n = Math.min(navigator.hardwareConcurrency / 2,  6);
+	const page_size = 48 << 10;
+	const stride = page_size * worker_n;
+	const end = oplog_info["opcount"];
+	var entry_count = 0;
+
+	const pages = new Array();
+
+	const executor = function(index, resolve, reject) {
+		const worker = new Worker("indexer/worker.js", { type: "module" });
+		worker.onerror = (evt) => reject(Error("Worker error"));
+		worker.onmessage = (evt) => {
+			const [msg_id, ...params] = evt.data;
+			switch (msg_id)
+			{
+			case Message.MapDone:
+				resolve();
+				worker.terminate();
+				break;
+
+			case Message.MapPage:
+				const [page] = params;
+				pages.push(page);
+				entry_count += page.length;
+				progress_cb("parsing", entry_count, end);
+				break;
+			}
+		}
+		worker.postMessage(init_msg);
+
+		const start = page_size * index;
+		const map_msg = Message.create(Message.Map, start, end, page_size, stride);
+		worker.postMessage(map_msg);
+	};
+
+	const workers = []
+	for (var i = 0; i < worker_n; ++i)
+	{
+		const worker = new Promise((...args) => executor(i, ...args));
+		workers.push(worker);
+	}
+
+	for (const worker of workers)
+		await worker;
+
+	return pages;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+async function load(progress_cb, oplog_info)
+{
+	const project_id = oplog_info["project"];
+	const cache = new Cache(project_id, "pages");
+	const meta = await cache.get("pages", "$");
+
+	var hit = false;
+	if (meta)
+	{
+		const yesterday = (Date.now() / 1000) - (24 * 60 * 60);
+		hit  = true;
+		hit &= (meta["total_size"] == oplog_info["totalsize"]);
+		hit &= (meta["op_count"] == oplog_info["opcount"]);
+		hit &= (meta["timestamp"] >= yesterday);
+	}
+	if (!hit)
+		return null;
+
+	const page_count = meta["page_count"];
+	const gets = new Array(page_count);
+	const pages = new Array(page_count);
+	for (var i = 0; i < page_count; ++i)
+		gets[i] = cache.get("pages", i);
+
+	progress_cb("loading", 0, page_count);
+	for (var i = 0; i < page_count; ++i)
+	{
+		pages[i] = await gets[i];
+		progress_cb("loading", i + 1, page_count);
+	}
+
+	return pages;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+export async function create_indexer(project_id, oplog, progress_cb)
+{
+	if (!window.Worker)
+		throw Error("browser does not support web workers");
+
+	const oplog_info = await new Fetcher()
+		.resource("prj", project_id, "oplog", oplog)
+		.json();
+
+	var pages = await load(progress_cb, oplog_info);
+	if (!pages)
+	{
+		pages = await build(progress_cb, oplog_info);
+		await save(progress_cb, oplog_info, pages);
+	}
+
+	return new Indexer(pages);
+}
diff --git a/src/zenserver/frontend/html/indexer/worker.js b/src/zenserver/frontend/html/indexer/worker.js
new file mode 100644
index 000000000..581746d6c
--- /dev/null
+++ b/src/zenserver/frontend/html/indexer/worker.js
@@ -0,0 +1,131 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { Fetcher } from "../util/fetcher.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Message
+{
+	static None		= 0;	//
+	static Init		= 1;	// project_id, oplog
+	static Map		= 2;	// start, end, page_size, stride
+	static MapPage	= 3;	// page
+	static MapDone	= 4;	//
+
+	static create(msg, ...args) { return [msg, ...args]; }
+}
+
+
+
+////////////////////////////////////////////////////////////////////////////////
+async function map_id_to_key(project_id, oplog, start, end, page_size, stride)
+{
+	const uri = "/prj/" + project_id + "/oplog/" + oplog + "/entries";
+	while (start < end)
+	{
+		performance.mark("fetch");
+		const cbo = new Fetcher()
+			.resource(uri)
+			.param("start", start)
+			.param("count", page_size)
+			.param("fieldfilter", "packagedata,key")
+			.cbo()
+
+		const entry_count = Math.min(page_size, -(start - end));
+		var result = new Array(entry_count);
+
+		var entries = (await cbo).as_object().find("entries");
+		if (entries == undefined)
+			break;
+
+		entries = entries.as_array();
+		if (entries.num() == 0)
+			break;
+
+		performance.mark("build");
+		var count = 0;
+		for (var entry of entries)
+		{
+			if (!entry.is_object())
+				continue
+			entry = entry.as_object();
+
+			var key = undefined;
+			var pkg_data = undefined;
+			for (const field of entry)
+			{
+				if (field.is_named("key")) key = field;
+				else if (field.is_named("packagedata")) pkg_data = field;
+			}
+			if (key == undefined || pkg_data == undefined)
+				continue;
+
+			var id = 0n;
+			for (var item of pkg_data.as_array())
+			{
+				var pkg_id = item.as_object().find("id");
+				if (pkg_id == undefined)
+					continue;
+
+				pkg_id = pkg_id.as_value().subarray(0, 8);
+				for (var i = 7; i >= 0; --i)
+				{
+					id <<= 8n;
+					id  |= BigInt(pkg_id[i]);
+				}
+				break;
+			}
+
+			if (id == 0)
+				continue;
+
+			result[count] = [id, key.as_value()];
+			count++;
+		}
+
+		start += stride;
+
+		if (count == 0)
+			continue;
+
+		if (count != result.length)
+			result = result.slice(0, count);
+
+		performance.mark("sort");
+		result.sort(function(l, r) { return Number(l[0] - r[0]); });
+
+		const msg = Message.create(Message.MapPage, result);
+		postMessage(msg);
+	}
+
+	postMessage(Message.create(Message.MapDone));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+function worker_scope()
+{
+	var project_id;
+	var oplog;
+
+	return (evt) => {
+		const [msg_id, ...params] = evt.data;
+		switch (msg_id)
+		{
+		case Message.Init:
+			[project_id, oplog] = params;
+			break;
+
+		case Message.Map:
+			var [start, end, page_size, stride] = params;
+			map_id_to_key(project_id, oplog, start, end, page_size, stride);
+			break;
+		}
+	}
+}
+
+////////////////////////////////////////////////////////////////////////////////
+if (typeof DedicatedWorkerGlobalScope != "undefined" && self instanceof DedicatedWorkerGlobalScope)
+{
+	onmessage = worker_scope();
+}