From 05d1044045539557dfe4e9c8996737d83f9dee89 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Mon, 11 Nov 2024 10:31:34 +0100 Subject: Self-hosted dashboard: Searchable oplog and links between oplog entry dependencies (#213) * Consistent use of semicolons * Added fallback if oplog entry assumptions do not hold * 'marker' and 'expired' cells were incorrectly friendly * Two spaces when there should only be one * Robustness against .text(undefined) calls * A single step into JavaScript modules * Turned Fetcher into a module * Friendly into a module * Specialise Cbo field name comparison as TextDecoder() is very slow * Prefer is_named() over get_name() * Incorrect logic checking if a server reply was okay * Try and make sure it's always numbers that flow through Friendly * Added a progress bar component * Swap key and package hash columns * CbObject cloning * Dark and light themes depending on browser settings * Adjust styling of input boxes * Add theme swatches to test page * Turns out one can nest CSS selectors * Separate swatch for links/actions * Generate theme by lerping intermediate colours * Clearer progress bar * Chromium was complaining about label-less input elements * Promise-based cache using an IndexedDb * WebWorker for generating map of package ids to names * Indexer class for building, loading, and saving map of ids to names * Added links to oplog entries of an entry's dependencies * This doesn't need to be decorated as async any longer * Implemented oplog searching * View and drop make no sense on package data payloads * Rudimentary search result truncation * Updated changelog * Updated HTML zip archive --- src/zenserver/frontend/html/util/compactbinary.js | 464 ++++++++++++++++++++++ 1 file changed, 464 insertions(+) create mode 100644 src/zenserver/frontend/html/util/compactbinary.js (limited to 'src/zenserver/frontend/html/util/compactbinary.js') diff --git a/src/zenserver/frontend/html/util/compactbinary.js b/src/zenserver/frontend/html/util/compactbinary.js new file mode 100644 index 000000000..366ec6aff --- /dev/null +++ b/src/zenserver/frontend/html/util/compactbinary.js @@ -0,0 +1,464 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +//////////////////////////////////////////////////////////////////////////////// +class VarInt +{ +} + +//////////////////////////////////////////////////////////////////////////////// +VarInt.measure = function(data_view) +{ + var value = data_view[0]; + var ret = 1; + for (; value & 0x80; value <<= 1, ++ret); + return ret; +} + +//////////////////////////////////////////////////////////////////////////////// +VarInt.read_uint = function(data_view, return_type=Number) +{ + const length = VarInt.measure(data_view); + var value = return_type(data_view[0] & (0xff >> length)); + for (var i = 1; i < length; ++i) + { + value <<= return_type(8); + value |= return_type(data_view[i]); + } + return [value, length]; +} + +//////////////////////////////////////////////////////////////////////////////// +VarInt.read_int = function(data_view, return_type=Number) +{ + var [value, length] = VarInt.read_uint(data_view, return_type); + value = -(value & return_type(1)) ^ (value >> return_type(1)); + return [value, length]; +} + + + +//////////////////////////////////////////////////////////////////////////////// +function cb_assert(expr_result) +{ + if (Boolean(expr_result) == false) + throw Error("compactbinary error"); +} + + + +//////////////////////////////////////////////////////////////////////////////// +const CbFieldType = { + None : 0x00, + Null : 0x01, + Object : 0x02, + UniformObject : 0x03, + Array : 0x04, + UniformArray : 0x05, + Binary : 0x06, + String : 0x07, + IntegerPositive : 0x08, + IntegerNegative : 0x09, + Float32 : 0x0a, + Float64 : 0x0b, + BoolFalse : 0x0c, + BoolTrue : 0x0d, + ObjectAttachment : 0x0e, + BinaryAttachment : 0x0f, + Hash : 0x10, + Uuid : 0x11, + DateTime : 0x12, + TimeSpan : 0x13, + ObjectId : 0x14, + CustomById : 0x1e, + CustomByName : 0x1f, + Reserved : 0x20, + HasFieldType : 0x40, + HasFieldName : 0x80, +} + +//////////////////////////////////////////////////////////////////////////////// +class CbFieldTypeOps +{ + static SerializedTypeMask = 0b10111111; + static TypeMask = 0b00111111; + static ObjectMask = 0b00111110; + static ObjectBase = 0b00000010; + static ArrayMask = 0b00111110; + static ArrayBase = 0b00000100; + static IntegerMask = 0b00111110; + static IntegerBase = 0b00001000; + static FloatMask = 0b00111100; + static FloatBase = 0b00001000; + static BoolMask = 0b00111110; + static BoolBase = 0b00001100; + static AttachmentMask = 0b00111110; + static AttachmentBase = 0b00001110; + + static get_type(type) { return type & CbFieldTypeOps.TypeMask; } + static get_serialized_type(type) { return type & CbFieldTypeOps.SerializedTypeMask; } + static has_field_type(type) { return (type & CbFieldType.HasFieldType) != 0; } + static has_field_name(type) { return (type & CbFieldType.HasFieldName) != 0; } + static is_none(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.None; } + static is_null(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.Null; } + static is_object(type) { return (type & CbFieldTypeOps.ObjectMask) == CbFieldTypeOps.ObjectBase; } + static is_array(type) { return (type & CbFieldTypeOps.ArrayMask) == CbFieldTypeOps.ArrayBase; } + static is_binary(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.Binary; } + static is_string(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.String; } + static is_integer(type) { return (type & CbFieldTypeOps.IntegerMask) == CbFieldTypeOps.IntegerBase; } + static is_float(type) { return (type & CbFieldTypeOps.FloatMask) == CbFieldTypeOps.FloatBase; } + static is_bool(type) { return (type & CbFieldTypeOps.BoolMask) == CbFieldTypeOps.BoolBase; } + static is_object_attachment(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.ObjectAttachment; } + static is_binary_attachment(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.BinaryAttachment; } + static is_attachment(type) { return (type & CbFieldTypeOps.AttachmentMask) == CbFieldTypeOps.AttachmentBase; } + static is_uuid(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.Uuid; } + static is_object_id(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.ObjectId; } + static is_custom_by_id(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.CustomById; } + static is_custom_by_name(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.CustomByName; } + static is_date_time(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.DateTime; } + static is_time_span(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.TimeSpan; } + static is_hash(type) { var t = CbFieldTypeOps.get_type(type); return t >= CbFieldType.ObjectAttachment && t <= CbFieldType.Hash; } + static may_contain_attachments(type){ var t = CbFieldTypeOps.get_type(type); return is_object(t) || is_array(t) || is_attachement(t); } +} + + + +//////////////////////////////////////////////////////////////////////////////// +class CbFieldView +{ + constructor() + { + this._type = CbFieldType.None; + this._name = undefined; + this._data_view = undefined; + } +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype._from_field = function(field) +{ + this._type = field._type; + this._name = field._name; + this._data_view = field._data_view; + return this; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype._from_data = function(data_view, type=CbFieldType.HasFieldType) +{ + if (CbFieldTypeOps.has_field_type(type)) + { + type = data_view[0] | CbFieldType.HasFieldType; + data_view = data_view.subarray(1); + } + + if (CbFieldTypeOps.has_field_name(type)) + { + const [n, varint_len] = VarInt.read_uint(data_view); + this._name = data_view.subarray(varint_len, n + varint_len); + data_view = data_view.subarray(n + varint_len); + } + + this._type = type; + this._data_view = data_view; + return this; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView._iterate = function*(data_view, uniform_type) +{ + while (data_view.length > 0) + { + const field = new CbFieldView()._from_data(data_view, uniform_type); + yield field; + + const field_size = field.get_payload_size(); + cb_assert(field_size <= data_view.length); + data_view = field.get_payload().subarray(field_size); + } +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.get_type = function() +{ + return this._type; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.get_name = function() +{ + return new TextDecoder().decode(this._name); +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.is_named = function(rhs) +{ + if (!this._name) return false; + if (rhs.length != this._name.length) return false; + for (var i = 0; i < rhs.length; ++i) + if (rhs.charCodeAt(i) != this._name[i]) + return false; + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.get_payload = function() +{ + return this._data_view; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.get_payload_size = function() +{ + switch (CbFieldTypeOps.get_type(this.get_type())) + { + case CbFieldType.None: + case CbFieldType.Null: + return 0; + case CbFieldType.Object: + case CbFieldType.UniformObject: + case CbFieldType.Array: + case CbFieldType.UniformArray: + case CbFieldType.Binary: + case CbFieldType.String: + const [value, varint_len] = VarInt.read_uint(this._data_view); + return value + varint_len; + case CbFieldType.IntegerPositive: + case CbFieldType.IntegerNegative: + return VarInt.measure(this._data_view); + case CbFieldType.Float32: + return 4; + case CbFieldType.Float64: + return 8; + case CbFieldType.BoolFalse: + case CbFieldType.BoolTrue: + return 0; + case CbFieldType.ObjectAttachment: + case CbFieldType.BinaryAttachment: + case CbFieldType.Hash: + return 20; + case CbFieldType.Uuid: + return 16; + case CbFieldType.ObjectId: + return 12; + case CbFieldType.DateTime: + case CbFieldType.TimeSpan: + return 8; + } + return 0; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype._is = function(func) { return func(this.get_type()); } +CbFieldView.prototype.is_null = function() { return this._is(CbFieldTypeOps.is_null); } +CbFieldView.prototype.is_object = function() { return this._is(CbFieldTypeOps.is_object); } +CbFieldView.prototype.is_array = function() { return this._is(CbFieldTypeOps.is_array); } +CbFieldView.prototype.is_binary = function() { return this._is(CbFieldTypeOps.is_binary); } +CbFieldView.prototype.is_string = function() { return this._is(CbFieldTypeOps.is_string); } +CbFieldView.prototype.is_integer = function() { return this._is(CbFieldTypeOps.is_integer); } +CbFieldView.prototype.is_float = function() { return this._is(CbFieldTypeOps.is_float); } +CbFieldView.prototype.is_bool = function() { return this._is(CbFieldTypeOps.is_bool); } +CbFieldView.prototype.is_object_attachment = function() { return this._is(CbFieldTypeOps.is_object_attachment); } +CbFieldView.prototype.is_binary_attachment = function() { return this._is(CbFieldTypeOps.is_binary_attachment); } +CbFieldView.prototype.is_attachment = function() { return this._is(CbFieldTypeOps.is_attachment); } +CbFieldView.prototype.is_hash = function() { return this._is(CbFieldTypeOps.is_hash); } +CbFieldView.prototype.is_uuid = function() { return this._is(CbFieldTypeOps.is_uuid); } +CbFieldView.prototype.is_object_id = function() { return this._is(CbFieldTypeOps.is_object_id); } +CbFieldView.prototype.is_date_time = function() { return this._is(CbFieldTypeOps.is_date_time); } +CbFieldView.prototype.is_time_span = function() { return this._is(CbFieldTypeOps.is_time_span); } + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.as_object = function() +{ + cb_assert(CbFieldTypeOps.is_object(this.get_type())); + return new CbObjectView()._from_field(this); +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.as_array = function() +{ + cb_assert(CbFieldTypeOps.is_array(this.get_type())); + return new CbArrayView()._from_field(this); +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.as_value = function(int_type=Number) +{ + switch (CbFieldTypeOps.get_type(this.get_type())) + { + case CbFieldType.None: return undefined; + case CbFieldType.Null: return null; + + case CbFieldType.Object: + case CbFieldType.UniformObject: return this.as_object(); + + case CbFieldType.Array: + case CbFieldType.UniformArray: return this.as_array(); + + case CbFieldType.Binary: { + const [n, vn] = VarInt.read_uint(this._data_view); + return this._data_view.subarray(vn, n + vn); + } + + case CbFieldType.String: { + const [n, vn] = VarInt.read_uint(this._data_view); + return new TextDecoder().decode(this._data_view.subarray(vn, n + vn)); + } + + case CbFieldType.IntegerPositive: return VarInt.read_uint(this._data_view, int_type)[0]; + case CbFieldType.IntegerNegative: return VarInt.read_int(this._data_view, int_type)[0]; + + case CbFieldType.Float32: return new DataView(this._data_view.subarray(0, 4)).getFloat32(0, false); + case CbFieldType.Float64: return new DataView(this._data_view.subarray(0, 8)).getFloat64(0, false); + case CbFieldType.BoolFalse: return false; + case CbFieldType.BoolTrue: return true; + + case CbFieldType.ObjectAttachment: + case CbFieldType.BinaryAttachment: + case CbFieldType.Hash: return this._data_view.subarray(0, 20); + + case CbFieldType.Uuid: return this._data_view.subarray(0, 16); + case CbFieldType.ObjectId: return this._data_view.subarray(0, 12); + + case CbFieldType.DateTime: + case CbFieldType.TimeSpan: return this._data_view.subarray(0, 8); + } + + cb_assert(false); +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.clone = function() +{ + const ret = new CbFieldView() + ret._type = this._type; + ret._name = ret._name; + ret._data_view = new Uint8Array(this._data_view); + return ret; +} + + + +//////////////////////////////////////////////////////////////////////////////// +class CbObjectView extends CbFieldView +{ +} + +//////////////////////////////////////////////////////////////////////////////// +CbObjectView.prototype[Symbol.iterator] = function() +{ + var data_view = this.get_payload(); + + const [payload_size, varint_len] = VarInt.read_uint(data_view); + if (payload_size == 0) + return {}; + data_view = data_view.subarray(varint_len, payload_size + varint_len); + + var uniform_type = CbFieldType.HasFieldType; + if (CbFieldTypeOps.get_type(this.get_type()) == CbFieldType.UniformObject) + { + uniform_type = data_view[0]; + data_view = data_view.subarray(1); + } + + return CbFieldView._iterate(data_view, uniform_type); +} + +//////////////////////////////////////////////////////////////////////////////// +CbObjectView.prototype.to_js_object = function() +{ + const impl = function(node) + { + if (node.is_object()) + { + const ret = {}; + for (var item of node.as_object()) + ret[item.get_name()] = impl(item); + return ret; + } + + if (node.is_array()) + { + const ret = []; + for (var item of node.as_array()) + ret.push(impl(item)); + return ret; + } + + if (node.is_string()) return node.as_value(); + if (node.is_float()) return node.as_value(); + if (node.is_integer()) return node.as_value(); + + var ret = node.as_value(); + if (ret instanceof Uint8Array) + { + ret = ""; + for (var x of node.as_value()) + ret += x.toString(16).padStart(2, "0"); + } + return ret; + }; + + return impl(this); +} + +//////////////////////////////////////////////////////////////////////////////// +CbObjectView.prototype.find = function(name) +{ + for (const field of this) + if (field.is_named(name)) + return field; +} + + + +//////////////////////////////////////////////////////////////////////////////// +class CbArrayView extends CbFieldView +{ +} + +//////////////////////////////////////////////////////////////////////////////// +CbArrayView.prototype[Symbol.iterator] = function() +{ + var data_view = this.get_payload(); + + const [payload_size, varint_len] = VarInt.read_uint(data_view); + data_view = data_view.subarray(varint_len, payload_size + varint_len); + + const item_count_bytes = VarInt.measure(data_view); + if (item_count_bytes >= payload_size) + return {}; + data_view = data_view.subarray(item_count_bytes); + + var uniform_type = CbFieldType.HasFieldType; + if (CbFieldTypeOps.get_type(this.get_type()) == CbFieldType.UniformArray) + { + uniform_type = data_view[0]; + data_view = data_view.subarray(1); + } + + return CbFieldView._iterate(data_view, uniform_type); +} + +//////////////////////////////////////////////////////////////////////////////// +CbArrayView.prototype.num = function() +{ + var data_view = this._data_view; + const [n, n_len] = VarInt.read_uint(data_view); + data_view = data_view.subarray(n_len); + return VarInt.read_uint(data_view)[0]; +} + + + +//////////////////////////////////////////////////////////////////////////////// +export class CbObject extends CbFieldView +{ + constructor(uint8_array) + { + super(); + this._from_data(uint8_array); + } +} -- cgit v1.2.3 From afda5c6345f4c0b274ae8084bfbd371169791c57 Mon Sep 17 00:00:00 2001 From: zousar Date: Fri, 11 Apr 2025 00:25:20 -0600 Subject: Avoid signed overflow using BigInt Bias for use of BigInt when consuming integer fields in compact binary to avoid values showing up as negative due to overflow on the Number type. --- src/zenserver/frontend/html/util/compactbinary.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/zenserver/frontend/html/util/compactbinary.js') diff --git a/src/zenserver/frontend/html/util/compactbinary.js b/src/zenserver/frontend/html/util/compactbinary.js index 366ec6aff..90e4249f6 100644 --- a/src/zenserver/frontend/html/util/compactbinary.js +++ b/src/zenserver/frontend/html/util/compactbinary.js @@ -284,7 +284,7 @@ CbFieldView.prototype.as_array = function() } //////////////////////////////////////////////////////////////////////////////// -CbFieldView.prototype.as_value = function(int_type=Number) +CbFieldView.prototype.as_value = function(int_type=BigInt) { switch (CbFieldTypeOps.get_type(this.get_type())) { @@ -388,8 +388,8 @@ CbObjectView.prototype.to_js_object = function() } if (node.is_string()) return node.as_value(); - if (node.is_float()) return node.as_value(); if (node.is_integer()) return node.as_value(); + if (node.is_float()) return node.as_value(); var ret = node.as_value(); if (ret instanceof Uint8Array) -- cgit v1.2.3