Merge branch 'main' into zs/zencli-list-namespaces-buckets

7 files changed, 129 insertions, 79 deletions

diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 31c104110..e81e8eb54 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -15,12 +15,14 @@
 #include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zenhttp/zenhttp.h>
+#include <zenutil/commandlineoptions.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/format.h>
 #include <fmt/ranges.h>
 #include <zencore/logging.h>
 #include <cxxopts.hpp>
+#include <json11.hpp>
 #include <sol/sol.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
@@ -178,27 +180,6 @@ ParseBucketConfigs(std::span<std::string> Buckets)
 	return Cfg;
 }
 
-static std::string
-MakeSafePath(const std::string_view Path)
-{
-#if ZEN_PLATFORM_WINDOWS
-	if (Path.empty())
-	{
-		return std::string(Path);
-	}
-
-	std::string FixedPath(Path);
-	std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\');
-	if (!FixedPath.starts_with("\\\\?\\"))
-	{
-		FixedPath.insert(0, "\\\\?\\");
-	}
-	return FixedPath;
-#else
-	return std::string(Path);
-#endif
-};
-
 class CachePolicyOption : public LuaConfig::OptionValue
 {
 public:
@@ -324,7 +305,7 @@ public:
 					std::string Name	  = Bucket.value().get_or("name", std::string("Default"));
 					std::string Directory = Bucket.value().get_or("directory", std::string());
 
-					Value.Buckets.push_back({.Name = std::move(Name), .Directory = LuaConfig::MakeSafePath(Directory)});
+					Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)});
 				}
 			}
 		}
@@ -369,6 +350,7 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
 	LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
 	LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv);
+	LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv);
 	LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv);
 	LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv);
 	LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "quiet"sv);
@@ -379,7 +361,7 @@ ParseConfigFile(const std::filesystem::path& Path,
 
 	////// buildsstore
 	LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv);
-	LuaOptions.AddOption("buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit");
+	LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit");
 
 	////// network
 	LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv);
@@ -525,7 +507,7 @@ ParseConfigFile(const std::filesystem::path& Path,
 
 	if (!OutputConfigFile.empty())
 	{
-		std::filesystem::path			  WritePath(MakeSafePath(OutputConfigFile));
+		std::filesystem::path			  WritePath(MakeSafeAbsolutePath(OutputConfigFile));
 		zen::ExtendableStringBuilder<512> ConfigStringBuilder;
 		LuaOptions.Print(ConfigStringBuilder, CmdLineResult);
 		zen::BasicFile Output;
@@ -535,6 +517,68 @@ ParseConfigFile(const std::filesystem::path& Path,
 }
 
 void
+ParsePluginsConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, int BasePort)
+{
+	using namespace std::literals;
+
+	IoBuffer	 Body = IoBufferBuilder::MakeFromFile(Path);
+	std::string	 JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize());
+	std::string	 JsonError;
+	json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError);
+	if (!JsonError.empty())
+	{
+		throw std::runtime_error(fmt::format("failed parsing json file '{}'. Reason: '{}'", Path, JsonError));
+	}
+	for (const json11::Json& PluginInfo : PluginsInfo.array_items())
+	{
+		if (!PluginInfo.is_object())
+		{
+			throw std::runtime_error(fmt::format("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'",
+												 Path,
+												 PluginInfo.dump()));
+		}
+
+		HttpServerPluginConfig Config = {};
+
+		bool bNeedsPort = true;
+
+		for (const std::pair<const std::string, json11::Json>& Items : PluginInfo.object_items())
+		{
+			if (!Items.second.is_string())
+			{
+				throw std::runtime_error(
+					fmt::format("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'",
+								Path,
+								Items.second.dump()));
+			}
+
+			const std::string& Name	 = Items.first;
+			const std::string& Value = Items.second.string_value();
+
+			if (Name == "name"sv)
+				Config.PluginName = Value;
+			else
+			{
+				Config.PluginOptions.push_back({Name, Value});
+
+				if (Name == "port"sv)
+				{
+					bNeedsPort = false;
+				}
+			}
+		}
+
+		// add a default base port in case if json config didn't provide one
+		if (bNeedsPort)
+		{
+			Config.PluginOptions.push_back({"port", std::to_string(BasePort)});
+		}
+
+		ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config);
+	}
+}
+
+void
 ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 {
 	const char* DefaultHttp = "asio";
@@ -566,6 +610,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	std::string ContentDir;
 	std::string AbsLogFile;
 	std::string ConfigFile;
+	std::string PluginsConfigFile;
 	std::string OutputConfigFile;
 	std::string BaseSnapshotDir;
 
@@ -593,6 +638,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 						  "Exit immediately after initialization is complete",
 						  cxxopts::value<bool>(ServerOptions.IsPowerCycle));
 	options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
+	options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value<std::string>(PluginsConfigFile));
 	options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile));
 	options.add_options()("no-sentry",
 						  "Disable Sentry crash handler",
@@ -1110,12 +1156,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 		}
 		logging::RefreshLogLevels();
 
-		ServerOptions.SystemRootDir					  = MakeSafePath(SystemRootDir);
-		ServerOptions.DataDir						  = MakeSafePath(DataDir);
-		ServerOptions.BaseSnapshotDir				  = MakeSafePath(BaseSnapshotDir);
-		ServerOptions.ContentDir					  = MakeSafePath(ContentDir);
-		ServerOptions.AbsLogFile					  = MakeSafePath(AbsLogFile);
-		ServerOptions.ConfigFile					  = MakeSafePath(ConfigFile);
+		ServerOptions.SystemRootDir					  = MakeSafeAbsolutePath(SystemRootDir);
+		ServerOptions.DataDir						  = MakeSafeAbsolutePath(DataDir);
+		ServerOptions.BaseSnapshotDir				  = MakeSafeAbsolutePath(BaseSnapshotDir);
+		ServerOptions.ContentDir					  = MakeSafeAbsolutePath(ContentDir);
+		ServerOptions.AbsLogFile					  = MakeSafeAbsolutePath(AbsLogFile);
+		ServerOptions.ConfigFile					  = MakeSafeAbsolutePath(ConfigFile);
+		ServerOptions.PluginsConfigFile				  = MakeSafeAbsolutePath(PluginsConfigFile);
 		ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions);
 
 		if (!BaseSnapshotDir.empty())
@@ -1149,6 +1196,11 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 			ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, OutputConfigFile);
 		}
 
+		if (!ServerOptions.PluginsConfigFile.empty())
+		{
+			ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort);
+		}
+
 		ValidateOptions(ServerOptions);
 	}
 	catch (const zen::OptionParseException& e)
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 6bd7aa357..bd277cd88 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -165,6 +165,7 @@ struct ZenServerOptions
 	std::filesystem::path	 ContentDir;				  // Root directory for serving frontend content (experimental)
 	std::filesystem::path	 AbsLogFile;				  // Absolute path to main log file
 	std::filesystem::path	 ConfigFile;				  // Path to Lua config file
+	std::filesystem::path	 PluginsConfigFile;			  // Path to plugins config file
 	std::filesystem::path	 BaseSnapshotDir;			  // Path to server state snapshot (will be copied into data dir on start)
 	std::string				 ChildId;					  // Id assigned by parent process (used for lifetime management)
 	std::string				 LogId;						  // Id for tagging log output
diff --git a/src/zenserver/config/luaconfig.cpp b/src/zenserver/config/luaconfig.cpp
index f742fa34a..2c54de29e 100644
--- a/src/zenserver/config/luaconfig.cpp
+++ b/src/zenserver/config/luaconfig.cpp
@@ -4,27 +4,6 @@
 
 namespace zen::LuaConfig {
 
-std::string
-MakeSafePath(const std::string_view Path)
-{
-#if ZEN_PLATFORM_WINDOWS
-	if (Path.empty())
-	{
-		return std::string(Path);
-	}
-
-	std::string FixedPath(Path);
-	std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\');
-	if (!FixedPath.starts_with("\\\\?\\"))
-	{
-		FixedPath.insert(0, "\\\\?\\");
-	}
-	return FixedPath;
-#else
-	return std::string(Path);
-#endif
-};
-
 void
 EscapeBackslash(std::string& InOutString)
 {
@@ -101,7 +80,7 @@ FilePathOption::Parse(sol::object Object)
 	std::string Str = Object.as<std::string>();
 	if (!Str.empty())
 	{
-		Value = MakeSafePath(Str);
+		Value = MakeSafeAbsolutePath(Str);
 	}
 }
 
diff --git a/src/zenserver/config/luaconfig.h b/src/zenserver/config/luaconfig.h
index 76b3088a3..ce7013a9a 100644
--- a/src/zenserver/config/luaconfig.h
+++ b/src/zenserver/config/luaconfig.h
@@ -4,10 +4,10 @@
 
 #include <zenbase/concepts.h>
 #include <zencore/fmtutils.h>
+#include <zenutil/commandlineoptions.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/format.h>
-#include <cxxopts.hpp>
 #include <sol/sol.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
@@ -20,8 +20,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen::LuaConfig {
 
-std::string MakeSafePath(const std::string_view Path);
-void		EscapeBackslash(std::string& InOutString);
+void EscapeBackslash(std::string& InOutString);
 
 class OptionValue
 {
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index c28404da6..eadfcf84c 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
diff --git a/src/zenserver/frontend/html/indexer/indexer.js b/src/zenserver/frontend/html/indexer/indexer.js
index 16b91e130..688bc71b0 100644
--- a/src/zenserver/frontend/html/indexer/indexer.js
+++ b/src/zenserver/frontend/html/indexer/indexer.js
@@ -61,7 +61,7 @@ class Indexer
 	{
 		for (const page of this._pages)
 			for (const [_, name, size, raw_size] of page)
-				yield [name, size|0, raw_size|0];
+				yield [name, size|0n, raw_size|0n];
 	}
 }
 
diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js
index 41fc47218..54fb11c18 100644
--- a/src/zenserver/frontend/html/pages/entry.js
+++ b/src/zenserver/frontend/html/pages/entry.js
@@ -79,10 +79,32 @@ export class Page extends ZenPage
 	async _build_meta(section, entry)
 	{
 		var tree = {}
-		const cookart = this._find_iohash_field(entry, "CookPackageArtifacts");
-		if (cookart != null)
+
+		for (const field of entry)
 		{
-			tree["cook"] = { CookPackageArtifacts: cookart};
+			var visibleKey = undefined;
+			const name = field.get_name();
+			if (name == "CookPackageArtifacts")
+			{
+				visibleKey = name;
+			}
+			else if (name.startsWith("meta."))
+			{
+				visibleKey = name.slice(5);
+			}
+
+			if (visibleKey != undefined)
+			{
+				var found_value = field.as_value();
+				if (found_value instanceof Uint8Array)
+				{
+					var ret = "";
+					for (var x of found_value)
+						ret += x.toString(16).padStart(2, "0");
+					tree[visibleKey] = ret;
+				}
+			}
+
 		}
 
 		if (Object.keys(tree).length == 0)
@@ -90,28 +112,25 @@ export class Page extends ZenPage
 
 		const sub_section = section.add_section("meta");
 
-		for (const cat_name in tree)
+		const table = sub_section.add_widget(
+			Table,
+			["name", "actions"], Table.Flag_PackRight
+		);
+		for (const key in tree)
 		{
-			const cat_section = sub_section.add_section(cat_name);
-			const table = cat_section.add_widget(
-				Table,
-				["name", "actions"], Table.Flag_PackRight
+			const row = table.add_row(key);
+			const value = tree[key];
+
+			const project = this.get_param("project");
+			const oplog = this.get_param("oplog");
+			const link = row.get_cell(0).link(
+				"/" + ["prj", project, "oplog", oplog, value+".json"].join("/")
 			);
-			Object.entries(tree[cat_name]).forEach(([key, value]) =>
-			{
-				const row = table.add_row(key);
-
-				const project = this.get_param("project");
-				const oplog = this.get_param("oplog");
-				const link = row.get_cell(0).link(
-					"/" + ["prj", project, "oplog", oplog, value+".json"].join("/")
-				);
-
-				const action_tb = new Toolbar(row.get_cell(-1), true);
-				action_tb.left().add("copy-hash").on_click(async (v) => {
-					await navigator.clipboard.writeText(v);
-				}, value);
-			});
+
+			const action_tb = new Toolbar(row.get_cell(-1), true);
+			action_tb.left().add("copy-hash").on_click(async (v) => {
+				await navigator.clipboard.writeText(v);
+			}, value);
 		}
 	}