compute orchestration (#763)

- Added local process runners for Linux/Wine, Mac with some sandboxing support
- Horde & Nomad provisioning for development and testing
- Client session queues with lifecycle management (active/draining/cancelled), automatic retry with configurable limits, and manual reschedule API
- Improved web UI for orchestrator, compute, and hub dashboards with WebSocket push updates
- Some security hardening
- Improved scalability and `zen exec` command

Still experimental - compute support is disabled by default

1 files changed, 257 insertions, 0 deletions

diff --git a/src/zenserver-test/logging-tests.cpp b/src/zenserver-test/logging-tests.cpp
new file mode 100644
index 000000000..fe39e14c0
--- /dev/null
+++ b/src/zenserver-test/logging-tests.cpp
@@ -0,0 +1,257 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/zencore.h>
+
+#if ZEN_WITH_TESTS
+
+#	include "zenserver-test.h"
+
+#	include <zencore/filesystem.h>
+#	include <zencore/logging.h>
+#	include <zencore/testing.h>
+#	include <zenutil/zenserverprocess.h>
+
+namespace zen::tests {
+
+using namespace std::literals;
+
+//////////////////////////////////////////////////////////////////////////
+
+static bool
+LogContains(const std::string& Log, std::string_view Needle)
+{
+	return Log.find(Needle) != std::string::npos;
+}
+
+static std::string
+ReadFileToString(const std::filesystem::path& Path)
+{
+	FileContents Contents = ReadFile(Path);
+	if (Contents.ErrorCode)
+	{
+		return {};
+	}
+
+	IoBuffer Content = Contents.Flatten();
+	if (!Content)
+	{
+		return {};
+	}
+
+	return std::string(static_cast<const char*>(Content.Data()), Content.Size());
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+// Verify that a log file is created at the default location (DataDir/logs/zenserver.log)
+// even without --abslog.  The file must contain "server session id" (logged at INFO
+// to all registered loggers during init) and "log starting at" (emitted once a file
+// sink is first opened).
+TEST_CASE("logging.file.default")
+{
+	const std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestDir);
+	const uint16_t Port = Instance.SpawnServerAndWaitUntilReady();
+	CHECK_MESSAGE(Port != 0, Instance.GetLogOutput());
+
+	Instance.Shutdown();
+
+	const std::filesystem::path DefaultLogFile = TestDir / "logs" / "zenserver.log";
+	CHECK_MESSAGE(std::filesystem::exists(DefaultLogFile), "Default log file was not created");
+	const std::string FileLog = ReadFileToString(DefaultLogFile);
+	CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog);
+	CHECK_MESSAGE(LogContains(FileLog, "log starting at"), FileLog);
+}
+
+// --quiet sets the console sink level to WARN.  The formatted "[info] ..."
+// entry written by the default logger's console sink must therefore not appear
+// in captured stdout.  (The "console" named logger — used by ZEN_CONSOLE_*
+// macros — may still emit plain-text messages without a level marker, so we
+// check for the absence of the full_formatter "[info]" prefix rather than the
+// message text itself.)
+TEST_CASE("logging.console.quiet")
+{
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestEnv.CreateNewTestDir());
+	const uint16_t Port = Instance.SpawnServerAndWaitUntilReady("--quiet");
+	CHECK_MESSAGE(Port != 0, Instance.GetLogOutput());
+
+	Instance.Shutdown();
+
+	const std::string Log = Instance.GetLogOutput();
+	CHECK_MESSAGE(!LogContains(Log, "[info] server session id"), Log);
+}
+
+// --noconsole removes the stdout sink entirely, so the captured console output
+// must not contain any log entries from the logging system.
+TEST_CASE("logging.console.disabled")
+{
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestEnv.CreateNewTestDir());
+	const uint16_t Port = Instance.SpawnServerAndWaitUntilReady("--noconsole");
+	CHECK_MESSAGE(Port != 0, Instance.GetLogOutput());
+
+	Instance.Shutdown();
+
+	const std::string Log = Instance.GetLogOutput();
+	CHECK_MESSAGE(!LogContains(Log, "server session id"), Log);
+}
+
+// --abslog <path> creates a rotating log file at the specified path.
+// The file must contain "server session id" (logged at INFO to all loggers
+// during init) and "log starting at" (emitted once a file sink is active).
+TEST_CASE("logging.file.basic")
+{
+	const std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+	const std::filesystem::path LogFile = TestDir / "test.log";
+
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestDir);
+
+	const std::string LogArg = fmt::format("--abslog {}", LogFile.string());
+	const uint16_t	  Port	 = Instance.SpawnServerAndWaitUntilReady(LogArg);
+	CHECK_MESSAGE(Port != 0, Instance.GetLogOutput());
+
+	Instance.Shutdown();
+
+	CHECK_MESSAGE(std::filesystem::exists(LogFile), "Log file was not created");
+	const std::string FileLog = ReadFileToString(LogFile);
+	CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog);
+	CHECK_MESSAGE(LogContains(FileLog, "log starting at"), FileLog);
+}
+
+// --abslog with a .json extension selects the JSON formatter.
+// Each log entry must be a JSON object containing at least the "message"
+// and "source" fields.
+TEST_CASE("logging.file.json")
+{
+	const std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+	const std::filesystem::path LogFile = TestDir / "test.json";
+
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestDir);
+
+	const std::string LogArg = fmt::format("--abslog {}", LogFile.string());
+	const uint16_t	  Port	 = Instance.SpawnServerAndWaitUntilReady(LogArg);
+	CHECK_MESSAGE(Port != 0, Instance.GetLogOutput());
+
+	Instance.Shutdown();
+
+	CHECK_MESSAGE(std::filesystem::exists(LogFile), "JSON log file was not created");
+	const std::string FileLog = ReadFileToString(LogFile);
+	CHECK_MESSAGE(LogContains(FileLog, "\"message\""), FileLog);
+	CHECK_MESSAGE(LogContains(FileLog, "\"source\": \"zenserver\""), FileLog);
+	CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog);
+}
+
+// --log-id <id> is automatically set to the server instance name in test mode.
+// The JSON formatter emits this value as the "id" field, so every entry in a
+// .json log file must carry a non-empty "id".
+TEST_CASE("logging.log_id")
+{
+	const std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+	const std::filesystem::path LogFile = TestDir / "test.json";
+
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestDir);
+
+	const std::string LogArg = fmt::format("--abslog {}", LogFile.string());
+	const uint16_t	  Port	 = Instance.SpawnServerAndWaitUntilReady(LogArg);
+	CHECK_MESSAGE(Port != 0, Instance.GetLogOutput());
+
+	Instance.Shutdown();
+
+	CHECK_MESSAGE(std::filesystem::exists(LogFile), "JSON log file was not created");
+	const std::string FileLog = ReadFileToString(LogFile);
+	// The JSON formatter writes the log-id as: "id": "<value>",
+	CHECK_MESSAGE(LogContains(FileLog, "\"id\": \""), FileLog);
+}
+
+// --log-warn <logger> raises the level threshold above INFO so that INFO messages
+// are filtered.  "server session id" is broadcast at INFO to all loggers: it must
+// appear in the main file sink (default logger unaffected) but must NOT appear in
+// http.log where the http_requests logger now has a WARN threshold.
+TEST_CASE("logging.level.warn_suppresses_info")
+{
+	const std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+	const std::filesystem::path LogFile = TestDir / "test.log";
+
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestDir);
+
+	const std::string LogArg = fmt::format("--abslog {} --log-warn http_requests", LogFile.string());
+	const uint16_t	  Port	 = Instance.SpawnServerAndWaitUntilReady(LogArg);
+	CHECK_MESSAGE(Port != 0, Instance.GetLogOutput());
+
+	Instance.Shutdown();
+
+	CHECK_MESSAGE(std::filesystem::exists(LogFile), "Log file was not created");
+	const std::string FileLog = ReadFileToString(LogFile);
+	CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog);
+
+	const std::filesystem::path HttpLogFile = TestDir / "logs" / "http.log";
+	CHECK_MESSAGE(std::filesystem::exists(HttpLogFile), "http.log was not created");
+	const std::string HttpLog = ReadFileToString(HttpLogFile);
+	CHECK_MESSAGE(!LogContains(HttpLog, "server session id"), HttpLog);
+}
+
+// --log-info <logger> sets an explicit INFO threshold.  The INFO "server session id"
+// broadcast must still land in http.log, confirming that INFO messages are not
+// filtered when the logger level is exactly INFO.
+TEST_CASE("logging.level.info_allows_info")
+{
+	const std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+	const std::filesystem::path LogFile = TestDir / "test.log";
+
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestDir);
+
+	const std::string LogArg = fmt::format("--abslog {} --log-info http_requests", LogFile.string());
+	const uint16_t	  Port	 = Instance.SpawnServerAndWaitUntilReady(LogArg);
+	CHECK_MESSAGE(Port != 0, Instance.GetLogOutput());
+
+	Instance.Shutdown();
+
+	const std::filesystem::path HttpLogFile = TestDir / "logs" / "http.log";
+	CHECK_MESSAGE(std::filesystem::exists(HttpLogFile), "http.log was not created");
+	const std::string HttpLog = ReadFileToString(HttpLogFile);
+	CHECK_MESSAGE(LogContains(HttpLog, "server session id"), HttpLog);
+}
+
+// --log-off <logger> silences a named logger entirely.
+// "server session id" is broadcast at INFO to all registered loggers via
+// spdlog::apply_all during init.  When the "http_requests" logger is set to
+// OFF its dedicated http.log file must not contain that message.
+// The main file sink (via --abslog) must be unaffected.
+TEST_CASE("logging.level.off_specific_logger")
+{
+	const std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
+	const std::filesystem::path LogFile = TestDir / "test.log";
+
+	ZenServerInstance Instance(TestEnv);
+	Instance.SetDataDir(TestDir);
+
+	const std::string LogArg = fmt::format("--abslog {} --log-off http_requests", LogFile.string());
+	const uint16_t	  Port	 = Instance.SpawnServerAndWaitUntilReady(LogArg);
+	CHECK_MESSAGE(Port != 0, Instance.GetLogOutput());
+
+	Instance.Shutdown();
+
+	// Main log file must still have the startup message
+	CHECK_MESSAGE(std::filesystem::exists(LogFile), "Log file was not created");
+	const std::string FileLog = ReadFileToString(LogFile);
+	CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog);
+
+	// http.log is created by the RotatingFileSink but the logger is OFF, so
+	// the broadcast "server session id" message must not have been written to it
+	const std::filesystem::path HttpLogFile = TestDir / "logs" / "http.log";
+	CHECK_MESSAGE(std::filesystem::exists(HttpLogFile), "http.log was not created");
+	const std::string HttpLog = ReadFileToString(HttpLogFile);
+	CHECK_MESSAGE(!LogContains(HttpLog, "server session id"), HttpLog);
+}
+
+}  // namespace zen::tests
+
+#endif