zenmaster + zenmaster-test skeletons

6 files changed, 525 insertions, 0 deletions

diff --git a/src/zenmaster-test/xmake.lua b/src/zenmaster-test/xmake.lua
new file mode 100644
index 000000000..ace6c4254
--- /dev/null
+++ b/src/zenmaster-test/xmake.lua
@@ -0,0 +1,17 @@
+-- Copyright Epic Games, Inc. All Rights Reserved.
+
+target("zenmaster-test")
+    set_kind("binary")
+    set_group("tests")
+    add_headerfiles("**.h")
+    add_files("*.cpp")
+    add_files("zenmaster-test.cpp", {unity_ignored  = true })
+    add_deps("zencore", "zenutil", "zenhttp")
+    add_deps("zenmaster", {inherit=false})
+    add_packages("vcpkg::cpr", "vcpkg::http-parser", "vcpkg::mimalloc")
+
+    if is_plat("macosx") then
+        add_ldflags("-framework CoreFoundation")
+        add_ldflags("-framework Security")
+        add_ldflags("-framework SystemConfiguration")
+    end
diff --git a/src/zenmaster-test/zenmaster-test.cpp b/src/zenmaster-test/zenmaster-test.cpp
new file mode 100644
index 000000000..7511d6b4a
--- /dev/null
+++ b/src/zenmaster-test/zenmaster-test.cpp
@@ -0,0 +1,96 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#define _SILENCE_CXX17_C_HEADER_DEPRECATION_WARNING
+
+#include <zenbase/refcount.h>
+#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinarypackage.h>
+#include <zencore/compress.h>
+#include <zencore/except.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/memoryview.h>
+#include <zencore/scopeguard.h>
+#include <zencore/stream.h>
+#include <zencore/string.h>
+#include <zencore/testutils.h>
+#include <zencore/timer.h>
+#include <zencore/xxhash.h>
+#include <zenutil/logging/testformatter.h>
+#include <zenutil/zenserverprocess.h>
+
+#include <atomic>
+#include <filesystem>
+#include <map>
+#include <random>
+#include <span>
+#include <thread>
+#include <typeindex>
+#include <unordered_map>
+
+#include <zencore/memory/newdelete.h>
+
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_WITH_TESTS
+#	define ZEN_TEST_WITH_RUNNER 1
+#	include <zencore/testing.h>
+#endif
+
+using namespace std::literals;
+
+#if !ZEN_WITH_TESTS
+int
+main()
+{
+}
+#else
+zen::ZenServerEnvironment TestEnv;
+
+int
+main(int argc, char** argv)
+{
+	using namespace std::literals;
+	using namespace zen;
+
+#	if ZEN_PLATFORM_LINUX
+	IgnoreChildSignals();
+#	endif
+
+	zen::TraceInit("zenmaster-test");
+	zen::logging::InitializeLogging();
+
+	zen::logging::SetLogLevel(zen::logging::level::Debug);
+	spdlog::set_formatter(std::make_unique<zen::logging::full_test_formatter>("test", std::chrono::system_clock::now()));
+
+	std::filesystem::path ProgramBaseDir = GetRunningExecutablePath().parent_path();
+	std::filesystem::path TestBaseDir	 = std::filesystem::current_path() / ".test";
+
+	// This is pretty janky because we're passing most of the options through to the test
+	// framework, so we can't just use cxxopts (I think). This should ideally be cleaned up
+	// somehow in the future
+
+	std::string ServerClass;
+
+	for (int i = 1; i < argc; ++i)
+	{
+		if (argv[i] == "--http"sv)
+		{
+			if ((i + 1) < argc)
+			{
+				ServerClass = argv[++i];
+			}
+		}
+	}
+
+	TestEnv.InitializeForTest(ProgramBaseDir, TestBaseDir, ServerClass);
+
+	ZEN_INFO("Running tests...(base dir: '{}')", TestBaseDir);
+
+	zen::testing::TestRunner Runner;
+	Runner.ApplyCommandLine(argc, argv);
+
+	return Runner.Run();
+}
+#endif
diff --git a/src/zenmaster/xmake.lua b/src/zenmaster/xmake.lua
new file mode 100644
index 000000000..52385a490
--- /dev/null
+++ b/src/zenmaster/xmake.lua
@@ -0,0 +1,31 @@
+-- Copyright Epic Games, Inc. All Rights Reserved.
+
+target("zenmaster")
+    set_kind("binary")
+    add_headerfiles("**.h")
+    add_files("**.cpp")
+    add_files("zenmaster.cpp", {unity_ignored  = true })
+    add_deps("zencore", "zenhttp", "zenstore", "zenutil")
+    add_includedirs(".")
+    set_symbols("debug")
+
+    if is_mode("release") then
+        set_optimize("fastest")
+    end
+
+    if is_plat("windows") then
+        add_files("zenmaster.rc")
+        add_ldflags("/subsystem:console,5.02")
+        add_ldflags("/LTCG")
+        add_links("crypt32", "wldap32", "Ws2_32", "Shlwapi")
+        add_links("dbghelp", "winhttp", "version")    -- for Sentry
+    end
+
+    if is_plat("macosx") then
+        add_ldflags("-framework CoreFoundation")
+        add_ldflags("-framework Foundation")
+        add_ldflags("-framework Security")
+        add_ldflags("-framework SystemConfiguration")
+    end
+
+    add_packages("vcpkg::cpr", "vcpkg::cxxopts", "vcpkg::mimalloc", "vcpkg::fmt")
diff --git a/src/zenmaster/zenmaster.cpp b/src/zenmaster/zenmaster.cpp
new file mode 100644
index 000000000..483ec6828
--- /dev/null
+++ b/src/zenmaster/zenmaster.cpp
@@ -0,0 +1,321 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+// Zen command line client utility
+//
+
+#include "zenmaster.h"
+
+#include <zencore/callstack.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/process.h>
+#include <zencore/scopeguard.h>
+#include <zencore/sentryintegration.h>
+#include <zencore/string.h>
+#include <zencore/trace.h>
+#include <zencore/windows.h>
+#include <zenhttp/httpcommon.h>
+#include <zenutil/environmentoptions.h>
+#include <zenutil/logging.h>
+#include <zenutil/workerpools.h>
+#include <zenutil/zenserverprocess.h>
+
+#include <zencore/memory/fmalloc.h>
+#include <zencore/memory/llm.h>
+#include <zencore/memory/memory.h>
+#include <zencore/memory/memorytrace.h>
+#include <zencore/memory/newdelete.h>
+
+#if ZEN_WITH_TESTS
+#	define ZEN_TEST_WITH_RUNNER 1
+#	include <zencore/testing.h>
+#endif
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <cpr/cpr.h>
+#include <spdlog/sinks/ansicolor_sink.h>
+#include <spdlog/spdlog.h>
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#include <zencore/memory/newdelete.h>
+
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+#	include <sys/ioctl.h>
+#	include <unistd.h>
+#endif
+
+//////////////////////////////////////////////////////////////////////////
+
+namespace zen {
+
+}
+
+int
+main(int argc, char** argv)
+{
+	zen::SetCurrentThreadName("main");
+
+	std::vector<std::string> Args;
+#if ZEN_PLATFORM_WINDOWS
+	LPWSTR		RawCommandLine = GetCommandLine();
+	std::string CommandLine	   = zen::WideToUtf8(RawCommandLine);
+	Args					   = zen::ParseCommandLine(CommandLine);
+#else
+	Args.reserve(argc);
+	for (int I = 0; I < argc; I++)
+	{
+		std::string Arg(argv[I]);
+		if ((!Arg.empty()) && (Arg != " "))
+		{
+			Args.emplace_back(std::move(Arg));
+		}
+	}
+#endif
+	std::vector<char*> RawArgs = zen::StripCommandlineQuotes(Args);
+
+	argc = gsl::narrow<int>(RawArgs.size());
+	argv = RawArgs.data();
+
+	using namespace zen;
+	using namespace std::literals;
+
+#if ZEN_WITH_TRACE
+	TraceInit("zen");
+	TraceOptions TraceCommandlineOptions;
+	if (GetTraceOptionsFromCommandline(TraceCommandlineOptions))
+	{
+		TraceConfigure(TraceCommandlineOptions);
+	}
+#endif	// ZEN_WITH_TRACE
+
+	// Split command line into options, commands and any pass-through arguments
+
+	std::string				 Passthrough;
+	std::string				 PassthroughArgs;
+	std::vector<std::string> PassthroughArgV;
+
+	for (int i = 1; i < argc; ++i)
+	{
+		if ("--"sv == argv[i])
+		{
+			bool							  IsFirst = true;
+			zen::ExtendableStringBuilder<256> Line;
+			zen::ExtendableStringBuilder<256> Arguments;
+
+			for (int j = i + 1; j < argc; ++j)
+			{
+				auto AppendAscii = [&](auto X) {
+					Line.Append(X);
+					if (!IsFirst)
+					{
+						Arguments.Append(X);
+					}
+				};
+
+				if (!IsFirst)
+				{
+					AppendAscii(" ");
+				}
+
+				std::string_view ThisArg(argv[j]);
+				PassthroughArgV.push_back(std::string(ThisArg));
+
+				const bool NeedsQuotes =
+					(ThisArg.find(' ') != std::string_view::npos) && !(ThisArg.starts_with("\"") && ThisArg.ends_with("\""));
+
+				if (NeedsQuotes)
+				{
+					AppendAscii("\"");
+				}
+
+				AppendAscii(ThisArg);
+
+				if (NeedsQuotes)
+				{
+					AppendAscii("\"");
+				}
+
+				IsFirst = false;
+			}
+
+			Passthrough		= Line.c_str();
+			PassthroughArgs = Arguments.c_str();
+
+			// This will "truncate" the arg vector and terminate the loop
+			argc = i;
+		}
+	}
+
+	// Parse global CLI arguments
+
+	ZenMasterCliOptions GlobalOptions;
+
+	GlobalOptions.PassthroughCommandLine = Passthrough;
+	GlobalOptions.PassthroughArgs		 = PassthroughArgs;
+	GlobalOptions.PassthroughArgV		 = PassthroughArgV;
+
+	std::string MemoryOptions;
+
+	std::string SubCommand = "<None>";
+
+	cxxopts::Options Options("zenmaster", "Zen master orchestration tool");
+
+	Options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(GlobalOptions.IsDebug));
+	Options.add_options()("v, verbose", "Enable verbose logging", cxxopts::value<bool>(GlobalOptions.IsVerbose));
+	Options.add_options()("malloc", "Configure memory allocator subsystem", cxxopts::value(MemoryOptions)->default_value("mimalloc"));
+	Options.add_options()("help", "Show command line help");
+
+#if ZEN_WITH_TRACE
+	// We only have this in options for command line help purposes - we parse these argument separately earlier using
+	// GetTraceOptionsFromCommandline()
+
+	Options.add_option("ue-trace",
+					   "",
+					   "trace",
+					   "Specify which trace channels should be enabled",
+					   cxxopts::value<std::string>(TraceCommandlineOptions.Channels)->default_value(""),
+					   "");
+
+	Options.add_option("ue-trace",
+					   "",
+					   "tracehost",
+					   "Hostname to send the trace to",
+					   cxxopts::value<std::string>(TraceCommandlineOptions.Host)->default_value(""),
+					   "");
+
+	Options.add_option("ue-trace",
+					   "",
+					   "tracefile",
+					   "Path to write a trace to",
+					   cxxopts::value<std::string>(TraceCommandlineOptions.File)->default_value(""),
+					   "");
+#endif	// ZEN_WITH_TRACE
+
+#if ZEN_USE_SENTRY
+	SentryIntegration::Config SentryConfig;
+
+	bool NoSentry = false;
+
+	Options
+		.add_option("sentry", "", "no-sentry", "Disable Sentry crash handler", cxxopts::value<bool>(NoSentry)->default_value("false"), "");
+	Options.add_option("sentry",
+					   "",
+					   "sentry-allow-personal-info",
+					   "Allow personally identifiable information in sentry crash reports",
+					   cxxopts::value<bool>(SentryConfig.AllowPII)->default_value("false"),
+					   "");
+	Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(SentryConfig.Dsn), "");
+	Options.add_option("sentry", "", "sentry-environment", "Sentry environment", cxxopts::value<std::string>(SentryConfig.Environment), "");
+	Options.add_options()("sentry-debug", "Enable debug mode for Sentry", cxxopts::value<bool>(SentryConfig.Debug)->default_value("false"));
+#endif
+
+	try
+	{
+		cxxopts::ParseResult ParseResult = Options.parse(argc, argv);
+
+		if (ParseResult.count("help"))
+		{
+			std::string Help = Options.help();
+
+			printf("%s\n", Help.c_str());
+
+			exit(0);
+		}
+
+#if ZEN_USE_SENTRY
+
+		{
+			EnvironmentOptions EnvOptions;
+
+			EnvOptions.AddOption("UE_ZEN_SENTRY_DSN"sv, SentryConfig.Dsn, "sentry-dsn"sv);
+			EnvOptions.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
+			EnvOptions.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, SentryConfig.Environment, "sentry-environment"sv);
+
+			bool EnvEnableSentry = !NoSentry;
+			EnvOptions.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv);
+
+			EnvOptions.AddOption("UE_ZEN_SENTRY_DEBUG"sv, SentryConfig.Debug, "sentry-debug"sv);
+
+			EnvOptions.Parse(ParseResult);
+
+			if (EnvEnableSentry != !NoSentry)
+			{
+				NoSentry = !EnvEnableSentry;
+			}
+		}
+
+		SentryIntegration Sentry;
+
+		if (NoSentry == false)
+		{
+			std::string SentryDatabasePath = (std::filesystem::temp_directory_path() / ".zen-sentry-native").string();
+
+			ExtendableStringBuilder<512> SB;
+			for (int i = 0; i < argc; ++i)
+			{
+				if (i)
+				{
+					SB.Append(' ');
+				}
+
+				SB.Append(argv[i]);
+			}
+
+			SentryConfig.DatabasePath = SentryDatabasePath;
+
+			Sentry.Initialize(SentryConfig, SB.ToString());
+
+			SentryIntegration::ClearCaches();
+		}
+#endif
+
+		zen::LoggingOptions LogOptions;
+		LogOptions.IsDebug	  = GlobalOptions.IsDebug;
+		LogOptions.IsVerbose  = GlobalOptions.IsVerbose;
+		LogOptions.AllowAsync = false;
+		zen::InitializeLogging(LogOptions);
+
+		std::set_terminate([]() {
+			void*			 Frames[8];
+			uint32_t		 FrameCount = GetCallstack(2, 8, Frames);
+			CallstackFrames* Callstack	= CreateCallstack(FrameCount, Frames);
+			ZEN_CRITICAL("Program exited abnormally via std::terminate()\n{}", CallstackToString(Callstack, "  "));
+			FreeCallstack(Callstack);
+		});
+
+		zen::MaximizeOpenFileCount();
+
+		//////////////////////////////////////////////////////////////////////////
+
+		auto _ = zen::MakeGuard([] {
+			ShutdownWorkerPools();
+			ShutdownLogging();
+		});
+
+		zen::Sleep(1000000);
+	}
+	catch (const OptionParseException& Ex)
+	{
+		std::string HelpMessage = Options.help();
+
+		printf("Error parsing program arguments: %s\n\n%s", Ex.what(), HelpMessage.c_str());
+
+		return 9;
+	}
+	catch (const std::system_error& Ex)
+	{
+		printf("System Error: %s\n", Ex.what());
+
+		return Ex.code() ? Ex.code().value() : 10;
+	}
+	catch (const std::exception& Ex)
+	{
+		printf("Error: %s\n", Ex.what());
+
+		return 11;
+	}
+
+	return 0;
+}
diff --git a/src/zenmaster/zenmaster.h b/src/zenmaster/zenmaster.h
new file mode 100644
index 000000000..1a247c5cd
--- /dev/null
+++ b/src/zenmaster/zenmaster.h
@@ -0,0 +1,27 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/except.h>
+#include <zencore/timer.h>
+#include <zencore/zencore.h>
+#include <zenutil/commandlineoptions.h>
+
+namespace cpr {
+class Response;
+}
+
+namespace zen {
+
+struct ZenMasterCliOptions
+{
+	bool IsDebug   = false;
+	bool IsVerbose = false;
+
+	// Arguments after " -- " on command line are passed through and not parsed
+	std::string				 PassthroughCommandLine;
+	std::string				 PassthroughArgs;
+	std::vector<std::string> PassthroughArgV;
+};
+
+}  // namespace zen
diff --git a/src/zenmaster/zenmaster.rc b/src/zenmaster/zenmaster.rc
new file mode 100644
index 000000000..6a18ebbb5
--- /dev/null
+++ b/src/zenmaster/zenmaster.rc
@@ -0,0 +1,33 @@
+#include "zencore/config.h"
+
+#define APSTUDIO_READONLY_SYMBOLS
+#include "winres.h"
+#undef APSTUDIO_READONLY_SYMBOLS
+
+LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
+#pragma code_page(1252)
+
+101               ICON                    "..\\UnrealEngine.ico"
+
+VS_VERSION_INFO VERSIONINFO
+FILEVERSION    ZEN_CFG_VERSION_MAJOR,ZEN_CFG_VERSION_MINOR,ZEN_CFG_VERSION_ALTER,0
+PRODUCTVERSION ZEN_CFG_VERSION_MAJOR,ZEN_CFG_VERSION_MINOR,ZEN_CFG_VERSION_ALTER,0
+{
+    BLOCK "StringFileInfo"
+    {
+        BLOCK "040904b0"
+        {
+            VALUE "CompanyName",        "Epic Games Inc\0"
+            VALUE "FileDescription",    "Instance orchestrator for Unreal Zen Storage Service\0"
+            VALUE "FileVersion",        ZEN_CFG_VERSION "\0"
+            VALUE "LegalCopyright",     "Copyright Epic Games Inc. All Rights Reserved\0"
+            VALUE "OriginalFilename",   "zenmaster.exe\0"
+            VALUE "ProductName",        "Unreal Zen Storage Server\0"
+			VALUE "ProductVersion",		ZEN_CFG_VERSION_BUILD_STRING_FULL "\0"
+        }
+    }
+    BLOCK "VarFileInfo"
+    {
+        VALUE "Translation", 0x409, 1200
+    }
+}