move all storage-related services into storage tree (#571)

* move all storage-related services into storage tree
* move config into config/
* also move admin service into storage since it mostly has storage related functionality
* header consolidation

1 files changed, 512 insertions, 0 deletions

diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp
new file mode 100644
index 000000000..d044c61d5
--- /dev/null
+++ b/src/zenserver/config/config.cpp
@@ -0,0 +1,512 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "config.h"
+
+#include "storage/storageconfig.h"
+
+#include "config/luaconfig.h"
+#include "diag/logging.h"
+
+#include <zencore/basicfile.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryutil.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/except.h>
+#include <zencore/fmtutils.h>
+#include <zencore/iobuffer.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+#include <zenutil/commandlineoptions.h>
+#include <zenutil/environmentoptions.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+#include <cxxopts.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <conio.h>
+#else
+#	include <unistd.h>
+#endif
+
+namespace zen {
+
+std::filesystem::path
+PickDefaultStateDirectory(std::filesystem::path SystemRoot)
+{
+	if (SystemRoot.empty())
+		return SystemRoot;
+
+	return SystemRoot / "Data";
+}
+
+void
+EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath)
+{
+	CbObjectWriter Cbo;
+	Cbo << "path" << ManifestPath.generic_wstring();
+	Cbo << "manifest" << Manifest;
+
+	const std::filesystem::path StatesPath = SystemRoot / "States";
+
+	CreateDirectories(StatesPath);
+	WriteFile(StatesPath / fmt::format("{}", Identifier), Cbo.Save().GetBuffer().AsIoBuffer());
+}
+
+std::vector<CbObject>
+ReadAllCentralManifests(const std::filesystem::path& SystemRoot)
+{
+	std::vector<CbObject> Manifests;
+
+	DirectoryContent Content;
+	GetDirectoryContent(SystemRoot / "States", DirectoryContentFlags::IncludeFiles, Content);
+
+	for (std::filesystem::path& File : Content.Files)
+	{
+		try
+		{
+			FileContents	FileData = ReadFile(File);
+			CbValidateError ValidateError;
+			if (CbObject Manifest = ValidateAndReadCompactBinaryObject(FileData.Flatten(), ValidateError);
+				ValidateError == CbValidateError::None)
+			{
+				Manifests.emplace_back(std::move(Manifest));
+			}
+			else
+			{
+				ZEN_WARN("failed to load manifest '{}': {}", File, ToString(ValidateError));
+			}
+		}
+		catch (const std::exception& Ex)
+		{
+			ZEN_WARN("failed to load manifest '{}': {}", File, Ex.what());
+		}
+	}
+
+	return Manifests;
+}
+
+void
+ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult)
+{
+	using namespace std::literals;
+
+	EnvironmentOptions Options;
+	Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
+	Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
+	Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv);
+
+	bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable;
+	Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv);
+
+	Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv);
+
+	Options.Parse(CmdLineResult);
+
+	if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable)
+	{
+		ServerOptions.SentryConfig.Disable = !EnvEnableSentry;
+	}
+}
+
+void
+AddServerConfigOptions(LuaConfig::Options& LuaOptions, ZenServerOptions& ServerOptions)
+{
+	using namespace std::literals;
+
+	// server
+
+	LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv);
+	LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv);
+	LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv);
+	LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
+	LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
+	LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv);
+	LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv);
+	LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv);
+	LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
+	LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
+	LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv);
+	LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv);
+	LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv);
+	LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv);
+	LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv);
+
+	////// network
+	LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv);
+	LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv);
+	LuaOptions.AddOption("network.port"sv, ServerOptions.BasePort, "port"sv);
+	LuaOptions.AddOption("network.forceloopback"sv, ServerOptions.HttpServerConfig.ForceLoopback, "http-forceloopback"sv);
+
+#if ZEN_WITH_HTTPSYS
+	LuaOptions.AddOption("network.httpsys.async.workthreads"sv,
+						 ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount,
+						 "httpsys-async-work-threads"sv);
+	LuaOptions.AddOption("network.httpsys.async.response"sv,
+						 ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled,
+						 "httpsys-enable-async-response"sv);
+	LuaOptions.AddOption("network.httpsys.requestlogging"sv,
+						 ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled,
+						 "httpsys-enable-request-logging"sv);
+#endif
+
+#if ZEN_WITH_TRACE
+	////// trace
+	LuaOptions.AddOption("trace.channels"sv, ServerOptions.TraceOptions.Channels, "trace"sv);
+	LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceOptions.Host, "tracehost"sv);
+	LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceOptions.File, "tracefile"sv);
+#endif
+
+	////// stats
+	LuaOptions.AddOption("stats.enable"sv, ServerOptions.StatsConfig.Enabled, "statsd"sv);
+	LuaOptions.AddOption("stats.host"sv, ServerOptions.StatsConfig.StatsdHost);
+	LuaOptions.AddOption("stats.port"sv, ServerOptions.StatsConfig.StatsdPort);
+}
+
+struct ZenServerCmdLineOptions
+{
+	// Note to those adding future options; std::filesystem::path-type options
+	// must be read into a std::string first. As of cxxopts-3.0.0 it uses a >>
+	// stream operator to convert argv value into the options type. std::fs::path
+	// expects paths in streams to be quoted but argv paths are unquoted. By
+	// going into a std::string first, paths with whitespace parse correctly.
+	std::string ConfigFile;
+	std::string OutputConfigFile;
+	std::string SystemRootDir;
+	std::string ContentDir;
+	std::string DataDir;
+	std::string AbsLogFile;
+
+	void AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions);
+	void ApplyOptions(ZenServerOptions& ServerOptions);
+};
+
+void
+ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions)
+{
+	const char* DefaultHttp = "asio";
+
+#if ZEN_WITH_HTTPSYS
+	if (!windows::IsRunningOnWine())
+	{
+		DefaultHttp = "httpsys";
+	}
+#endif
+
+	options.add_options()("dedicated",
+						  "Enable dedicated server mode",
+						  cxxopts::value<bool>(ServerOptions.IsDedicated)->default_value("false"));
+	options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(ServerOptions.IsDebug)->default_value("false"));
+	options.add_options()("clean",
+						  "Clean out all state at startup",
+						  cxxopts::value<bool>(ServerOptions.IsCleanStart)->default_value("false"));
+	options.add_options()("help", "Show command line help");
+	options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
+
+	options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir));
+	options.add_options()("system-dir", "Specify system root", cxxopts::value<std::string>(SystemRootDir));
+	options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::string>(ContentDir));
+	options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
+	options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile));
+
+	options.add_options()("no-sentry",
+						  "Disable Sentry crash handler",
+						  cxxopts::value<bool>(ServerOptions.SentryConfig.Disable)->default_value("false"));
+	options.add_options()("sentry-allow-personal-info",
+						  "Allow personally identifiable information in sentry crash reports",
+						  cxxopts::value<bool>(ServerOptions.SentryConfig.AllowPII)->default_value("false"));
+	options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(ServerOptions.SentryConfig.Dsn));
+	options.add_options()("sentry-environment", "Sentry environment", cxxopts::value<std::string>(ServerOptions.SentryConfig.Environment));
+	options.add_options()("sentry-debug",
+						  "Enable debug mode for Sentry",
+						  cxxopts::value<bool>(ServerOptions.SentryConfig.Debug)->default_value("false"));
+	options.add_options()("detach",
+						  "Indicate whether zenserver should detach from parent process group",
+						  cxxopts::value<bool>(ServerOptions.Detach)->default_value("true"));
+	options.add_options()("malloc",
+						  "Configure memory allocator subsystem",
+						  cxxopts::value(ServerOptions.MemoryOptions)->default_value("mimalloc"));
+	options.add_options()("powercycle",
+						  "Exit immediately after initialization is complete",
+						  cxxopts::value<bool>(ServerOptions.IsPowerCycle));
+
+	options.add_option("diagnostics",
+					   "",
+					   "crash",
+					   "Simulate a crash",
+					   cxxopts::value<bool>(ServerOptions.ShouldCrash)->default_value("false"),
+					   "");
+
+	// clang-format off
+	options.add_options("logging")
+		("abslog",			"Path to log file",									cxxopts::value<std::string>(AbsLogFile))
+		("log-id",			"Specify id for adding context to log output",		cxxopts::value<std::string>(ServerOptions.LogId))
+		("quiet",			"Configure console logger output to level WARN",	cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false"))
+		("noconsole",		"Disable console logging",							cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
+		("log-trace", 		"Change selected loggers to level TRACE", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
+		("log-debug", 		"Change selected loggers to level DEBUG", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
+		("log-info", 		"Change selected loggers to level INFO", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
+		("log-warn", 		"Change selected loggers to level WARN", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
+		("log-error", 		"Change selected loggers to level ERROR", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
+		("log-critical", 	"Change selected loggers to level CRITICAL", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
+		("log-off", 		"Change selected loggers to level OFF", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
+	;
+	// clang-format on
+
+	options
+		.add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value<int>(ServerOptions.OwnerPid), "<identifier>");
+	options.add_option("lifetime",
+					   "",
+					   "child-id",
+					   "Specify id which can be used to signal parent",
+					   cxxopts::value<std::string>(ServerOptions.ChildId),
+					   "<identifier>");
+
+#if ZEN_PLATFORM_WINDOWS
+	options.add_option("lifetime",
+					   "",
+					   "install",
+					   "Install zenserver as a Windows service",
+					   cxxopts::value<bool>(ServerOptions.InstallService),
+					   "");
+	options.add_option("lifetime",
+					   "",
+					   "uninstall",
+					   "Uninstall zenserver as a Windows service",
+					   cxxopts::value<bool>(ServerOptions.UninstallService),
+					   "");
+#endif
+
+	options.add_option("network",
+					   "",
+					   "http-threads",
+					   "Number of http server connection threads",
+					   cxxopts::value<unsigned int>(ServerOptions.HttpServerConfig.ThreadCount),
+					   "<http threads>");
+
+	options.add_option("network",
+					   "p",
+					   "port",
+					   "Select HTTP port",
+					   cxxopts::value<int>(ServerOptions.BasePort)->default_value("8558"),
+					   "<port number>");
+
+	options.add_option("network",
+					   "",
+					   "http-forceloopback",
+					   "Force using local loopback interface",
+					   cxxopts::value<bool>(ServerOptions.HttpServerConfig.ForceLoopback)->default_value("false"),
+					   "<http forceloopback>");
+
+#if ZEN_WITH_HTTPSYS
+	options.add_option("httpsys",
+					   "",
+					   "httpsys-async-work-threads",
+					   "Number of HttpSys async worker threads",
+					   cxxopts::value<unsigned int>(ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount),
+					   "<httpsys workthreads>");
+
+	options.add_option("httpsys",
+					   "",
+					   "httpsys-enable-async-response",
+					   "Enables Httpsys async response",
+					   cxxopts::value<bool>(ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled)->default_value("true"),
+					   "<httpsys async response>");
+
+	options.add_option("httpsys",
+					   "",
+					   "httpsys-enable-request-logging",
+					   "Enables Httpsys request logging",
+					   cxxopts::value<bool>(ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled),
+					   "<httpsys request logging>");
+#endif
+
+	options.add_option("network",
+					   "",
+					   "http",
+					   "Select HTTP server implementation (asio|"
+#if ZEN_WITH_HTTPSYS
+					   "httpsys|"
+#endif
+					   "null)",
+					   cxxopts::value<std::string>(ServerOptions.HttpServerConfig.ServerClass)->default_value(DefaultHttp),
+					   "<http class>");
+
+#if ZEN_WITH_TRACE
+	// We only have this in options for command line help purposes - we parse these argument separately earlier using
+	// GetTraceOptionsFromCommandline()
+
+	options.add_option("ue-trace",
+					   "",
+					   "trace",
+					   "Specify which trace channels should be enabled",
+					   cxxopts::value<std::string>(ServerOptions.TraceOptions.Channels)->default_value(""),
+					   "");
+
+	options.add_option("ue-trace",
+					   "",
+					   "tracehost",
+					   "Hostname to send the trace to",
+					   cxxopts::value<std::string>(ServerOptions.TraceOptions.Host)->default_value(""),
+					   "");
+
+	options.add_option("ue-trace",
+					   "",
+					   "tracefile",
+					   "Path to write a trace to",
+					   cxxopts::value<std::string>(ServerOptions.TraceOptions.File)->default_value(""),
+					   "");
+#endif	// ZEN_WITH_TRACE
+
+	options.add_option("stats",
+					   "",
+					   "statsd",
+					   "",
+					   cxxopts::value<bool>(ServerOptions.StatsConfig.Enabled)->default_value("false"),
+					   "Enable statsd reporter (localhost:8125)");
+}
+
+void
+ZenServerCmdLineOptions::ApplyOptions(ZenServerOptions& ServerOptions)
+{
+	ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir);
+	ServerOptions.DataDir		= MakeSafeAbsolutePath(DataDir);
+	ServerOptions.ContentDir	= MakeSafeAbsolutePath(ContentDir);
+	ServerOptions.AbsLogFile	= MakeSafeAbsolutePath(AbsLogFile);
+	ServerOptions.ConfigFile	= MakeSafeAbsolutePath(ConfigFile);
+}
+
+void
+ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions)
+{
+	for (int i = 0; i < argc; ++i)
+	{
+		if (i)
+		{
+			ServerOptions.CommandLine.push_back(' ');
+		}
+
+		ServerOptions.CommandLine += argv[i];
+	}
+
+	cxxopts::Options options("zenserver", "Zen Storage Server");
+
+	ZenServerCmdLineOptions BaseOptions;
+	BaseOptions.AddCliOptions(options, ServerOptions);
+
+	ZenStorageServerCmdLineOptions StorageOptions;
+	StorageOptions.AddCliOptions(options, ServerOptions);
+
+	try
+	{
+		cxxopts::ParseResult Result;
+
+		try
+		{
+			Result = options.parse(argc, argv);
+		}
+		catch (const std::exception& Ex)
+		{
+			throw OptionParseException(Ex.what(), options.help());
+		}
+
+		if (Result.count("help"))
+		{
+			ZEN_CONSOLE("{}", options.help());
+
+#if ZEN_PLATFORM_WINDOWS
+			ZEN_CONSOLE("Press any key to exit!");
+			_getch();
+#else
+			// Assume the user's in a terminal on all other platforms and that
+			// they'll use less/more/etc. if need be.
+#endif
+
+			exit(0);
+		}
+
+		if (!ServerOptions.HasTraceCommandlineOptions)
+		{
+			// Apply any Lua settings if we don't have them set from the command line
+			TraceConfigure(ServerOptions.TraceOptions);
+		}
+
+		if (ServerOptions.QuietConsole)
+		{
+			bool HasExplicitConsoleLevel = false;
+			for (int i = 0; i < logging::level::LogLevelCount; ++i)
+			{
+				if (ServerOptions.Loggers[i].find("console") != std::string::npos)
+				{
+					HasExplicitConsoleLevel = true;
+					break;
+				}
+			}
+
+			if (!HasExplicitConsoleLevel)
+			{
+				std::string& WarnLoggers = ServerOptions.Loggers[logging::level::Warn];
+				if (!WarnLoggers.empty())
+				{
+					WarnLoggers += ",";
+				}
+				WarnLoggers += "console";
+			}
+		}
+
+		for (int i = 0; i < logging::level::LogLevelCount; ++i)
+		{
+			logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]);
+		}
+		logging::RefreshLogLevels();
+
+		BaseOptions.ApplyOptions(ServerOptions);
+		StorageOptions.ApplyOptions(options, ServerOptions);
+
+		ParseEnvVariables(ServerOptions, Result);
+
+		ZEN_TRACE_CPU("ConfigParse");
+
+		if (!ServerOptions.ConfigFile.empty())
+		{
+			ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, BaseOptions.OutputConfigFile);
+		}
+		else
+		{
+			ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, BaseOptions.OutputConfigFile);
+		}
+
+		if (!ServerOptions.PluginsConfigFile.empty())
+		{
+			ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort);
+		}
+
+		ValidateOptions(ServerOptions);
+	}
+	catch (const OptionParseException& e)
+	{
+		ZEN_CONSOLE("{}\n", options.help());
+		ZEN_CONSOLE_ERROR("Invalid zenserver arguments: {}", e.what());
+		throw;
+	}
+
+	if (ServerOptions.SystemRootDir.empty())
+	{
+		ServerOptions.SystemRootDir = PickDefaultSystemRootDirectory();
+	}
+
+	if (ServerOptions.DataDir.empty())
+	{
+		ServerOptions.DataDir = PickDefaultStateDirectory(ServerOptions.SystemRootDir);
+	}
+
+	if (ServerOptions.AbsLogFile.empty())
+	{
+		ServerOptions.AbsLogFile = ServerOptions.DataDir / "logs" / "zenserver.log";
+	}
+
+	ServerOptions.HttpServerConfig.IsDedicatedServer = ServerOptions.IsDedicated;
+}
+
+}  // namespace zen