move all storage-related services into storage tree (#571)

* move all storage-related services into storage tree
* move config into config/
* also move admin service into storage since it mostly has storage related functionality
* header consolidation

1 files changed, 0 insertions, 512 deletions

diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
deleted file mode 100644
index 8ee50cee2..000000000
--- a/src/zenserver/config.cpp
+++ /dev/null
@@ -1,512 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "config.h"
-
-#include "storageconfig.h"
-
-#include "config/luaconfig.h"
-#include "diag/logging.h"
-
-#include <zencore/basicfile.h>
-#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinaryutil.h>
-#include <zencore/compactbinaryvalidation.h>
-#include <zencore/except.h>
-#include <zencore/fmtutils.h>
-#include <zencore/iobuffer.h>
-#include <zencore/logging.h>
-#include <zencore/string.h>
-#include <zenutil/commandlineoptions.h>
-#include <zenutil/environmentoptions.h>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <fmt/format.h>
-#include <cxxopts.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
-
-#if ZEN_PLATFORM_WINDOWS
-#	include <conio.h>
-#else
-#	include <unistd.h>
-#endif
-
-namespace zen {
-
-std::filesystem::path
-PickDefaultStateDirectory(std::filesystem::path SystemRoot)
-{
-	if (SystemRoot.empty())
-		return SystemRoot;
-
-	return SystemRoot / "Data";
-}
-
-void
-EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath)
-{
-	CbObjectWriter Cbo;
-	Cbo << "path" << ManifestPath.generic_wstring();
-	Cbo << "manifest" << Manifest;
-
-	const std::filesystem::path StatesPath = SystemRoot / "States";
-
-	CreateDirectories(StatesPath);
-	WriteFile(StatesPath / fmt::format("{}", Identifier), Cbo.Save().GetBuffer().AsIoBuffer());
-}
-
-std::vector<CbObject>
-ReadAllCentralManifests(const std::filesystem::path& SystemRoot)
-{
-	std::vector<CbObject> Manifests;
-
-	DirectoryContent Content;
-	GetDirectoryContent(SystemRoot / "States", DirectoryContentFlags::IncludeFiles, Content);
-
-	for (std::filesystem::path& File : Content.Files)
-	{
-		try
-		{
-			FileContents	FileData = ReadFile(File);
-			CbValidateError ValidateError;
-			if (CbObject Manifest = ValidateAndReadCompactBinaryObject(FileData.Flatten(), ValidateError);
-				ValidateError == CbValidateError::None)
-			{
-				Manifests.emplace_back(std::move(Manifest));
-			}
-			else
-			{
-				ZEN_WARN("failed to load manifest '{}': {}", File, ToString(ValidateError));
-			}
-		}
-		catch (const std::exception& Ex)
-		{
-			ZEN_WARN("failed to load manifest '{}': {}", File, Ex.what());
-		}
-	}
-
-	return Manifests;
-}
-
-void
-ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult)
-{
-	using namespace std::literals;
-
-	EnvironmentOptions Options;
-	Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
-	Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
-	Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv);
-
-	bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable;
-	Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv);
-
-	Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv);
-
-	Options.Parse(CmdLineResult);
-
-	if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable)
-	{
-		ServerOptions.SentryConfig.Disable = !EnvEnableSentry;
-	}
-}
-
-void
-AddServerConfigOptions(LuaConfig::Options& LuaOptions, ZenServerOptions& ServerOptions)
-{
-	using namespace std::literals;
-
-	// server
-
-	LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv);
-	LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv);
-	LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv);
-	LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
-	LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
-	LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv);
-	LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv);
-	LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv);
-	LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
-	LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
-	LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv);
-	LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv);
-	LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv);
-	LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv);
-	LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv);
-
-	////// network
-	LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv);
-	LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv);
-	LuaOptions.AddOption("network.port"sv, ServerOptions.BasePort, "port"sv);
-	LuaOptions.AddOption("network.forceloopback"sv, ServerOptions.HttpServerConfig.ForceLoopback, "http-forceloopback"sv);
-
-#if ZEN_WITH_HTTPSYS
-	LuaOptions.AddOption("network.httpsys.async.workthreads"sv,
-						 ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount,
-						 "httpsys-async-work-threads"sv);
-	LuaOptions.AddOption("network.httpsys.async.response"sv,
-						 ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled,
-						 "httpsys-enable-async-response"sv);
-	LuaOptions.AddOption("network.httpsys.requestlogging"sv,
-						 ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled,
-						 "httpsys-enable-request-logging"sv);
-#endif
-
-#if ZEN_WITH_TRACE
-	////// trace
-	LuaOptions.AddOption("trace.channels"sv, ServerOptions.TraceOptions.Channels, "trace"sv);
-	LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceOptions.Host, "tracehost"sv);
-	LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceOptions.File, "tracefile"sv);
-#endif
-
-	////// stats
-	LuaOptions.AddOption("stats.enable"sv, ServerOptions.StatsConfig.Enabled, "statsd"sv);
-	LuaOptions.AddOption("stats.host"sv, ServerOptions.StatsConfig.StatsdHost);
-	LuaOptions.AddOption("stats.port"sv, ServerOptions.StatsConfig.StatsdPort);
-}
-
-struct ZenServerCmdLineOptions
-{
-	// Note to those adding future options; std::filesystem::path-type options
-	// must be read into a std::string first. As of cxxopts-3.0.0 it uses a >>
-	// stream operator to convert argv value into the options type. std::fs::path
-	// expects paths in streams to be quoted but argv paths are unquoted. By
-	// going into a std::string first, paths with whitespace parse correctly.
-	std::string ConfigFile;
-	std::string OutputConfigFile;
-	std::string SystemRootDir;
-	std::string ContentDir;
-	std::string DataDir;
-	std::string AbsLogFile;
-
-	void AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions);
-	void ApplyOptions(ZenServerOptions& ServerOptions);
-};
-
-void
-ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions)
-{
-	const char* DefaultHttp = "asio";
-
-#if ZEN_WITH_HTTPSYS
-	if (!windows::IsRunningOnWine())
-	{
-		DefaultHttp = "httpsys";
-	}
-#endif
-
-	options.add_options()("dedicated",
-						  "Enable dedicated server mode",
-						  cxxopts::value<bool>(ServerOptions.IsDedicated)->default_value("false"));
-	options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(ServerOptions.IsDebug)->default_value("false"));
-	options.add_options()("clean",
-						  "Clean out all state at startup",
-						  cxxopts::value<bool>(ServerOptions.IsCleanStart)->default_value("false"));
-	options.add_options()("help", "Show command line help");
-	options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
-
-	options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir));
-	options.add_options()("system-dir", "Specify system root", cxxopts::value<std::string>(SystemRootDir));
-	options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::string>(ContentDir));
-	options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
-	options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile));
-
-	options.add_options()("no-sentry",
-						  "Disable Sentry crash handler",
-						  cxxopts::value<bool>(ServerOptions.SentryConfig.Disable)->default_value("false"));
-	options.add_options()("sentry-allow-personal-info",
-						  "Allow personally identifiable information in sentry crash reports",
-						  cxxopts::value<bool>(ServerOptions.SentryConfig.AllowPII)->default_value("false"));
-	options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(ServerOptions.SentryConfig.Dsn));
-	options.add_options()("sentry-environment", "Sentry environment", cxxopts::value<std::string>(ServerOptions.SentryConfig.Environment));
-	options.add_options()("sentry-debug",
-						  "Enable debug mode for Sentry",
-						  cxxopts::value<bool>(ServerOptions.SentryConfig.Debug)->default_value("false"));
-	options.add_options()("detach",
-						  "Indicate whether zenserver should detach from parent process group",
-						  cxxopts::value<bool>(ServerOptions.Detach)->default_value("true"));
-	options.add_options()("malloc",
-						  "Configure memory allocator subsystem",
-						  cxxopts::value(ServerOptions.MemoryOptions)->default_value("mimalloc"));
-	options.add_options()("powercycle",
-						  "Exit immediately after initialization is complete",
-						  cxxopts::value<bool>(ServerOptions.IsPowerCycle));
-
-	options.add_option("diagnostics",
-					   "",
-					   "crash",
-					   "Simulate a crash",
-					   cxxopts::value<bool>(ServerOptions.ShouldCrash)->default_value("false"),
-					   "");
-
-	// clang-format off
-	options.add_options("logging")
-		("abslog",			"Path to log file",									cxxopts::value<std::string>(AbsLogFile))
-		("log-id",			"Specify id for adding context to log output",		cxxopts::value<std::string>(ServerOptions.LogId))
-		("quiet",			"Configure console logger output to level WARN",	cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false"))
-		("noconsole",		"Disable console logging",							cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
-		("log-trace", 		"Change selected loggers to level TRACE", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
-		("log-debug", 		"Change selected loggers to level DEBUG", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
-		("log-info", 		"Change selected loggers to level INFO", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
-		("log-warn", 		"Change selected loggers to level WARN", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
-		("log-error", 		"Change selected loggers to level ERROR", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
-		("log-critical", 	"Change selected loggers to level CRITICAL", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
-		("log-off", 		"Change selected loggers to level OFF", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
-	;
-	// clang-format on
-
-	options
-		.add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value<int>(ServerOptions.OwnerPid), "<identifier>");
-	options.add_option("lifetime",
-					   "",
-					   "child-id",
-					   "Specify id which can be used to signal parent",
-					   cxxopts::value<std::string>(ServerOptions.ChildId),
-					   "<identifier>");
-
-#if ZEN_PLATFORM_WINDOWS
-	options.add_option("lifetime",
-					   "",
-					   "install",
-					   "Install zenserver as a Windows service",
-					   cxxopts::value<bool>(ServerOptions.InstallService),
-					   "");
-	options.add_option("lifetime",
-					   "",
-					   "uninstall",
-					   "Uninstall zenserver as a Windows service",
-					   cxxopts::value<bool>(ServerOptions.UninstallService),
-					   "");
-#endif
-
-	options.add_option("network",
-					   "",
-					   "http-threads",
-					   "Number of http server connection threads",
-					   cxxopts::value<unsigned int>(ServerOptions.HttpServerConfig.ThreadCount),
-					   "<http threads>");
-
-	options.add_option("network",
-					   "p",
-					   "port",
-					   "Select HTTP port",
-					   cxxopts::value<int>(ServerOptions.BasePort)->default_value("8558"),
-					   "<port number>");
-
-	options.add_option("network",
-					   "",
-					   "http-forceloopback",
-					   "Force using local loopback interface",
-					   cxxopts::value<bool>(ServerOptions.HttpServerConfig.ForceLoopback)->default_value("false"),
-					   "<http forceloopback>");
-
-#if ZEN_WITH_HTTPSYS
-	options.add_option("httpsys",
-					   "",
-					   "httpsys-async-work-threads",
-					   "Number of HttpSys async worker threads",
-					   cxxopts::value<unsigned int>(ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount),
-					   "<httpsys workthreads>");
-
-	options.add_option("httpsys",
-					   "",
-					   "httpsys-enable-async-response",
-					   "Enables Httpsys async response",
-					   cxxopts::value<bool>(ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled)->default_value("true"),
-					   "<httpsys async response>");
-
-	options.add_option("httpsys",
-					   "",
-					   "httpsys-enable-request-logging",
-					   "Enables Httpsys request logging",
-					   cxxopts::value<bool>(ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled),
-					   "<httpsys request logging>");
-#endif
-
-	options.add_option("network",
-					   "",
-					   "http",
-					   "Select HTTP server implementation (asio|"
-#if ZEN_WITH_HTTPSYS
-					   "httpsys|"
-#endif
-					   "null)",
-					   cxxopts::value<std::string>(ServerOptions.HttpServerConfig.ServerClass)->default_value(DefaultHttp),
-					   "<http class>");
-
-#if ZEN_WITH_TRACE
-	// We only have this in options for command line help purposes - we parse these argument separately earlier using
-	// GetTraceOptionsFromCommandline()
-
-	options.add_option("ue-trace",
-					   "",
-					   "trace",
-					   "Specify which trace channels should be enabled",
-					   cxxopts::value<std::string>(ServerOptions.TraceOptions.Channels)->default_value(""),
-					   "");
-
-	options.add_option("ue-trace",
-					   "",
-					   "tracehost",
-					   "Hostname to send the trace to",
-					   cxxopts::value<std::string>(ServerOptions.TraceOptions.Host)->default_value(""),
-					   "");
-
-	options.add_option("ue-trace",
-					   "",
-					   "tracefile",
-					   "Path to write a trace to",
-					   cxxopts::value<std::string>(ServerOptions.TraceOptions.File)->default_value(""),
-					   "");
-#endif	// ZEN_WITH_TRACE
-
-	options.add_option("stats",
-					   "",
-					   "statsd",
-					   "",
-					   cxxopts::value<bool>(ServerOptions.StatsConfig.Enabled)->default_value("false"),
-					   "Enable statsd reporter (localhost:8125)");
-}
-
-void
-ZenServerCmdLineOptions::ApplyOptions(ZenServerOptions& ServerOptions)
-{
-	ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir);
-	ServerOptions.DataDir		= MakeSafeAbsolutePath(DataDir);
-	ServerOptions.ContentDir	= MakeSafeAbsolutePath(ContentDir);
-	ServerOptions.AbsLogFile	= MakeSafeAbsolutePath(AbsLogFile);
-	ServerOptions.ConfigFile	= MakeSafeAbsolutePath(ConfigFile);
-}
-
-void
-ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions)
-{
-	for (int i = 0; i < argc; ++i)
-	{
-		if (i)
-		{
-			ServerOptions.CommandLine.push_back(' ');
-		}
-
-		ServerOptions.CommandLine += argv[i];
-	}
-
-	cxxopts::Options options("zenserver", "Zen Storage Server");
-
-	ZenServerCmdLineOptions BaseOptions;
-	BaseOptions.AddCliOptions(options, ServerOptions);
-
-	ZenStorageServerCmdLineOptions StorageOptions;
-	StorageOptions.AddCliOptions(options, ServerOptions);
-
-	try
-	{
-		cxxopts::ParseResult Result;
-
-		try
-		{
-			Result = options.parse(argc, argv);
-		}
-		catch (const std::exception& Ex)
-		{
-			throw OptionParseException(Ex.what(), options.help());
-		}
-
-		if (Result.count("help"))
-		{
-			ZEN_CONSOLE("{}", options.help());
-
-#if ZEN_PLATFORM_WINDOWS
-			ZEN_CONSOLE("Press any key to exit!");
-			_getch();
-#else
-			// Assume the user's in a terminal on all other platforms and that
-			// they'll use less/more/etc. if need be.
-#endif
-
-			exit(0);
-		}
-
-		if (!ServerOptions.HasTraceCommandlineOptions)
-		{
-			// Apply any Lua settings if we don't have them set from the command line
-			TraceConfigure(ServerOptions.TraceOptions);
-		}
-
-		if (ServerOptions.QuietConsole)
-		{
-			bool HasExplicitConsoleLevel = false;
-			for (int i = 0; i < logging::level::LogLevelCount; ++i)
-			{
-				if (ServerOptions.Loggers[i].find("console") != std::string::npos)
-				{
-					HasExplicitConsoleLevel = true;
-					break;
-				}
-			}
-
-			if (!HasExplicitConsoleLevel)
-			{
-				std::string& WarnLoggers = ServerOptions.Loggers[logging::level::Warn];
-				if (!WarnLoggers.empty())
-				{
-					WarnLoggers += ",";
-				}
-				WarnLoggers += "console";
-			}
-		}
-
-		for (int i = 0; i < logging::level::LogLevelCount; ++i)
-		{
-			logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]);
-		}
-		logging::RefreshLogLevels();
-
-		BaseOptions.ApplyOptions(ServerOptions);
-		StorageOptions.ApplyOptions(options, ServerOptions);
-
-		ParseEnvVariables(ServerOptions, Result);
-
-		ZEN_TRACE_CPU("ConfigParse");
-
-		if (!ServerOptions.ConfigFile.empty())
-		{
-			ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, BaseOptions.OutputConfigFile);
-		}
-		else
-		{
-			ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, BaseOptions.OutputConfigFile);
-		}
-
-		if (!ServerOptions.PluginsConfigFile.empty())
-		{
-			ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort);
-		}
-
-		ValidateOptions(ServerOptions);
-	}
-	catch (const OptionParseException& e)
-	{
-		ZEN_CONSOLE("{}\n", options.help());
-		ZEN_CONSOLE_ERROR("Invalid zenserver arguments: {}", e.what());
-		throw;
-	}
-
-	if (ServerOptions.SystemRootDir.empty())
-	{
-		ServerOptions.SystemRootDir = PickDefaultSystemRootDirectory();
-	}
-
-	if (ServerOptions.DataDir.empty())
-	{
-		ServerOptions.DataDir = PickDefaultStateDirectory(ServerOptions.SystemRootDir);
-	}
-
-	if (ServerOptions.AbsLogFile.empty())
-	{
-		ServerOptions.AbsLogFile = ServerOptions.DataDir / "logs" / "zenserver.log";
-	}
-
-	ServerOptions.HttpServerConfig.IsDedicatedServer = ServerOptions.IsDedicated;
-}
-
-}  // namespace zen