// Copyright Epic Games, Inc. All Rights Reserved. #include "config.h" #include "diag/logging.h" #include #include #include #include #include #include #include ZEN_THIRD_PARTY_INCLUDES_START #include #include #include #include ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_PLATFORM_WINDOWS # include #else # include # include #endif #include #include #if ZEN_PLATFORM_WINDOWS // Used for getting My Documents for default data directory # include # pragma comment(lib, "shell32.lib") # pragma comment(lib, "ole32.lib") namespace zen { std::filesystem::path PickDefaultStateDirectory() { // Pick sensible default PWSTR ProgramDataDir = nullptr; HRESULT hRes = SHGetKnownFolderPath(FOLDERID_ProgramData, 0, NULL, &ProgramDataDir); if (SUCCEEDED(hRes)) { std::filesystem::path FinalPath(ProgramDataDir); FinalPath /= L"Epic\\Zen\\Data"; ::CoTaskMemFree(ProgramDataDir); return FinalPath; } return L""; } } // namespace zen #else namespace zen { std::filesystem::path PickDefaultStateDirectory() { int UserId = getuid(); const passwd* Passwd = getpwuid(UserId); return std::filesystem::path(Passwd->pw_dir) / ".zen"; } } // namespace zen #endif namespace zen { void ValidateOptions(ZenServerOptions& ServerOptions) { if (ServerOptions.EncryptionKey.empty() == false) { const auto Key = zen::AesKey256Bit::FromString(ServerOptions.EncryptionKey); if (Key.IsValid() == false) { throw zen::OptionParseException("Invalid AES encryption key"); } } if (ServerOptions.EncryptionIV.empty() == false) { const auto IV = zen::AesIV128Bit::FromString(ServerOptions.EncryptionIV); if (IV.IsValid() == false) { throw zen::OptionParseException("Invalid AES initialization vector"); } } if (ServerOptions.HttpServerConfig.ForceLoopback && ServerOptions.IsDedicated) { throw zen::OptionParseException("Dedicated server can not be used with forced local server address"); } } UpstreamCachePolicy ParseUpstreamCachePolicy(std::string_view Options) { if (Options == "readonly") { return UpstreamCachePolicy::Read; } else if (Options == "writeonly") { return UpstreamCachePolicy::Write; } else if (Options == "disabled") { return UpstreamCachePolicy::Disabled; } else { return UpstreamCachePolicy::ReadWrite; } } ZenObjectStoreConfig ParseBucketConfigs(std::span Buckets) { using namespace std::literals; ZenObjectStoreConfig Cfg; // split bucket args in the form of "{BucketName};{LocalPath}" for (std::string_view Bucket : Buckets) { ZenObjectStoreConfig::BucketConfig NewBucket; if (auto Idx = Bucket.find_first_of(";"); Idx != std::string_view::npos) { NewBucket.Name = Bucket.substr(0, Idx); NewBucket.Directory = Bucket.substr(Idx + 1); } else { NewBucket.Name = Bucket; } Cfg.Buckets.push_back(std::move(NewBucket)); } return Cfg; } static std::string MakeSafePath(const std::string_view Path) { #if ZEN_PLATFORM_WINDOWS if (Path.empty()) { return std::string(Path); } std::string FixedPath(Path); std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\'); if (!FixedPath.starts_with("\\\\?\\")) { FixedPath.insert(0, "\\\\?\\"); } return FixedPath; #else return std::string(Path); #endif }; namespace LuaConfig { void EscapeBackslash(std::string& InOutString) { std::size_t BackslashPos = InOutString.find('\\'); if (BackslashPos != std::string::npos) { std::size_t Offset = 0; zen::ExtendableStringBuilder<512> PathBuilder; while (BackslashPos != std::string::npos) { PathBuilder.Append(InOutString.substr(Offset, BackslashPos + 1 - Offset)); PathBuilder.Append('\\'); Offset = BackslashPos + 1; BackslashPos = InOutString.find('\\', Offset); } PathBuilder.Append(InOutString.substr(Offset, BackslashPos)); InOutString = PathBuilder.ToString(); } } class OptionValue { public: virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) = 0; virtual void Parse(sol::object Object) = 0; virtual ~OptionValue() {} }; typedef std::shared_ptr TOptionValue; class StringOption : public OptionValue { public: StringOption(std::string& Value) : Value(Value) {} virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override { StringBuilder.Append(fmt::format("\"{}\"", Value)); } virtual void Parse(sol::object Object) override { Value = Object.as(); } std::string& Value; }; class FilePathOption : public OptionValue { public: FilePathOption(std::filesystem::path& Value) : Value(Value) {} virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override { std::string Path = Value.string(); EscapeBackslash(Path); StringBuilder.Append(fmt::format("\"{}\"", Path)); } virtual void Parse(sol::object Object) override { std::string Str = Object.as(); if (!Str.empty()) { Value = MakeSafePath(Str); } } std::filesystem::path& Value; }; class BoolOption : public OptionValue { public: BoolOption(bool& Value) : Value(Value) {} virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override { StringBuilder.Append(Value ? "true" : "false"); } virtual void Parse(sol::object Object) override { Value = Object.as(); } bool& Value; }; class CachePolicyOption : public OptionValue { public: CachePolicyOption(UpstreamCachePolicy& Value) : Value(Value) {} virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override { switch (Value) { case UpstreamCachePolicy::Read: StringBuilder.Append("readonly"); break; case UpstreamCachePolicy::Write: StringBuilder.Append("writeonly"); break; case UpstreamCachePolicy::Disabled: StringBuilder.Append("disabled"); break; case UpstreamCachePolicy::ReadWrite: StringBuilder.Append("readwrite"); break; default: ZEN_ASSERT(false); } } virtual void Parse(sol::object Object) override { std::string PolicyString = Object.as(); if (PolicyString == "readonly") { Value = UpstreamCachePolicy::Read; } else if (PolicyString == "writeonly") { Value = UpstreamCachePolicy::Write; } else if (PolicyString == "disabled") { Value = UpstreamCachePolicy::Disabled; } else if (PolicyString == "readwrite") { Value = UpstreamCachePolicy::ReadWrite; } } UpstreamCachePolicy& Value; }; template class NumberOption : public OptionValue { public: NumberOption(T& Value) : Value(Value) {} virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override { StringBuilder.Append(fmt::format("{}", Value)); } virtual void Parse(sol::object Object) override { Value = Object.as(); } T& Value; }; class LuaContainerWriter { public: LuaContainerWriter(zen::StringBuilderBase& StringBuilder, std::string_view Indent) : StringBuilder(StringBuilder) , InitialIndent(Indent.length()) , LocalIndent(Indent) { StringBuilder.Append("{\n"); LocalIndent.push_back('\t'); } ~LuaContainerWriter() { LocalIndent.pop_back(); StringBuilder.Append(LocalIndent); StringBuilder.Append("}"); } void BeginContainer(std::string_view Name) { StringBuilder.Append(LocalIndent); if (!Name.empty()) { StringBuilder.Append(Name); StringBuilder.Append(" = {\n"); } else { StringBuilder.Append("{\n"); } LocalIndent.push_back('\t'); } void WriteValue(std::string_view Name, std::string_view Value) { if (Name.empty()) { StringBuilder.Append(fmt::format("{}\"{}\",\n", LocalIndent, Value)); } else { StringBuilder.Append(fmt::format("{}{} = \"{}\",\n", LocalIndent, Name, Value)); } } void EndContainer() { LocalIndent.pop_back(); StringBuilder.Append(LocalIndent); StringBuilder.Append("}"); StringBuilder.Append(",\n"); } private: zen::StringBuilderBase& StringBuilder; const std::size_t InitialIndent; std::string LocalIndent; }; class StringArrayOption : public OptionValue { public: StringArrayOption(std::vector& Value) : Value(Value) {} virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override { if (Value.empty()) { StringBuilder.Append("{}"); } if (Value.size() == 1) { StringBuilder.Append(fmt::format("\"{}\"", Value[0])); } else { LuaContainerWriter Writer(StringBuilder, Indent); for (std::string String : Value) { Writer.WriteValue("", String); } } } virtual void Parse(sol::object Object) override { if (Object.get_type() == sol::type::string) { Value.push_back(Object.as()); } else if (Object.get_type() == sol::type::table) { for (const auto& Kv : Object.as()) { Value.push_back(Kv.second.as()); } } } private: std::vector& Value; }; class ZenAuthConfigOption : public OptionValue { public: ZenAuthConfigOption(ZenAuthConfig& Value) : Value(Value) {} virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override { if (Value.OpenIdProviders.empty()) { StringBuilder.Append("{}"); return; } LuaContainerWriter Writer(StringBuilder, Indent); for (const ZenOpenIdProviderConfig& Config : Value.OpenIdProviders) { Writer.BeginContainer(""); { Writer.WriteValue("name", Config.Name); Writer.WriteValue("url", Config.Url); Writer.WriteValue("clientid", Config.ClientId); } Writer.EndContainer(); } } virtual void Parse(sol::object Object) override { if (sol::optional OpenIdProviders = Object.as()) { for (const auto& Kv : OpenIdProviders.value()) { if (sol::optional OpenIdProvider = Kv.second.as()) { std::string Name = OpenIdProvider.value().get_or("name", std::string("Default")); std::string Url = OpenIdProvider.value().get_or("url", std::string()); std::string ClientId = OpenIdProvider.value().get_or("clientid", std::string()); Value.OpenIdProviders.push_back({.Name = std::move(Name), .Url = std::move(Url), .ClientId = std::move(ClientId)}); } } } } ZenAuthConfig& Value; }; class ZenObjectStoreConfigOption : public OptionValue { public: ZenObjectStoreConfigOption(ZenObjectStoreConfig& Value) : Value(Value) {} virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override { if (Value.Buckets.empty()) { StringBuilder.Append("{}"); return; } LuaContainerWriter Writer(StringBuilder, Indent); for (const ZenObjectStoreConfig::BucketConfig& Config : Value.Buckets) { Writer.BeginContainer(""); { Writer.WriteValue("name", Config.Name); std::string Directory = Config.Directory.string(); EscapeBackslash(Directory); Writer.WriteValue("directory", Directory); } Writer.EndContainer(); } } virtual void Parse(sol::object Object) override { if (sol::optional Buckets = Object.as()) { for (const auto& Kv : Buckets.value()) { if (sol::optional Bucket = Kv.second.as()) { std::string Name = Bucket.value().get_or("name", std::string("Default")); std::string Directory = Bucket.value().get_or("directory", std::string()); Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafePath(Directory)}); } } } } ZenObjectStoreConfig& Value; }; std::shared_ptr MakeOption(std::string& Value) { return std::make_shared(Value); }; std::shared_ptr MakeOption(std::filesystem::path& Value) { return std::make_shared(Value); }; template std::shared_ptr MakeOption(T& Value) { return std::make_shared>(Value); }; std::shared_ptr MakeOption(bool& Value) { return std::make_shared(Value); }; std::shared_ptr MakeOption(UpstreamCachePolicy& Value) { return std::make_shared(Value); }; std::shared_ptr MakeOption(std::vector& Value) { return std::make_shared(Value); }; std::shared_ptr MakeOption(ZenAuthConfig& Value) { return std::make_shared(Value); }; std::shared_ptr MakeOption(ZenObjectStoreConfig& Value) { return std::make_shared(Value); }; struct Option { std::string CommandLineOptionName; TOptionValue Value; }; struct Options { public: template void AddOption(std::string_view Key, T& Value, std::string_view CommandLineOptionName = "") { OptionMap.insert_or_assign(std::string(Key), Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); }; void Parse(const std::filesystem::path& Path, const cxxopts::ParseResult& CmdLineResult) { zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path); if (LuaScript) { sol::state lua; lua.open_libraries(sol::lib::base); lua.set_function("getenv", [&](const std::string env) -> sol::object { #if ZEN_PLATFORM_WINDOWS std::wstring EnvVarValue; size_t RequiredSize = 0; std::wstring EnvWide = zen::Utf8ToWide(env); _wgetenv_s(&RequiredSize, nullptr, 0, EnvWide.c_str()); if (RequiredSize == 0) return sol::make_object(lua, sol::lua_nil); EnvVarValue.resize(RequiredSize); _wgetenv_s(&RequiredSize, EnvVarValue.data(), RequiredSize, EnvWide.c_str()); return sol::make_object(lua, zen::WideToUtf8(EnvVarValue.c_str())); #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC char* EnvVariable = getenv(env.c_str()); if (EnvVariable == nullptr) { return sol::make_object(lua, sol::lua_nil); } return sol::make_object(lua, EnvVariable); #else ZEN_UNUSED(env); return sol::make_object(lua, sol::lua_nil); #endif }); try { sol::load_result config = lua.load(std::string_view((const char*)LuaScript.Data(), LuaScript.Size()), "zen_cfg"); if (!config.valid()) { sol::error err = config; std::string ErrorString = sol::to_string(config.status()); throw std::runtime_error(fmt::format("{} error: {}", ErrorString, err.what())); } config(); } catch (std::exception& e) { throw std::runtime_error(fmt::format("failed to load config script ('{}'): {}", Path, e.what()).c_str()); } Parse(lua, CmdLineResult); } } void Parse(const sol::state& LuaState, const cxxopts::ParseResult& CmdLineResult) { for (auto It : LuaState) { sol::object Key = It.first; sol::type KeyType = Key.get_type(); if (KeyType == sol::type::string) { sol::type ValueType = It.second.get_type(); switch (ValueType) { case sol::type::table: { std::string Name = Key.as(); if (Name.starts_with("_")) { continue; } if (Name == "base") { continue; } Traverse(It.second.as(), Name, CmdLineResult); } break; default: break; } } } } void Touch(std::string_view Key) { UsedKeys.insert(std::string(Key)); } void Print(zen::StringBuilderBase& SB, const cxxopts::ParseResult& CmdLineResult) { for (auto It : OptionMap) { if (CmdLineResult.count(It.second.CommandLineOptionName) != 0) { UsedKeys.insert(It.first); } } std::vector SortedKeys(UsedKeys.begin(), UsedKeys.end()); std::sort(SortedKeys.begin(), SortedKeys.end()); auto GetTablePath = [](const std::string& Key) -> std::vector { std::vector Path; zen::ForEachStrTok(Key, '.', [&Path](std::string_view Part) { Path.push_back(std::string(Part)); return true; }); return Path; }; std::vector CurrentTablePath; std::string Indent; auto It = SortedKeys.begin(); for (const std::string& Key : SortedKeys) { std::vector KeyPath = GetTablePath(Key); std::string Name = KeyPath.back(); KeyPath.pop_back(); if (CurrentTablePath != KeyPath) { size_t EqualCount = 0; while (EqualCount < CurrentTablePath.size() && EqualCount < KeyPath.size() && CurrentTablePath[EqualCount] == KeyPath[EqualCount]) { EqualCount++; } while (CurrentTablePath.size() > EqualCount) { CurrentTablePath.pop_back(); Indent.pop_back(); SB.Append(Indent); SB.Append("}"); if (CurrentTablePath.size() == EqualCount && !Indent.empty() && KeyPath.size() >= EqualCount) { SB.Append(","); } SB.Append("\n"); if (Indent.empty()) { SB.Append("\n"); } } while (EqualCount < KeyPath.size()) { SB.Append(Indent); SB.Append(KeyPath[EqualCount]); SB.Append(" = {\n"); Indent.push_back('\t'); CurrentTablePath.push_back(KeyPath[EqualCount]); EqualCount++; } } SB.Append(Indent); SB.Append(Name); SB.Append(" = "); OptionMap[Key].Value->Print(Indent, SB); SB.Append(",\n"); } while (!CurrentTablePath.empty()) { Indent.pop_back(); SB.Append(Indent); SB.Append("}\n"); CurrentTablePath.pop_back(); } } private: void Traverse(sol::table Table, std::string_view PathPrefix, const cxxopts::ParseResult& CmdLineResult) { for (auto It : Table) { sol::object Key = It.first; sol::type KeyType = Key.get_type(); if (KeyType == sol::type::string || KeyType == sol::type::number) { sol::type ValueType = It.second.get_type(); switch (ValueType) { case sol::type::table: case sol::type::string: case sol::type::number: case sol::type::boolean: { std::string Name = Key.as(); if (Name.starts_with("_")) { continue; } Name = std::string(PathPrefix) + "." + Key.as(); auto OptionIt = OptionMap.find(Name); if (OptionIt != OptionMap.end()) { UsedKeys.insert(Name); if (CmdLineResult.count(OptionIt->second.CommandLineOptionName) != 0) { continue; } OptionIt->second.Value->Parse(It.second); continue; } if (ValueType == sol::type::table) { if (Name == "base") { continue; } Traverse(It.second.as(), Name, CmdLineResult); } } break; default: break; } } } } std::unordered_map OptionMap; std::unordered_set UsedKeys; }; } // namespace LuaConfig void ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult, std::string_view OutputConfigFile) { using namespace std::literals; LuaConfig::Options LuaOptions; ////// server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv); LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv); LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv); LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "quiet"sv); ////// objectstore LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); ////// network LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv); LuaOptions.AddOption("network.port"sv, ServerOptions.BasePort, "port"sv); LuaOptions.AddOption("network.forceloopback"sv, ServerOptions.HttpServerConfig.ForceLoopback, "http-forceloopback"sv); LuaOptions.AddOption("network.httpsys.async.workthreads"sv, ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount, "httpsys-async-work-threads"sv); LuaOptions.AddOption("network.httpsys.async.response"sv, ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled, "httpsys-enable-async-response"sv); LuaOptions.AddOption("network.httpsys.requestlogging"sv, ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled, "httpsys-enable-request-logging"sv); ////// trace LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceHost, "tracehost"sv); LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceFile, "tracefile"sv); ////// stats LuaOptions.AddOption("stats.enable"sv, ServerOptions.StatsConfig.Enabled); LuaOptions.AddOption("stats.host"sv, ServerOptions.StatsConfig.StatsdHost); LuaOptions.AddOption("stats.port"sv, ServerOptions.StatsConfig.StatsdPort); ////// cache LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled); LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log"); LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"); LuaOptions.AddOption("cache.referencecache"sv, ServerOptions.StructuredCacheConfig.EnableReferenceCaching, "cache-reference-cache-enabled"); LuaOptions.AddOption("cache.memlayer.sizethreshold"sv, ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold, "cache-memlayer-sizethreshold"); LuaOptions.AddOption("cache.memlayer.targetfootprint"sv, ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes, "cache-memlayer-targetfootprint"); LuaOptions.AddOption("cache.memlayer.triminterval"sv, ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds, "cache-memlayer-triminterval"); LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"); ////// cache.upstream LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv); LuaOptions.AddOption("cache.upstream.upstreamthreadcount"sv, ServerOptions.UpstreamCacheConfig.UpstreamThreadCount, "upstream-thread-count"sv); LuaOptions.AddOption("cache.upstream.connecttimeoutms"sv, ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds, "upstream-connect-timeout-ms"sv); LuaOptions.AddOption("cache.upstream.timeoutms"sv, ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds, "upstream-timeout-ms"sv); ////// cache.upstream.jupiter LuaOptions.AddOption("cache.upstream.jupiter.name"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Name); LuaOptions.AddOption("cache.upstream.jupiter.url"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Url, "upstream-jupiter-url"sv); LuaOptions.AddOption("cache.upstream.jupiter.oauthprovider"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl, "upstream-jupiter-oauth-url"sv); LuaOptions.AddOption("cache.upstream.jupiter.oauthclientid"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId, "upstream-jupiter-oauth-clientid"); LuaOptions.AddOption("cache.upstream.jupiter.oauthclientsecret"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret, "upstream-jupiter-oauth-clientsecret"sv); LuaOptions.AddOption("cache.upstream.jupiter.openidprovider"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider, "upstream-jupiter-openid-provider"sv); LuaOptions.AddOption("cache.upstream.jupiter.token"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken, "upstream-jupiter-token"sv); LuaOptions.AddOption("cache.upstream.jupiter.namespace"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace, "upstream-jupiter-namespace"sv); LuaOptions.AddOption("cache.upstream.jupiter.ddcnamespace"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace, "upstream-jupiter-namespace-ddc"sv); ////// cache.upstream.zen // LuaOptions.AddOption("cache.upstream.zen"sv, ServerOptions.UpstreamCacheConfig.ZenConfig); LuaOptions.AddOption("cache.upstream.zen.name"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Name); LuaOptions.AddOption("cache.upstream.zen.dns"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Dns); LuaOptions.AddOption("cache.upstream.zen.url"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Urls); LuaOptions.AddOption("gc.enabled"sv, ServerOptions.GcConfig.Enabled, "gc-enabled"sv); LuaOptions.AddOption("gc.v2"sv, ServerOptions.GcConfig.UseGCV2, "gc-v2"sv); LuaOptions.AddOption("gc.monitorintervalseconds"sv, ServerOptions.GcConfig.MonitorIntervalSeconds, "gc-monitor-interval-seconds"sv); LuaOptions.AddOption("gc.intervalseconds"sv, ServerOptions.GcConfig.IntervalSeconds, "gc-interval-seconds"sv); LuaOptions.AddOption("gc.collectsmallobjects"sv, ServerOptions.GcConfig.CollectSmallObjects, "gc-small-objects"sv); LuaOptions.AddOption("gc.diskreservesize"sv, ServerOptions.GcConfig.DiskReserveSize, "disk-reserve-size"sv); LuaOptions.AddOption("gc.disksizesoftlimit"sv, ServerOptions.GcConfig.DiskSizeSoftLimit, "gc-disksize-softlimit"sv); LuaOptions.AddOption("gc.lowdiskspacethreshold"sv, ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, "gc-low-diskspace-threshold"sv); LuaOptions.AddOption("gc.lightweightntervalseconds"sv, ServerOptions.GcConfig.LightweightIntervalSeconds, "gc-lightweight-interval-seconds"sv); LuaOptions.AddOption("gc.compactblockthreshold"sv, ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, "gc-compactblock-threshold"sv); LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv); ////// gc LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv); ////// security LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); LuaOptions.AddOption("security.encryptionaesiv"sv, ServerOptions.EncryptionIV, "encryption-aes-iv"sv); LuaOptions.AddOption("security.openidproviders"sv, ServerOptions.AuthConfig); LuaOptions.Parse(Path, CmdLineResult); // These have special command line processing so we make sure we export them if they were configured on command line if (!ServerOptions.AuthConfig.OpenIdProviders.empty()) { LuaOptions.Touch("security.openidproviders"sv); } if (!ServerOptions.ObjectStoreConfig.Buckets.empty()) { LuaOptions.Touch("server.objectstore.buckets"sv); } if (!OutputConfigFile.empty()) { std::filesystem::path WritePath(MakeSafePath(OutputConfigFile)); zen::ExtendableStringBuilder<512> ConfigStringBuilder; LuaOptions.Print(ConfigStringBuilder, CmdLineResult); zen::BasicFile Output; Output.Open(WritePath, zen::BasicFile::Mode::kTruncate); Output.Write(ConfigStringBuilder.Data(), ConfigStringBuilder.Size(), 0); } } void ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) { #if ZEN_WITH_HTTPSYS const char* DefaultHttp = "httpsys"; #else const char* DefaultHttp = "asio"; #endif // Note to those adding future options; std::filesystem::path-type options // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> // stream operator to convert argv value into the options type. std::fs::path // expects paths in streams to be quoted but argv paths are unquoted. By // going into a std::string first, paths with whitespace parse correctly. std::string DataDir; std::string ContentDir; std::string AbsLogFile; std::string ConfigFile; std::string OutputConfigFile; std::string BaseSnapshotDir; cxxopts::Options options("zenserver", "Zen Server"); options.add_options()("dedicated", "Enable dedicated server mode", cxxopts::value(ServerOptions.IsDedicated)->default_value("false")); options.add_options()("d, debug", "Enable debugging", cxxopts::value(ServerOptions.IsDebug)->default_value("false")); options.add_options()("clean", "Clean out all state at startup", cxxopts::value(ServerOptions.IsCleanStart)->default_value("false")); options.add_options()("help", "Show command line help"); options.add_options()("t, test", "Enable test mode", cxxopts::value(ServerOptions.IsTest)->default_value("false")); options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value(ServerOptions.LogId)); options.add_options()("data-dir", "Specify persistence root", cxxopts::value(DataDir)); options.add_options()("snapshot-dir", "Specify a snapshot of server state to mirror into the persistence root at startup", cxxopts::value(BaseSnapshotDir)); options.add_options()("content-dir", "Frontend content directory", cxxopts::value(ContentDir)); options.add_options()("abslog", "Path to log file", cxxopts::value(AbsLogFile)); options.add_options()("config", "Path to Lua config file", cxxopts::value(ConfigFile)); options.add_options()("write-config", "Path to output Lua config file", cxxopts::value(OutputConfigFile)); options.add_options()("no-sentry", "Disable Sentry crash handler", cxxopts::value(ServerOptions.NoSentry)->default_value("false")); options.add_options()("sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", cxxopts::value(ServerOptions.SentryAllowPII)->default_value("false")); options.add_options()("quiet", "Disable console logging", cxxopts::value(ServerOptions.NoConsoleOutput)->default_value("false")); options.add_option("security", "", "encryption-aes-key", "256 bit AES encryption key", cxxopts::value(ServerOptions.EncryptionKey), ""); options.add_option("security", "", "encryption-aes-iv", "128 bit AES encryption initialization vector", cxxopts::value(ServerOptions.EncryptionIV), ""); std::string OpenIdProviderName; options.add_option("security", "", "openid-provider-name", "Open ID provider name", cxxopts::value(OpenIdProviderName), "Default"); std::string OpenIdProviderUrl; options.add_option("security", "", "openid-provider-url", "Open ID provider URL", cxxopts::value(OpenIdProviderUrl), ""); std::string OpenIdClientId; options.add_option("security", "", "openid-client-id", "Open ID client ID", cxxopts::value(OpenIdClientId), ""); options .add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value(ServerOptions.OwnerPid), ""); options.add_option("lifetime", "", "child-id", "Specify id which can be used to signal parent", cxxopts::value(ServerOptions.ChildId), ""); #if ZEN_PLATFORM_WINDOWS options.add_option("lifetime", "", "install", "Install zenserver as a Windows service", cxxopts::value(ServerOptions.InstallService), ""); options.add_option("lifetime", "", "uninstall", "Uninstall zenserver as a Windows service", cxxopts::value(ServerOptions.UninstallService), ""); #endif options.add_option("network", "", "http", "Select HTTP server implementation (asio|httpsys|null)", cxxopts::value(ServerOptions.HttpServerConfig.ServerClass)->default_value(DefaultHttp), ""); options.add_option("network", "", "http-threads", "Number of http server connection threads", cxxopts::value(ServerOptions.HttpServerConfig.ThreadCount), ""); options.add_option("network", "p", "port", "Select HTTP port", cxxopts::value(ServerOptions.BasePort)->default_value("8558"), ""); options.add_option("network", "", "http-forceloopback", "Force using local loopback interface", cxxopts::value(ServerOptions.HttpServerConfig.ForceLoopback)->default_value("false"), ""); options.add_option("httpsys", "", "httpsys-async-work-threads", "Number of HttpSys async worker threads", cxxopts::value(ServerOptions.HttpServerConfig.HttpSys.AsyncWorkThreadCount), ""); options.add_option("httpsys", "", "httpsys-enable-async-response", "Enables Httpsys async response", cxxopts::value(ServerOptions.HttpServerConfig.HttpSys.IsAsyncResponseEnabled)->default_value("true"), ""); options.add_option("httpsys", "", "httpsys-enable-request-logging", "Enables Httpsys request logging", cxxopts::value(ServerOptions.HttpServerConfig.HttpSys.IsRequestLoggingEnabled), ""); #if ZEN_WITH_TRACE options.add_option("ue-trace", "", "tracehost", "Hostname to send the trace to", cxxopts::value(ServerOptions.TraceHost)->default_value(""), ""); options.add_option("ue-trace", "", "tracefile", "Path to write a trace to", cxxopts::value(ServerOptions.TraceFile)->default_value(""), ""); #endif // ZEN_WITH_TRACE options.add_option("diagnostics", "", "crash", "Simulate a crash", cxxopts::value(ServerOptions.ShouldCrash)->default_value("false"), ""); std::string UpstreamCachePolicyOptions; options.add_option("cache", "", "upstream-cache-policy", "", cxxopts::value(UpstreamCachePolicyOptions)->default_value(""), "Upstream cache policy (readwrite|readonly|writeonly|disabled)"); options.add_option("cache", "", "upstream-jupiter-url", "URL to a Jupiter instance", cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.Url)->default_value(""), ""); options.add_option("cache", "", "upstream-jupiter-oauth-url", "URL to the OAuth provier", cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl)->default_value(""), ""); options.add_option("cache", "", "upstream-jupiter-oauth-clientid", "The OAuth client ID", cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId)->default_value(""), ""); options.add_option("cache", "", "upstream-jupiter-oauth-clientsecret", "The OAuth client secret", cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret)->default_value(""), ""); options.add_option("cache", "", "upstream-jupiter-openid-provider", "Name of a registered Open ID provider", cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider)->default_value(""), ""); options.add_option("cache", "", "upstream-jupiter-token", "A static authentication token", cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken)->default_value(""), ""); options.add_option("cache", "", "upstream-jupiter-namespace", "The Common Blob Store API namespace", cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace)->default_value(""), ""); options.add_option("cache", "", "upstream-jupiter-namespace-ddc", "The lecacy DDC namespace", cxxopts::value(ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace)->default_value(""), ""); options.add_option("cache", "", "upstream-zen-url", "URL to remote Zen server. Use a comma separated list to choose the one with the best latency.", cxxopts::value>(ServerOptions.UpstreamCacheConfig.ZenConfig.Urls), ""); options.add_option("cache", "", "upstream-zen-dns", "DNS that resolves to one or more Zen server instance(s)", cxxopts::value>(ServerOptions.UpstreamCacheConfig.ZenConfig.Dns), ""); options.add_option("cache", "", "upstream-thread-count", "Number of threads used for upstream procsssing", cxxopts::value(ServerOptions.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"), ""); options.add_option("cache", "", "upstream-connect-timeout-ms", "Connect timeout in millisecond(s). Default 5000 ms.", cxxopts::value(ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds)->default_value("5000"), ""); options.add_option("cache", "", "upstream-timeout-ms", "Timeout in millisecond(s). Default 0 ms", cxxopts::value(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"), ""); options.add_option("cache", "", "cache-write-log", "Whether cache write log is enabled", cxxopts::value(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("false"), ""); options.add_option("cache", "", "cache-access-log", "Whether cache access log is enabled", cxxopts::value(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"), ""); options.add_option("cache", "", "cache-reference-cache-enabled", "Whether caching of references is enabled", cxxopts::value(ServerOptions.StructuredCacheConfig.EnableReferenceCaching)->default_value("false"), ""); options.add_option( "cache", "", "cache-memlayer-sizethreshold", "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching.", cxxopts::value(ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold)->default_value("1024"), ""); options.add_option("cache", "", "cache-memlayer-targetfootprint", "Max allowed memory used by cache memory layer per namespace in bytes. Default set to 536870912 (512 Mb).", cxxopts::value(ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes)->default_value("536870912"), ""); options.add_option("cache", "", "cache-memlayer-triminterval", "Minimum time between each attempt to trim cache memory layers in seconds. Default set to 60 (1 min).", cxxopts::value(ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds)->default_value("60"), ""); options.add_option("cache", "", "cache-memlayer-maxage", "Maximum age of payloads when trimming cache memory layers in seconds. Default set to 86400 (1 day).", cxxopts::value(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"), ""); options.add_option("gc", "", "gc-enabled", "Whether garbage collection is enabled or not.", cxxopts::value(ServerOptions.GcConfig.Enabled)->default_value("true"), ""); options.add_option("gc", "", "gc-v2", "Use V2 of GC implementation or not.", cxxopts::value(ServerOptions.GcConfig.UseGCV2)->default_value("false"), ""); options.add_option("gc", "", "gc-small-objects", "Whether garbage collection of small objects is enabled or not.", cxxopts::value(ServerOptions.GcConfig.CollectSmallObjects)->default_value("true"), ""); options.add_option("gc", "", "gc-interval-seconds", "Garbage collection interval in seconds. Default set to 3600 (1 hour).", cxxopts::value(ServerOptions.GcConfig.IntervalSeconds)->default_value("3600"), ""); options.add_option("gc", "", "gc-lightweight-interval-seconds", "Lightweight garbage collection interval in seconds. Default set to 900 (30 min).", cxxopts::value(ServerOptions.GcConfig.LightweightIntervalSeconds)->default_value("900"), ""); options.add_option("gc", "", "gc-cache-duration-seconds", "Max duration in seconds before Z$ entries get evicted. Default set to 1209600 (2 weeks)", cxxopts::value(ServerOptions.GcConfig.Cache.MaxDurationSeconds)->default_value("1209600"), ""); options.add_option("gc", "", "gc-projectstore-duration-seconds", "Max duration in seconds before project store entries get evicted. Default set to 1209600 (2 weeks)", cxxopts::value(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds)->default_value("1209600"), ""); options.add_option("gc", "", "disk-reserve-size", "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.", cxxopts::value(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"), ""); options.add_option("gc", "", "gc-monitor-interval-seconds", "Garbage collection monitoring interval in seconds. Default set to 30 (30 seconds)", cxxopts::value(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"), ""); options.add_option("gc", "", "gc-low-diskspace-threshold", "Minimum free space on disk to allow writes to disk. Default set to 268435456 (256 Mb). Set to zero to disable.", cxxopts::value(ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites)->default_value("268435456"), ""); options.add_option("gc", "", "gc-disksize-softlimit", "Garbage collection disk usage soft limit. Default set to 0 (Off).", cxxopts::value(ServerOptions.GcConfig.DiskSizeSoftLimit)->default_value("0"), ""); options.add_option("gc", "", "gc-compactblock-threshold", "Garbage collection - how much of a compact block should be used to skip compacting the block. 0 - compact only " "empty eligible blocks, 100 - compact all non-full eligible blocks.", cxxopts::value(ServerOptions.GcConfig.CompactBlockUsageThresholdPercent)->default_value("60"), ""); options.add_option("gc", "", "gc-verbose", "Enable verbose logging for GC.", cxxopts::value(ServerOptions.GcConfig.Verbose)->default_value("false"), ""); options.add_option("objectstore", "", "objectstore-enabled", "Whether the object store is enabled or not.", cxxopts::value(ServerOptions.ObjectStoreEnabled)->default_value("false"), ""); std::vector BucketConfigs; options.add_option("objectstore", "", "objectstore-bucket", "Object store bucket mappings.", cxxopts::value>(BucketConfigs), ""); options.add_option("stats", "", "statsd", "", cxxopts::value(ServerOptions.StatsConfig.Enabled)->default_value("false"), "Enable statsd reporter (localhost:8125)"); try { cxxopts::ParseResult Result; try { Result = options.parse(argc, argv); } catch (std::exception& Ex) { throw zen::OptionParseException(Ex.what()); } if (Result.count("help")) { ZEN_CONSOLE("{}", options.help()); #if ZEN_PLATFORM_WINDOWS ZEN_CONSOLE("Press any key to exit!"); _getch(); #else // Assume the user's in a terminal on all other platforms and that // they'll use less/more/etc. if need be. #endif exit(0); } ServerOptions.DataDir = MakeSafePath(DataDir); ServerOptions.BaseSnapshotDir = MakeSafePath(BaseSnapshotDir); ServerOptions.ContentDir = MakeSafePath(ContentDir); ServerOptions.AbsLogFile = MakeSafePath(AbsLogFile); ServerOptions.ConfigFile = MakeSafePath(ConfigFile); ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); if (!BaseSnapshotDir.empty()) { if (DataDir.empty()) throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot"); if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir)) throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir)); } if (OpenIdProviderUrl.empty() == false) { if (OpenIdClientId.empty()) { throw zen::OptionParseException("Invalid OpenID client ID"); } ServerOptions.AuthConfig.OpenIdProviders.push_back( {.Name = OpenIdProviderName, .Url = OpenIdProviderUrl, .ClientId = OpenIdClientId}); } ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); if (!ServerOptions.ConfigFile.empty()) { ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile); } else { ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, OutputConfigFile); } ValidateOptions(ServerOptions); } catch (zen::OptionParseException& e) { ZEN_CONSOLE_ERROR("Error parsing zenserver arguments: {}\n\n{}", e.what(), options.help()); throw; } if (ServerOptions.DataDir.empty()) { ServerOptions.DataDir = PickDefaultStateDirectory(); } if (ServerOptions.AbsLogFile.empty()) { ServerOptions.AbsLogFile = ServerOptions.DataDir / "logs" / "zenserver.log"; } ServerOptions.HttpServerConfig.IsDedicatedServer = ServerOptions.IsDedicated; } } // namespace zen