diff options
| author | Stefan Boberg <[email protected]> | 2025-10-13 16:52:19 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-10-13 16:52:19 +0200 |
| commit | 61a0439e0257766b5b4bed163db2acce7dadde64 (patch) | |
| tree | 518a235e070a5d70458e3704a2c296da7bad826f /src | |
| parent | extract storage server into separate source files (#569) (diff) | |
| download | zen-61a0439e0257766b5b4bed163db2acce7dadde64.tar.xz zen-61a0439e0257766b5b4bed163db2acce7dadde64.zip | |
split storage config from base config (#570)
this change splits the configuration logic into a base part and a storage server part
manually tested with shared server config file
Diffstat (limited to 'src')
| -rw-r--r-- | src/zenserver/config.cpp | 1131 | ||||
| -rw-r--r-- | src/zenserver/config.h | 170 | ||||
| -rw-r--r-- | src/zenserver/main.cpp | 2 | ||||
| -rw-r--r-- | src/zenserver/storageconfig.cpp | 1055 | ||||
| -rw-r--r-- | src/zenserver/storageconfig.h | 203 | ||||
| -rw-r--r-- | src/zenserver/zenstorageserver.cpp | 2 |
6 files changed, 1355 insertions, 1208 deletions
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index ce3a70184..8ee50cee2 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -2,6 +2,8 @@ #include "config.h" +#include "storageconfig.h" + #include "config/luaconfig.h" #include "diag/logging.h" @@ -9,23 +11,17 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinaryutil.h> #include <zencore/compactbinaryvalidation.h> -#include <zencore/crypto.h> #include <zencore/except.h> #include <zencore/fmtutils.h> #include <zencore/iobuffer.h> #include <zencore/logging.h> #include <zencore/string.h> -#include <zenhttp/zenhttp.h> #include <zenutil/commandlineoptions.h> #include <zenutil/environmentoptions.h> ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> -#include <fmt/ranges.h> -#include <zencore/logging.h> #include <cxxopts.hpp> -#include <json11.hpp> -#include <sol/sol.hpp> ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_PLATFORM_WINDOWS @@ -34,9 +30,6 @@ ZEN_THIRD_PARTY_INCLUDES_END # include <unistd.h> #endif -#include <unordered_map> -#include <unordered_set> - namespace zen { std::filesystem::path @@ -95,352 +88,7 @@ ReadAllCentralManifests(const std::filesystem::path& SystemRoot) } void -ValidateOptions(ZenStorageServerOptions& ServerOptions) -{ - if (ServerOptions.EncryptionKey.empty() == false) - { - const auto Key = AesKey256Bit::FromString(ServerOptions.EncryptionKey); - - if (Key.IsValid() == false) - { - throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", ServerOptions.EncryptionKey), {}); - } - } - - if (ServerOptions.EncryptionIV.empty() == false) - { - const auto IV = AesIV128Bit::FromString(ServerOptions.EncryptionIV); - - if (IV.IsValid() == false) - { - throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", ServerOptions.EncryptionIV), {}); - } - } - if (ServerOptions.HttpServerConfig.ForceLoopback && ServerOptions.IsDedicated) - { - throw OptionParseException("'--dedicated' conflicts with '--http-forceloopback'", {}); - } - if (ServerOptions.GcConfig.AttachmentPassCount > ZenGcConfig::GcMaxAttachmentPassCount) - { - throw OptionParseException(fmt::format("'--gc-attachment-passes' ('{}') is invalid, maximum is {}.", - ServerOptions.GcConfig.AttachmentPassCount, - ZenGcConfig::GcMaxAttachmentPassCount), - {}); - } - if (ServerOptions.GcConfig.UseGCV2 == false) - { - ZEN_WARN("'--gc-v2=false' is deprecated, reverting to '--gc-v2=true'"); - ServerOptions.GcConfig.UseGCV2 = true; - } -} - -UpstreamCachePolicy -ParseUpstreamCachePolicy(std::string_view Options) -{ - if (Options == "readonly") - { - return UpstreamCachePolicy::Read; - } - else if (Options == "writeonly") - { - return UpstreamCachePolicy::Write; - } - else if (Options == "disabled") - { - return UpstreamCachePolicy::Disabled; - } - else - { - return UpstreamCachePolicy::ReadWrite; - } -} - -ZenObjectStoreConfig -ParseBucketConfigs(std::span<std::string> Buckets) -{ - using namespace std::literals; - - ZenObjectStoreConfig Cfg; - - // split bucket args in the form of "{BucketName};{LocalPath}" - for (std::string_view Bucket : Buckets) - { - ZenObjectStoreConfig::BucketConfig NewBucket; - - if (auto Idx = Bucket.find_first_of(";"); Idx != std::string_view::npos) - { - NewBucket.Name = Bucket.substr(0, Idx); - NewBucket.Directory = Bucket.substr(Idx + 1); - } - else - { - NewBucket.Name = Bucket; - } - - Cfg.Buckets.push_back(std::move(NewBucket)); - } - - return Cfg; -} - -class CachePolicyOption : public LuaConfig::OptionValue -{ -public: - CachePolicyOption(UpstreamCachePolicy& Value) : Value(Value) {} - virtual void Print(std::string_view, StringBuilderBase& StringBuilder) override - { - switch (Value) - { - case UpstreamCachePolicy::Read: - StringBuilder.Append("readonly"); - break; - case UpstreamCachePolicy::Write: - StringBuilder.Append("writeonly"); - break; - case UpstreamCachePolicy::Disabled: - StringBuilder.Append("disabled"); - break; - case UpstreamCachePolicy::ReadWrite: - StringBuilder.Append("readwrite"); - break; - default: - ZEN_ASSERT(false); - } - } - virtual void Parse(sol::object Object) override - { - std::string PolicyString = Object.as<std::string>(); - if (PolicyString == "readonly") - { - Value = UpstreamCachePolicy::Read; - } - else if (PolicyString == "writeonly") - { - Value = UpstreamCachePolicy::Write; - } - else if (PolicyString == "disabled") - { - Value = UpstreamCachePolicy::Disabled; - } - else if (PolicyString == "readwrite") - { - Value = UpstreamCachePolicy::ReadWrite; - } - } - UpstreamCachePolicy& Value; -}; - -class ZenAuthConfigOption : public LuaConfig::OptionValue -{ -public: - ZenAuthConfigOption(ZenAuthConfig& Value) : Value(Value) {} - virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override - { - if (Value.OpenIdProviders.empty()) - { - StringBuilder.Append("{}"); - return; - } - LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); - for (const ZenOpenIdProviderConfig& Config : Value.OpenIdProviders) - { - Writer.BeginContainer(""); - { - Writer.WriteValue("name", Config.Name); - Writer.WriteValue("url", Config.Url); - Writer.WriteValue("clientid", Config.ClientId); - } - Writer.EndContainer(); - } - } - virtual void Parse(sol::object Object) override - { - if (sol::optional<sol::table> OpenIdProviders = Object.as<sol::table>()) - { - for (const auto& Kv : OpenIdProviders.value()) - { - if (sol::optional<sol::table> OpenIdProvider = Kv.second.as<sol::table>()) - { - std::string Name = OpenIdProvider.value().get_or("name", std::string("Default")); - std::string Url = OpenIdProvider.value().get_or("url", std::string()); - std::string ClientId = OpenIdProvider.value().get_or("clientid", std::string()); - - Value.OpenIdProviders.push_back({.Name = std::move(Name), .Url = std::move(Url), .ClientId = std::move(ClientId)}); - } - } - } - } - ZenAuthConfig& Value; -}; - -class ZenObjectStoreConfigOption : public LuaConfig::OptionValue -{ -public: - ZenObjectStoreConfigOption(ZenObjectStoreConfig& Value) : Value(Value) {} - virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override - { - if (Value.Buckets.empty()) - { - StringBuilder.Append("{}"); - return; - } - LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); - for (const ZenObjectStoreConfig::BucketConfig& Config : Value.Buckets) - { - Writer.BeginContainer(""); - { - Writer.WriteValue("name", Config.Name); - std::string Directory = Config.Directory.string(); - LuaConfig::EscapeBackslash(Directory); - Writer.WriteValue("directory", Directory); - } - Writer.EndContainer(); - } - } - virtual void Parse(sol::object Object) override - { - if (sol::optional<sol::table> Buckets = Object.as<sol::table>()) - { - for (const auto& Kv : Buckets.value()) - { - if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>()) - { - std::string Name = Bucket.value().get_or("name", std::string("Default")); - std::string Directory = Bucket.value().get_or("directory", std::string()); - - Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)}); - } - } - } - } - ZenObjectStoreConfig& Value; -}; - -class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue -{ -public: - ZenStructuredCacheBucketsConfigOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value) : Value(Value) {} - virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override - { - if (Value.empty()) - { - StringBuilder.Append("{}"); - return; - } - LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); - for (const std::pair<std::string, ZenStructuredCacheBucketConfig>& Bucket : Value) - { - Writer.BeginContainer(""); - { - Writer.WriteValue("name", Bucket.first); - const ZenStructuredCacheBucketConfig& BucketConfig = Bucket.second; - - Writer.WriteValue("maxblocksize", fmt::format("{}", BucketConfig.MaxBlockSize)); - Writer.BeginContainer("memlayer"); - { - Writer.WriteValue("sizethreshold", fmt::format("{}", BucketConfig.MemCacheSizeThreshold)); - } - Writer.EndContainer(); - - Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment)); - Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment)); - Writer.WriteValue("limitoverwrites", fmt::format("{}", BucketConfig.LimitOverwrites)); - } - Writer.EndContainer(); - } - } - virtual void Parse(sol::object Object) override - { - if (sol::optional<sol::table> Buckets = Object.as<sol::table>()) - { - for (const auto& Kv : Buckets.value()) - { - if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>()) - { - ZenStructuredCacheBucketConfig BucketConfig; - std::string Name = Kv.first.as<std::string>(); - if (Name.empty()) - { - throw OptionParseException("Cache bucket option must have a name.", {}); - } - - const uint64_t MaxBlockSize = Bucket.value().get_or("maxblocksize", BucketConfig.MaxBlockSize); - if (MaxBlockSize == 0) - { - throw OptionParseException( - fmt::format("'maxblocksize' option for cache bucket '{}' is invalid. It must be non-zero.", Name), - {}); - } - BucketConfig.MaxBlockSize = MaxBlockSize; - - if (sol::optional<sol::table> Memlayer = Bucket.value().get_or("memlayer", sol::table())) - { - const uint64_t MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); - if (MemCacheSizeThreshold == 0) - { - throw OptionParseException( - fmt::format("'memlayer.sizethreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), - {}); - } - BucketConfig.MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); - } - - const uint32_t PayloadAlignment = Bucket.value().get_or("payloadalignment", BucketConfig.PayloadAlignment); - if (PayloadAlignment == 0 || !IsPow2(PayloadAlignment)) - { - throw OptionParseException( - fmt::format( - "'payloadalignment' option for cache bucket '{}' is invalid. It needs to be non-zero and a power of two.", - Name), - {}); - } - BucketConfig.PayloadAlignment = PayloadAlignment; - - const uint64_t LargeObjectThreshold = Bucket.value().get_or("largeobjectthreshold", BucketConfig.LargeObjectThreshold); - if (LargeObjectThreshold == 0) - { - throw OptionParseException( - fmt::format("'largeobjectthreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), - {}); - } - BucketConfig.LargeObjectThreshold = LargeObjectThreshold; - - BucketConfig.LimitOverwrites = Bucket.value().get_or("limitoverwrites", BucketConfig.LimitOverwrites); - - Value.push_back(std::make_pair(std::move(Name), BucketConfig)); - } - } - } - } - std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value; -}; - -std::shared_ptr<LuaConfig::OptionValue> -MakeOption(UpstreamCachePolicy& Value) -{ - return std::make_shared<CachePolicyOption>(Value); -}; - -std::shared_ptr<LuaConfig::OptionValue> -MakeOption(ZenAuthConfig& Value) -{ - return std::make_shared<ZenAuthConfigOption>(Value); -}; - -std::shared_ptr<LuaConfig::OptionValue> -MakeOption(ZenObjectStoreConfig& Value) -{ - return std::make_shared<ZenObjectStoreConfigOption>(Value); -}; - -std::shared_ptr<LuaConfig::OptionValue> -MakeOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value) -{ - return std::make_shared<ZenStructuredCacheBucketsConfigOption>(Value); -}; - -void -ParseEnvVariables(ZenStorageServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult) +ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult) { using namespace std::literals; @@ -463,18 +111,12 @@ ParseEnvVariables(ZenStorageServerOptions& ServerOptions, const cxxopts::ParseRe } void -ParseConfigFile(const std::filesystem::path& Path, - ZenStorageServerOptions& ServerOptions, - const cxxopts::ParseResult& CmdLineResult, - std::string_view OutputConfigFile) +AddServerConfigOptions(LuaConfig::Options& LuaOptions, ZenServerOptions& ServerOptions) { - ZEN_TRACE_CPU("ParseConfigFile"); - using namespace std::literals; - LuaConfig::Options LuaOptions; + // server - ////// server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); @@ -486,20 +128,11 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv); - LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv); LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv); LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv); - ////// objectstore - LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); - LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); - - ////// buildsstore - LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); - LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); - ////// network LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv); @@ -529,219 +162,28 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("stats.enable"sv, ServerOptions.StatsConfig.Enabled, "statsd"sv); LuaOptions.AddOption("stats.host"sv, ServerOptions.StatsConfig.StatsdHost); LuaOptions.AddOption("stats.port"sv, ServerOptions.StatsConfig.StatsdPort); - - ////// cache - LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled); - LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log"sv); - LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"sv); - - LuaOptions.AddOption("cache.memlayer.sizethreshold"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, - "cache-memlayer-sizethreshold"sv); - LuaOptions.AddOption("cache.memlayer.targetfootprint"sv, - ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes, - "cache-memlayer-targetfootprint"sv); - LuaOptions.AddOption("cache.memlayer.triminterval"sv, - ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds, - "cache-memlayer-triminterval"sv); - LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"sv); - - LuaOptions.AddOption("cache.bucket.maxblocksize"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, - "cache-bucket-maxblocksize"sv); - LuaOptions.AddOption("cache.bucket.memlayer.sizethreshold"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, - "cache-bucket-memlayer-sizethreshold"sv); - LuaOptions.AddOption("cache.bucket.payloadalignment"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, - "cache-bucket-payloadalignment"sv); - LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, - "cache-bucket-largeobjectthreshold"sv); - LuaOptions.AddOption("cache.bucket.limitoverwrites"sv, - ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites, - "cache-bucket-limit-overwrites"sv); - - ////// cache.upstream - LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv); - LuaOptions.AddOption("cache.upstream.upstreamthreadcount"sv, - ServerOptions.UpstreamCacheConfig.UpstreamThreadCount, - "upstream-thread-count"sv); - LuaOptions.AddOption("cache.upstream.connecttimeoutms"sv, - ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds, - "upstream-connect-timeout-ms"sv); - LuaOptions.AddOption("cache.upstream.timeoutms"sv, ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds, "upstream-timeout-ms"sv); - - ////// cache.upstream.jupiter - LuaOptions.AddOption("cache.upstream.jupiter.name"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Name); - LuaOptions.AddOption("cache.upstream.jupiter.url"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Url, "upstream-jupiter-url"sv); - LuaOptions.AddOption("cache.upstream.jupiter.oauthprovider"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl, - "upstream-jupiter-oauth-url"sv); - LuaOptions.AddOption("cache.upstream.jupiter.oauthclientid"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId, - "upstream-jupiter-oauth-clientid"); - LuaOptions.AddOption("cache.upstream.jupiter.oauthclientsecret"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret, - "upstream-jupiter-oauth-clientsecret"sv); - LuaOptions.AddOption("cache.upstream.jupiter.openidprovider"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider, - "upstream-jupiter-openid-provider"sv); - LuaOptions.AddOption("cache.upstream.jupiter.token"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken, - "upstream-jupiter-token"sv); - LuaOptions.AddOption("cache.upstream.jupiter.namespace"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace, - "upstream-jupiter-namespace"sv); - LuaOptions.AddOption("cache.upstream.jupiter.ddcnamespace"sv, - ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace, - "upstream-jupiter-namespace-ddc"sv); - - ////// cache.upstream.zen - // LuaOptions.AddOption("cache.upstream.zen"sv, ServerOptions.UpstreamCacheConfig.ZenConfig); - LuaOptions.AddOption("cache.upstream.zen.name"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Name); - LuaOptions.AddOption("cache.upstream.zen.dns"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Dns); - LuaOptions.AddOption("cache.upstream.zen.url"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Urls); - - ////// gc - LuaOptions.AddOption("gc.enabled"sv, ServerOptions.GcConfig.Enabled, "gc-enabled"sv); - LuaOptions.AddOption("gc.v2"sv, ServerOptions.GcConfig.UseGCV2, "gc-v2"sv); - - LuaOptions.AddOption("gc.monitorintervalseconds"sv, ServerOptions.GcConfig.MonitorIntervalSeconds, "gc-monitor-interval-seconds"sv); - LuaOptions.AddOption("gc.intervalseconds"sv, ServerOptions.GcConfig.IntervalSeconds, "gc-interval-seconds"sv); - LuaOptions.AddOption("gc.collectsmallobjects"sv, ServerOptions.GcConfig.CollectSmallObjects, "gc-small-objects"sv); - LuaOptions.AddOption("gc.diskreservesize"sv, ServerOptions.GcConfig.DiskReserveSize, "disk-reserve-size"sv); - LuaOptions.AddOption("gc.disksizesoftlimit"sv, ServerOptions.GcConfig.DiskSizeSoftLimit, "gc-disksize-softlimit"sv); - LuaOptions.AddOption("gc.lowdiskspacethreshold"sv, - ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, - "gc-low-diskspace-threshold"sv); - LuaOptions.AddOption("gc.lightweightintervalseconds"sv, - ServerOptions.GcConfig.LightweightIntervalSeconds, - "gc-lightweight-interval-seconds"sv); - LuaOptions.AddOption("gc.compactblockthreshold"sv, - ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, - "gc-compactblock-threshold"sv); - LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv); - LuaOptions.AddOption("gc.single-threaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv); - LuaOptions.AddOption("gc.cache.attachment.store"sv, ServerOptions.GcConfig.StoreCacheAttachmentMetaData, "gc-cache-attachment-store"); - LuaOptions.AddOption("gc.projectstore.attachment.store"sv, - ServerOptions.GcConfig.StoreProjectAttachmentMetaData, - "gc-projectstore-attachment-store"); - LuaOptions.AddOption("gc.attachment.passes"sv, ServerOptions.GcConfig.AttachmentPassCount, "gc-attachment-passes"sv); - LuaOptions.AddOption("gc.validation"sv, ServerOptions.GcConfig.EnableValidation, "gc-validation"); - - LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv); - LuaOptions.AddOption("gc.projectstore.duration.seconds"sv, - ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds, - "gc-projectstore-duration-seconds"); - LuaOptions.AddOption("gc.buildstore.duration.seconds"sv, - ServerOptions.GcConfig.BuildStore.MaxDurationSeconds, - "gc-buildstore-duration-seconds"); - - ////// security - LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); - LuaOptions.AddOption("security.encryptionaesiv"sv, ServerOptions.EncryptionIV, "encryption-aes-iv"sv); - LuaOptions.AddOption("security.openidproviders"sv, ServerOptions.AuthConfig); - - ////// workspaces - LuaOptions.AddOption("workspaces.enabled"sv, ServerOptions.WorksSpacesConfig.Enabled, "workspaces-enabled"sv); - LuaOptions.AddOption("workspaces.allowconfigchanges"sv, - ServerOptions.WorksSpacesConfig.AllowConfigurationChanges, - "workspaces-allow-changes"sv); - - LuaOptions.AddOption("cache.buckets"sv, ServerOptions.StructuredCacheConfig.PerBucketConfigs, "cache.buckets"sv); - - LuaOptions.Parse(Path, CmdLineResult); - - // These have special command line processing so we make sure we export them if they were configured on command line - if (!ServerOptions.AuthConfig.OpenIdProviders.empty()) - { - LuaOptions.Touch("security.openidproviders"sv); - } - if (!ServerOptions.ObjectStoreConfig.Buckets.empty()) - { - LuaOptions.Touch("server.objectstore.buckets"sv); - } - if (!ServerOptions.StructuredCacheConfig.PerBucketConfigs.empty()) - { - LuaOptions.Touch("cache.buckets"sv); - } - - if (!OutputConfigFile.empty()) - { - std::filesystem::path WritePath(MakeSafeAbsolutePath(OutputConfigFile)); - ExtendableStringBuilder<512> ConfigStringBuilder; - LuaOptions.Print(ConfigStringBuilder, CmdLineResult); - BasicFile Output; - Output.Open(WritePath, BasicFile::Mode::kTruncate); - Output.Write(ConfigStringBuilder.Data(), ConfigStringBuilder.Size(), 0); - } } -void -ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort) +struct ZenServerCmdLineOptions { - using namespace std::literals; - - IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); - std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize()); - std::string JsonError; - json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError); - if (!JsonError.empty()) - { - ZEN_WARN("failed parsing plugins config file '{}'. Reason: '{}'", Path, JsonError); - return; - } - for (const json11::Json& PluginInfo : PluginsInfo.array_items()) - { - if (!PluginInfo.is_object()) - { - ZEN_WARN("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", Path, PluginInfo.dump()); - continue; - } - - HttpServerPluginConfig Config = {}; - - bool bNeedsPort = true; - - for (const std::pair<const std::string, json11::Json>& Items : PluginInfo.object_items()) - { - if (!Items.second.is_string()) - { - ZEN_WARN("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", - Path, - Items.second.dump()); - continue; - } - - const std::string& Name = Items.first; - const std::string& Value = Items.second.string_value(); - - if (Name == "name"sv) - Config.PluginName = Value; - else - { - Config.PluginOptions.push_back({Name, Value}); - - if (Name == "port"sv) - { - bNeedsPort = false; - } - } - } - - // add a default base port in case if json config didn't provide one - if (bNeedsPort) - { - Config.PluginOptions.push_back({"port", std::to_string(BasePort)}); - } + // Note to those adding future options; std::filesystem::path-type options + // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> + // stream operator to convert argv value into the options type. std::fs::path + // expects paths in streams to be quoted but argv paths are unquoted. By + // going into a std::string first, paths with whitespace parse correctly. + std::string ConfigFile; + std::string OutputConfigFile; + std::string SystemRootDir; + std::string ContentDir; + std::string DataDir; + std::string AbsLogFile; - ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config); - } -} + void AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions); + void ApplyOptions(ZenServerOptions& ServerOptions); +}; void -ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) +ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerOptions& ServerOptions) { const char* DefaultHttp = "asio"; @@ -752,31 +194,6 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) } #endif - for (int i = 0; i < argc; ++i) - { - if (i) - { - ServerOptions.CommandLine.push_back(' '); - } - - ServerOptions.CommandLine += argv[i]; - } - - // Note to those adding future options; std::filesystem::path-type options - // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> - // stream operator to convert argv value into the options type. std::fs::path - // expects paths in streams to be quoted but argv paths are unquoted. By - // going into a std::string first, paths with whitespace parse correctly. - std::string SystemRootDir; - std::string DataDir; - std::string ContentDir; - std::string AbsLogFile; - std::string ConfigFile; - std::string PluginsConfigFile; - std::string OutputConfigFile; - std::string BaseSnapshotDir; - - cxxopts::Options options("zenserver", "Zen Server"); options.add_options()("dedicated", "Enable dedicated server mode", cxxopts::value<bool>(ServerOptions.IsDedicated)->default_value("false")); @@ -784,24 +201,15 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) options.add_options()("clean", "Clean out all state at startup", cxxopts::value<bool>(ServerOptions.IsCleanStart)->default_value("false")); - options.add_options()("scrub", - "Validate state at startup", - cxxopts::value(ServerOptions.ScrubOptions)->implicit_value("yes"), - "(nocas,nogc,nodelete,yes,no)*"); options.add_options()("help", "Show command line help"); options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false")); + options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir)); options.add_options()("system-dir", "Specify system root", cxxopts::value<std::string>(SystemRootDir)); - options.add_options()("snapshot-dir", - "Specify a snapshot of server state to mirror into the persistence root at startup", - cxxopts::value<std::string>(BaseSnapshotDir)); options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::string>(ContentDir)); - options.add_options()("powercycle", - "Exit immediately after initialization is complete", - cxxopts::value<bool>(ServerOptions.IsPowerCycle)); options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile)); - options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value<std::string>(PluginsConfigFile)); options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile)); + options.add_options()("no-sentry", "Disable Sentry crash handler", cxxopts::value<bool>(ServerOptions.SentryConfig.Disable)->default_value("false")); @@ -819,6 +227,16 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) options.add_options()("malloc", "Configure memory allocator subsystem", cxxopts::value(ServerOptions.MemoryOptions)->default_value("mimalloc")); + options.add_options()("powercycle", + "Exit immediately after initialization is complete", + cxxopts::value<bool>(ServerOptions.IsPowerCycle)); + + options.add_option("diagnostics", + "", + "crash", + "Simulate a crash", + cxxopts::value<bool>(ServerOptions.ShouldCrash)->default_value("false"), + ""); // clang-format off options.add_options("logging") @@ -836,34 +254,6 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) ; // clang-format on - options.add_option("security", - "", - "encryption-aes-key", - "256 bit AES encryption key", - cxxopts::value<std::string>(ServerOptions.EncryptionKey), - ""); - - options.add_option("security", - "", - "encryption-aes-iv", - "128 bit AES encryption initialization vector", - cxxopts::value<std::string>(ServerOptions.EncryptionIV), - ""); - - std::string OpenIdProviderName; - options.add_option("security", - "", - "openid-provider-name", - "Open ID provider name", - cxxopts::value<std::string>(OpenIdProviderName), - "Default"); - - std::string OpenIdProviderUrl; - options.add_option("security", "", "openid-provider-url", "Open ID provider URL", cxxopts::value<std::string>(OpenIdProviderUrl), ""); - - std::string OpenIdClientId; - options.add_option("security", "", "openid-client-id", "Open ID client ID", cxxopts::value<std::string>(OpenIdClientId), ""); - options .add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value<int>(ServerOptions.OwnerPid), "<identifier>"); options.add_option("lifetime", @@ -890,17 +280,6 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) options.add_option("network", "", - "http", - "Select HTTP server implementation (asio|" -#if ZEN_WITH_HTTPSYS - "httpsys|" -#endif - "null)", - cxxopts::value<std::string>(ServerOptions.HttpServerConfig.ServerClass)->default_value(DefaultHttp), - "<http class>"); - - options.add_option("network", - "", "http-threads", "Number of http server connection threads", cxxopts::value<unsigned int>(ServerOptions.HttpServerConfig.ThreadCount), @@ -943,6 +322,17 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) "<httpsys request logging>"); #endif + options.add_option("network", + "", + "http", + "Select HTTP server implementation (asio|" +#if ZEN_WITH_HTTPSYS + "httpsys|" +#endif + "null)", + cxxopts::value<std::string>(ServerOptions.HttpServerConfig.ServerClass)->default_value(DefaultHttp), + "<http class>"); + #if ZEN_WITH_TRACE // We only have this in options for command line help purposes - we parse these argument separately earlier using // GetTraceOptionsFromCommandline() @@ -969,376 +359,45 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) ""); #endif // ZEN_WITH_TRACE - options.add_option("diagnostics", - "", - "crash", - "Simulate a crash", - cxxopts::value<bool>(ServerOptions.ShouldCrash)->default_value("false"), - ""); - - std::string UpstreamCachePolicyOptions; - options.add_option("cache", - "", - "upstream-cache-policy", - "", - cxxopts::value<std::string>(UpstreamCachePolicyOptions)->default_value(""), - "Upstream cache policy (readwrite|readonly|writeonly|disabled)"); - - options.add_option("cache", - "", - "upstream-jupiter-url", - "URL to a Jupiter instance", - cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.Url)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-oauth-url", - "URL to the OAuth provier", - cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-oauth-clientid", - "The OAuth client ID", - cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-oauth-clientsecret", - "The OAuth client secret", - cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-openid-provider", - "Name of a registered Open ID provider", - cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-token", - "A static authentication token", - cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-namespace", - "The Common Blob Store API namespace", - cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-jupiter-namespace-ddc", - "The lecacy DDC namespace", - cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace)->default_value(""), - ""); - - options.add_option("cache", - "", - "upstream-zen-url", - "URL to remote Zen server. Use a comma separated list to choose the one with the best latency.", - cxxopts::value<std::vector<std::string>>(ServerOptions.UpstreamCacheConfig.ZenConfig.Urls), - ""); - - options.add_option("cache", - "", - "upstream-zen-dns", - "DNS that resolves to one or more Zen server instance(s)", - cxxopts::value<std::vector<std::string>>(ServerOptions.UpstreamCacheConfig.ZenConfig.Dns), - ""); - - options.add_option("cache", - "", - "upstream-thread-count", - "Number of threads used for upstream procsssing", - cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"), - ""); - - options.add_option("cache", - "", - "upstream-connect-timeout-ms", - "Connect timeout in millisecond(s). Default 5000 ms.", - cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds)->default_value("5000"), - ""); - - options.add_option("cache", - "", - "upstream-timeout-ms", - "Timeout in millisecond(s). Default 0 ms", - cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"), - ""); - - options.add_option("cache", - "", - "cache-write-log", - "Whether cache write log is enabled", - cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("false"), - ""); - - options.add_option("cache", - "", - "cache-access-log", - "Whether cache access log is enabled", - cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"), - ""); - - options.add_option( - "cache", - "", - "cache-memlayer-sizethreshold", - "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching. " - "Obsolete, replaced by `--cache-bucket-memlayer-sizethreshold`", - cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), - ""); - - options.add_option("cache", - "", - "cache-memlayer-targetfootprint", - "Max allowed memory used by cache memory layer per namespace in bytes. Default set to 536870912 (512 Mb).", - cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes)->default_value("536870912"), - ""); - - options.add_option("cache", - "", - "cache-memlayer-triminterval", - "Minimum time between each attempt to trim cache memory layers in seconds. Default set to 60 (1 min).", - cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds)->default_value("60"), - ""); - - options.add_option("cache", - "", - "cache-memlayer-maxage", - "Maximum age of payloads when trimming cache memory layers in seconds. Default set to 86400 (1 day).", - cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"), - ""); - - options.add_option("cache", - "", - "cache-bucket-maxblocksize", - "Max size of cache bucket blocks. Default set to 1073741824 (1GB).", - cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize)->default_value("1073741824"), - ""); - - options.add_option("cache", - "", - "cache-bucket-payloadalignment", - "Payload alignement for cache bucket blocks. Default set to 16.", - cxxopts::value<uint32_t>(ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment)->default_value("16"), - ""); - - options.add_option( - "cache", - "", - "cache-bucket-largeobjectthreshold", - "Threshold for storing cache bucket values as loose files. Default set to 131072 (128 KB).", - cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold)->default_value("131072"), - ""); - - options.add_option( - "cache", - "", - "cache-bucket-memlayer-sizethreshold", - "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching.", - cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), - ""); - - options.add_option("cache", - "", - "cache-bucket-limit-overwrites", - "Whether to require policy flag pattern before allowing overwrites in cache bucket", - cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites)->default_value("false"), - ""); - - options.add_option("gc", - "", - "gc-cache-attachment-store", - "Enable storing attachments referenced by a cache record in block store meta data.", - cxxopts::value<bool>(ServerOptions.GcConfig.StoreCacheAttachmentMetaData)->default_value("false"), - ""); - - options.add_option("gc", - "", - "gc-projectstore-attachment-store", - "Enable storing attachments referenced by project oplogs in meta data.", - cxxopts::value<bool>(ServerOptions.GcConfig.StoreProjectAttachmentMetaData)->default_value("false"), - ""); - - options.add_option("gc", - "", - "gc-validation", - "Enable validation of references after full GC.", - cxxopts::value<bool>(ServerOptions.GcConfig.EnableValidation)->default_value("true"), - ""); - - options.add_option("gc", - "", - "gc-enabled", - "Whether garbage collection is enabled or not.", - cxxopts::value<bool>(ServerOptions.GcConfig.Enabled)->default_value("true"), - ""); - - options.add_option("gc", - "", - "gc-v2", - "Use V2 of GC implementation or not.", - cxxopts::value<bool>(ServerOptions.GcConfig.UseGCV2)->default_value("true"), - ""); - - options.add_option("gc", - "", - "gc-small-objects", - "Whether garbage collection of small objects is enabled or not.", - cxxopts::value<bool>(ServerOptions.GcConfig.CollectSmallObjects)->default_value("true"), - ""); - - options.add_option("gc", - "", - "gc-interval-seconds", - "Garbage collection interval in seconds. Default set to 3600 (1 hour).", - cxxopts::value<int32_t>(ServerOptions.GcConfig.IntervalSeconds)->default_value("3600"), - ""); - - options.add_option("gc", - "", - "gc-lightweight-interval-seconds", - "Lightweight garbage collection interval in seconds. Default set to 900 (30 min).", - cxxopts::value<int32_t>(ServerOptions.GcConfig.LightweightIntervalSeconds)->default_value("900"), - ""); - - options.add_option("gc", - "", - "gc-cache-duration-seconds", - "Max duration in seconds before Z$ entries get evicted. Default set to 1209600 (2 weeks)", - cxxopts::value<int32_t>(ServerOptions.GcConfig.Cache.MaxDurationSeconds)->default_value("1209600"), - ""); - - options.add_option("gc", - "", - "gc-projectstore-duration-seconds", - "Max duration in seconds before project store entries get evicted. Default set to 1209600 (2 weeks)", - cxxopts::value<int32_t>(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds)->default_value("1209600"), - ""); - - options.add_option("gc", - "", - "gc-buildstore-duration-seconds", - "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)", - cxxopts::value<int32_t>(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"), - ""); - - options.add_option("gc", - "", - "disk-reserve-size", - "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.", - cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"), - ""); - - options.add_option("gc", - "", - "gc-monitor-interval-seconds", - "Garbage collection monitoring interval in seconds. Default set to 30 (30 seconds)", - cxxopts::value<int32_t>(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"), - ""); - - options.add_option("gc", - "", - "gc-low-diskspace-threshold", - "Minimum free space on disk to allow writes to disk. Default set to 268435456 (256 Mb). Set to zero to disable.", - cxxopts::value<uint64_t>(ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites)->default_value("268435456"), - ""); - - options.add_option("gc", - "", - "gc-disksize-softlimit", - "Garbage collection disk usage soft limit. Default set to 0 (Off).", - cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskSizeSoftLimit)->default_value("0"), - ""); - - options.add_option("gc", - "", - "gc-compactblock-threshold", - "Garbage collection - how much of a compact block should be used to skip compacting the block. 0 - compact only " - "empty eligible blocks, 100 - compact all non-full eligible blocks.", - cxxopts::value<uint32_t>(ServerOptions.GcConfig.CompactBlockUsageThresholdPercent)->default_value("60"), - ""); - - options.add_option("gc", + options.add_option("stats", "", - "gc-verbose", - "Enable verbose logging for GC.", - cxxopts::value<bool>(ServerOptions.GcConfig.Verbose)->default_value("false"), - ""); - - options.add_option("gc", + "statsd", "", - "gc-single-threaded", - "Force GC to run single threaded.", - cxxopts::value<bool>(ServerOptions.GcConfig.SingleThreaded)->default_value("false"), - ""); + cxxopts::value<bool>(ServerOptions.StatsConfig.Enabled)->default_value("false"), + "Enable statsd reporter (localhost:8125)"); +} - options.add_option("gc", - "", - "gc-attachment-passes", - "Limit the range of unreferenced attachments included in GC check by breaking it into passes. Default is one pass " - "which includes all the attachments.", - cxxopts::value<uint16_t>(ServerOptions.GcConfig.AttachmentPassCount)->default_value("1"), - ""); +void +ZenServerCmdLineOptions::ApplyOptions(ZenServerOptions& ServerOptions) +{ + ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); + ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); + ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); + ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile); + ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); +} - options.add_option("objectstore", - "", - "objectstore-enabled", - "Whether the object store is enabled or not.", - cxxopts::value<bool>(ServerOptions.ObjectStoreEnabled)->default_value("false"), - ""); +void +ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) +{ + for (int i = 0; i < argc; ++i) + { + if (i) + { + ServerOptions.CommandLine.push_back(' '); + } - std::vector<std::string> BucketConfigs; - options.add_option("objectstore", - "", - "objectstore-bucket", - "Object store bucket mappings.", - cxxopts::value<std::vector<std::string>>(BucketConfigs), - ""); + ServerOptions.CommandLine += argv[i]; + } - options.add_option("buildstore", - "", - "buildstore-enabled", - "Whether the builds store is enabled or not.", - cxxopts::value<bool>(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), - ""); - options.add_option("buildstore", - "", - "buildstore-disksizelimit", - "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)", - cxxopts::value<uint64_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"), - ""); + cxxopts::Options options("zenserver", "Zen Storage Server"); - options.add_option("stats", - "", - "statsd", - "", - cxxopts::value<bool>(ServerOptions.StatsConfig.Enabled)->default_value("false"), - "Enable statsd reporter (localhost:8125)"); + ZenServerCmdLineOptions BaseOptions; + BaseOptions.AddCliOptions(options, ServerOptions); - options.add_option("workspaces", - "", - "workspaces-enabled", - "", - cxxopts::value<bool>(ServerOptions.WorksSpacesConfig.Enabled)->default_value("true"), - "Enable workspaces support with folder sharing"); + ZenStorageServerCmdLineOptions StorageOptions; + StorageOptions.AddCliOptions(options, ServerOptions); - options.add_option("workspaces", - "", - "workspaces-allow-changes", - "", - cxxopts::value<bool>(ServerOptions.WorksSpacesConfig.AllowConfigurationChanges)->default_value("false"), - "Allow adding/modifying/deleting of workspace and shares via http endpoint"); try { cxxopts::ParseResult Result; @@ -1355,6 +414,7 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) if (Result.count("help")) { ZEN_CONSOLE("{}", options.help()); + #if ZEN_PLATFORM_WINDOWS ZEN_CONSOLE("Press any key to exit!"); _getch(); @@ -1362,6 +422,7 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) // Assume the user's in a terminal on all other platforms and that // they'll use less/more/etc. if need be. #endif + exit(0); } @@ -1371,8 +432,6 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) TraceConfigure(ServerOptions.TraceOptions); } - ZEN_TRACE_CPU("ConfigParse"); - if (ServerOptions.QuietConsole) { bool HasExplicitConsoleLevel = false; @@ -1402,46 +461,20 @@ ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions) } logging::RefreshLogLevels(); - ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); - ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); - ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); - ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); - ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile); - ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); - ServerOptions.PluginsConfigFile = MakeSafeAbsolutePath(PluginsConfigFile); - ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); - - if (!BaseSnapshotDir.empty()) - { - if (DataDir.empty()) - throw OptionParseException("'--snapshot-dir' requires '--data-dir'", options.help()); - - if (!IsDir(ServerOptions.BaseSnapshotDir)) - throw std::runtime_error(fmt::format("'--snapshot-dir' ('{}') must be a directory", BaseSnapshotDir)); - } - - if (OpenIdProviderUrl.empty() == false) - { - if (OpenIdClientId.empty()) - { - throw OptionParseException("'--openid-provider-url' requires '--openid-client-id'", options.help()); - } - - ServerOptions.AuthConfig.OpenIdProviders.push_back( - {.Name = OpenIdProviderName, .Url = OpenIdProviderUrl, .ClientId = OpenIdClientId}); - } - - ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); + BaseOptions.ApplyOptions(ServerOptions); + StorageOptions.ApplyOptions(options, ServerOptions); ParseEnvVariables(ServerOptions, Result); + ZEN_TRACE_CPU("ConfigParse"); + if (!ServerOptions.ConfigFile.empty()) { - ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile); + ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, BaseOptions.OutputConfigFile); } else { - ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, OutputConfigFile); + ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, BaseOptions.OutputConfigFile); } if (!ServerOptions.PluginsConfigFile.empty()) diff --git a/src/zenserver/config.h b/src/zenserver/config.h index d2311396e..8471ee89b 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -10,108 +10,16 @@ #include <string> #include <vector> +namespace zen::LuaConfig { +struct Options; +} +namespace cxxopts { +class Options; +class ParseResult; +} // namespace cxxopts namespace zen { -struct ZenUpstreamJupiterConfig -{ - std::string Name; - std::string Url; - std::string OAuthUrl; - std::string OAuthClientId; - std::string OAuthClientSecret; - std::string OpenIdProvider; - std::string AccessToken; - std::string Namespace; - std::string DdcNamespace; -}; - -struct ZenUpstreamZenConfig -{ - std::string Name; - std::vector<std::string> Urls; - std::vector<std::string> Dns; -}; - -enum class UpstreamCachePolicy : uint8_t -{ - Disabled = 0, - Read = 1 << 0, - Write = 1 << 1, - ReadWrite = Read | Write -}; - -struct ZenUpstreamCacheConfig -{ - ZenUpstreamJupiterConfig JupiterConfig; - ZenUpstreamZenConfig ZenConfig; - int32_t UpstreamThreadCount = 4; - int32_t ConnectTimeoutMilliseconds = 5000; - int32_t TimeoutMilliseconds = 0; - UpstreamCachePolicy CachePolicy = UpstreamCachePolicy::ReadWrite; -}; - -struct ZenCacheEvictionPolicy -{ - int32_t MaxDurationSeconds = 24 * 60 * 60; -}; - -struct ZenProjectStoreEvictionPolicy -{ - int32_t MaxDurationSeconds = 7 * 24 * 60 * 60; -}; - -struct ZenBuildStoreEvictionPolicy -{ - int32_t MaxDurationSeconds = 3 * 24 * 60 * 60; -}; - -struct ZenGcConfig -{ - // ZenCasEvictionPolicy Cas; - ZenCacheEvictionPolicy Cache; - ZenProjectStoreEvictionPolicy ProjectStore; - ZenBuildStoreEvictionPolicy BuildStore; - int32_t MonitorIntervalSeconds = 30; - int32_t IntervalSeconds = 0; - bool CollectSmallObjects = true; - bool Enabled = true; - uint64_t DiskReserveSize = 1ul << 28; - uint64_t DiskSizeSoftLimit = 0; - int32_t LightweightIntervalSeconds = 0; - uint64_t MinimumFreeDiskSpaceToAllowWrites = 1ul << 28; - bool UseGCV2 = false; - uint32_t CompactBlockUsageThresholdPercent = 90; - bool Verbose = false; - bool SingleThreaded = false; - static constexpr uint16_t GcMaxAttachmentPassCount = 256; - uint16_t AttachmentPassCount = 1; - bool StoreCacheAttachmentMetaData = false; - bool StoreProjectAttachmentMetaData = false; - bool EnableValidation = true; -}; - -struct ZenOpenIdProviderConfig -{ - std::string Name; - std::string Url; - std::string ClientId; -}; - -struct ZenAuthConfig -{ - std::vector<ZenOpenIdProviderConfig> OpenIdProviders; -}; - -struct ZenObjectStoreConfig -{ - struct BucketConfig - { - std::string Name; - std::filesystem::path Directory; - }; - - std::vector<BucketConfig> Buckets; -}; +struct ZenStorageServerOptions; struct ZenStatsConfig { @@ -120,45 +28,6 @@ struct ZenStatsConfig int StatsdPort = 8125; }; -struct ZenStructuredCacheBucketConfig -{ - uint64_t MaxBlockSize = 1ull << 30; - uint32_t PayloadAlignment = 1u << 4; - uint64_t MemCacheSizeThreshold = 1 * 1024; - uint64_t LargeObjectThreshold = 128 * 1024; - bool LimitOverwrites = false; -}; - -struct ZenStructuredCacheConfig -{ - bool Enabled = true; - bool WriteLogEnabled = false; - bool AccessLogEnabled = false; - std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>> PerBucketConfigs; - ZenStructuredCacheBucketConfig BucketConfig; - uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024; - uint64_t MemTrimIntervalSeconds = 60; - uint64_t MemMaxAgeSeconds = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count()); -}; - -struct ZenProjectStoreConfig -{ - bool StoreCacheAttachmentMetaData = false; - bool StoreProjectAttachmentMetaData = false; -}; - -struct ZenBuildStoreConfig -{ - bool Enabled = false; - uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB -}; - -struct ZenWorkspacesConfig -{ - bool Enabled = false; - bool AllowConfigurationChanges = false; -}; - struct ZenSentryConfig { bool Disable = false; @@ -185,6 +54,7 @@ struct ZenServerOptions bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements bool ShouldCrash = false; // Option for testing crash handling bool IsFirstRun = false; + std::filesystem::path ConfigFile; // Path to Lua config file std::filesystem::path SystemRootDir; // System root directory (used for machine level config) std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) std::filesystem::path DataDir; // Root directory for state (used for testing) @@ -202,25 +72,9 @@ struct ZenServerOptions std::string EncryptionIV; // 128 bit AES initialization vector ZenStatsConfig StatsConfig; -}; -struct ZenStorageServerOptions : public ZenServerOptions -{ - ZenUpstreamCacheConfig UpstreamCacheConfig; - ZenGcConfig GcConfig; - ZenAuthConfig AuthConfig; - ZenObjectStoreConfig ObjectStoreConfig; - ZenStructuredCacheConfig StructuredCacheConfig; - ZenProjectStoreConfig ProjectStoreConfig; - ZenBuildStoreConfig BuildStoreConfig; - ZenWorkspacesConfig WorksSpacesConfig; - std::filesystem::path ConfigFile; // Path to Lua config file - std::filesystem::path PluginsConfigFile; // Path to plugins config file - std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) - bool InstallService = false; // Flag used to initiate service install (temporary) - bool UninstallService = false; // Flag used to initiate service uninstall (temporary) - bool ObjectStoreEnabled = false; - std::string ScrubOptions; + bool InstallService = false; // Flag used to initiate service install (temporary) + bool UninstallService = false; // Flag used to initiate service uninstall (temporary) }; void ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOptions); @@ -228,4 +82,6 @@ void ParseCliOptions(int argc, char* argv[], ZenStorageServerOptions& ServerOpti void EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath); std::vector<CbObject> ReadAllCentralManifests(const std::filesystem::path& SystemRoot); +void AddServerConfigOptions(LuaConfig::Options& LuaOptions, ZenServerOptions& ServerOptions); + } // namespace zen diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index f1c1a9394..97847b65d 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -24,8 +24,8 @@ #include <zenutil/service.h> -#include "config.h" #include "diag/logging.h" +#include "storageconfig.h" #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> diff --git a/src/zenserver/storageconfig.cpp b/src/zenserver/storageconfig.cpp new file mode 100644 index 000000000..86bb09c21 --- /dev/null +++ b/src/zenserver/storageconfig.cpp @@ -0,0 +1,1055 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "storageconfig.h" + +#include <zencore/basicfile.h> +#include <zencore/crypto.h> +#include <zencore/except.h> +#include <zencore/logging.h> + +#include "config/luaconfig.h" + +ZEN_THIRD_PARTY_INCLUDES_START +#include <fmt/format.h> +#include <cxxopts.hpp> +#include <json11.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +void +ValidateOptions(ZenStorageServerOptions& ServerOptions) +{ + if (ServerOptions.EncryptionKey.empty() == false) + { + const auto Key = AesKey256Bit::FromString(ServerOptions.EncryptionKey); + + if (Key.IsValid() == false) + { + throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", ServerOptions.EncryptionKey), {}); + } + } + + if (ServerOptions.EncryptionIV.empty() == false) + { + const auto IV = AesIV128Bit::FromString(ServerOptions.EncryptionIV); + + if (IV.IsValid() == false) + { + throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", ServerOptions.EncryptionIV), {}); + } + } + if (ServerOptions.HttpServerConfig.ForceLoopback && ServerOptions.IsDedicated) + { + throw OptionParseException("'--dedicated' conflicts with '--http-forceloopback'", {}); + } + if (ServerOptions.GcConfig.AttachmentPassCount > ZenGcConfig::GcMaxAttachmentPassCount) + { + throw OptionParseException(fmt::format("'--gc-attachment-passes' ('{}') is invalid, maximum is {}.", + ServerOptions.GcConfig.AttachmentPassCount, + ZenGcConfig::GcMaxAttachmentPassCount), + {}); + } + if (ServerOptions.GcConfig.UseGCV2 == false) + { + ZEN_WARN("'--gc-v2=false' is deprecated, reverting to '--gc-v2=true'"); + ServerOptions.GcConfig.UseGCV2 = true; + } +} + +class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue +{ +public: + ZenStructuredCacheBucketsConfigOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const std::pair<std::string, ZenStructuredCacheBucketConfig>& Bucket : Value) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Bucket.first); + const ZenStructuredCacheBucketConfig& BucketConfig = Bucket.second; + + Writer.WriteValue("maxblocksize", fmt::format("{}", BucketConfig.MaxBlockSize)); + Writer.BeginContainer("memlayer"); + { + Writer.WriteValue("sizethreshold", fmt::format("{}", BucketConfig.MemCacheSizeThreshold)); + } + Writer.EndContainer(); + + Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("limitoverwrites", fmt::format("{}", BucketConfig.LimitOverwrites)); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional<sol::table> Buckets = Object.as<sol::table>()) + { + for (const auto& Kv : Buckets.value()) + { + if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>()) + { + ZenStructuredCacheBucketConfig BucketConfig; + std::string Name = Kv.first.as<std::string>(); + if (Name.empty()) + { + throw OptionParseException("Cache bucket option must have a name.", {}); + } + + const uint64_t MaxBlockSize = Bucket.value().get_or("maxblocksize", BucketConfig.MaxBlockSize); + if (MaxBlockSize == 0) + { + throw OptionParseException( + fmt::format("'maxblocksize' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.MaxBlockSize = MaxBlockSize; + + if (sol::optional<sol::table> Memlayer = Bucket.value().get_or("memlayer", sol::table())) + { + const uint64_t MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + if (MemCacheSizeThreshold == 0) + { + throw OptionParseException( + fmt::format("'memlayer.sizethreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + } + + const uint32_t PayloadAlignment = Bucket.value().get_or("payloadalignment", BucketConfig.PayloadAlignment); + if (PayloadAlignment == 0 || !IsPow2(PayloadAlignment)) + { + throw OptionParseException( + fmt::format( + "'payloadalignment' option for cache bucket '{}' is invalid. It needs to be non-zero and a power of two.", + Name), + {}); + } + BucketConfig.PayloadAlignment = PayloadAlignment; + + const uint64_t LargeObjectThreshold = Bucket.value().get_or("largeobjectthreshold", BucketConfig.LargeObjectThreshold); + if (LargeObjectThreshold == 0) + { + throw OptionParseException( + fmt::format("'largeobjectthreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.LargeObjectThreshold = LargeObjectThreshold; + + BucketConfig.LimitOverwrites = Bucket.value().get_or("limitoverwrites", BucketConfig.LimitOverwrites); + + Value.push_back(std::make_pair(std::move(Name), BucketConfig)); + } + } + } + } + std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value; +}; + +UpstreamCachePolicy +ParseUpstreamCachePolicy(std::string_view Options) +{ + if (Options == "readonly") + { + return UpstreamCachePolicy::Read; + } + else if (Options == "writeonly") + { + return UpstreamCachePolicy::Write; + } + else if (Options == "disabled") + { + return UpstreamCachePolicy::Disabled; + } + else + { + return UpstreamCachePolicy::ReadWrite; + } +} + +ZenObjectStoreConfig +ParseBucketConfigs(std::span<std::string> Buckets) +{ + using namespace std::literals; + + ZenObjectStoreConfig Cfg; + + // split bucket args in the form of "{BucketName};{LocalPath}" + for (std::string_view Bucket : Buckets) + { + ZenObjectStoreConfig::BucketConfig NewBucket; + + if (auto Idx = Bucket.find_first_of(";"); Idx != std::string_view::npos) + { + NewBucket.Name = Bucket.substr(0, Idx); + NewBucket.Directory = Bucket.substr(Idx + 1); + } + else + { + NewBucket.Name = Bucket; + } + + Cfg.Buckets.push_back(std::move(NewBucket)); + } + + return Cfg; +} + +class CachePolicyOption : public LuaConfig::OptionValue +{ +public: + CachePolicyOption(UpstreamCachePolicy& Value) : Value(Value) {} + virtual void Print(std::string_view, StringBuilderBase& StringBuilder) override + { + switch (Value) + { + case UpstreamCachePolicy::Read: + StringBuilder.Append("readonly"); + break; + case UpstreamCachePolicy::Write: + StringBuilder.Append("writeonly"); + break; + case UpstreamCachePolicy::Disabled: + StringBuilder.Append("disabled"); + break; + case UpstreamCachePolicy::ReadWrite: + StringBuilder.Append("readwrite"); + break; + default: + ZEN_ASSERT(false); + } + } + virtual void Parse(sol::object Object) override + { + std::string PolicyString = Object.as<std::string>(); + if (PolicyString == "readonly") + { + Value = UpstreamCachePolicy::Read; + } + else if (PolicyString == "writeonly") + { + Value = UpstreamCachePolicy::Write; + } + else if (PolicyString == "disabled") + { + Value = UpstreamCachePolicy::Disabled; + } + else if (PolicyString == "readwrite") + { + Value = UpstreamCachePolicy::ReadWrite; + } + } + UpstreamCachePolicy& Value; +}; + +class ZenAuthConfigOption : public LuaConfig::OptionValue +{ +public: + ZenAuthConfigOption(ZenAuthConfig& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.OpenIdProviders.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const ZenOpenIdProviderConfig& Config : Value.OpenIdProviders) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Config.Name); + Writer.WriteValue("url", Config.Url); + Writer.WriteValue("clientid", Config.ClientId); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional<sol::table> OpenIdProviders = Object.as<sol::table>()) + { + for (const auto& Kv : OpenIdProviders.value()) + { + if (sol::optional<sol::table> OpenIdProvider = Kv.second.as<sol::table>()) + { + std::string Name = OpenIdProvider.value().get_or("name", std::string("Default")); + std::string Url = OpenIdProvider.value().get_or("url", std::string()); + std::string ClientId = OpenIdProvider.value().get_or("clientid", std::string()); + + Value.OpenIdProviders.push_back({.Name = std::move(Name), .Url = std::move(Url), .ClientId = std::move(ClientId)}); + } + } + } + } + ZenAuthConfig& Value; +}; + +class ZenObjectStoreConfigOption : public LuaConfig::OptionValue +{ +public: + ZenObjectStoreConfigOption(ZenObjectStoreConfig& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.Buckets.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const ZenObjectStoreConfig::BucketConfig& Config : Value.Buckets) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Config.Name); + std::string Directory = Config.Directory.string(); + LuaConfig::EscapeBackslash(Directory); + Writer.WriteValue("directory", Directory); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional<sol::table> Buckets = Object.as<sol::table>()) + { + for (const auto& Kv : Buckets.value()) + { + if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>()) + { + std::string Name = Bucket.value().get_or("name", std::string("Default")); + std::string Directory = Bucket.value().get_or("directory", std::string()); + + Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)}); + } + } + } + } + ZenObjectStoreConfig& Value; +}; + +std::shared_ptr<LuaConfig::OptionValue> +MakeOption(UpstreamCachePolicy& Value) +{ + return std::make_shared<CachePolicyOption>(Value); +}; + +std::shared_ptr<LuaConfig::OptionValue> +MakeOption(ZenAuthConfig& Value) +{ + return std::make_shared<ZenAuthConfigOption>(Value); +}; + +std::shared_ptr<LuaConfig::OptionValue> +MakeOption(ZenObjectStoreConfig& Value) +{ + return std::make_shared<ZenObjectStoreConfigOption>(Value); +}; + +std::shared_ptr<LuaConfig::OptionValue> +MakeOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value) +{ + return std::make_shared<ZenStructuredCacheBucketsConfigOption>(Value); +}; + +void +ParseConfigFile(const std::filesystem::path& Path, + ZenStorageServerOptions& ServerOptions, + const cxxopts::ParseResult& CmdLineResult, + std::string_view OutputConfigFile) +{ + ZEN_TRACE_CPU("ParseConfigFile"); + + using namespace std::literals; + + LuaConfig::Options LuaOptions; + + AddServerConfigOptions(LuaOptions, ServerOptions); + + ////// server + LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv); + + ////// objectstore + LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); + LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); + + ////// buildsstore + LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); + LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); + + ////// cache + LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled); + LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log"sv); + LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"sv); + + LuaOptions.AddOption("cache.buckets"sv, ServerOptions.StructuredCacheConfig.PerBucketConfigs, "cache.buckets"sv); + + LuaOptions.AddOption("cache.memlayer.sizethreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + "cache-memlayer-sizethreshold"sv); + LuaOptions.AddOption("cache.memlayer.targetfootprint"sv, + ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes, + "cache-memlayer-targetfootprint"sv); + LuaOptions.AddOption("cache.memlayer.triminterval"sv, + ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds, + "cache-memlayer-triminterval"sv); + LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"sv); + + LuaOptions.AddOption("cache.bucket.maxblocksize"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + "cache-bucket-maxblocksize"sv); + LuaOptions.AddOption("cache.bucket.memlayer.sizethreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + "cache-bucket-memlayer-sizethreshold"sv); + LuaOptions.AddOption("cache.bucket.payloadalignment"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + "cache-bucket-payloadalignment"sv); + LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + "cache-bucket-largeobjectthreshold"sv); + LuaOptions.AddOption("cache.bucket.limitoverwrites"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites, + "cache-bucket-limit-overwrites"sv); + + ////// cache.upstream + LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv); + LuaOptions.AddOption("cache.upstream.upstreamthreadcount"sv, + ServerOptions.UpstreamCacheConfig.UpstreamThreadCount, + "upstream-thread-count"sv); + LuaOptions.AddOption("cache.upstream.connecttimeoutms"sv, + ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds, + "upstream-connect-timeout-ms"sv); + LuaOptions.AddOption("cache.upstream.timeoutms"sv, ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds, "upstream-timeout-ms"sv); + + ////// cache.upstream.jupiter + LuaOptions.AddOption("cache.upstream.jupiter.name"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Name); + LuaOptions.AddOption("cache.upstream.jupiter.url"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Url, "upstream-jupiter-url"sv); + LuaOptions.AddOption("cache.upstream.jupiter.oauthprovider"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl, + "upstream-jupiter-oauth-url"sv); + LuaOptions.AddOption("cache.upstream.jupiter.oauthclientid"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId, + "upstream-jupiter-oauth-clientid"); + LuaOptions.AddOption("cache.upstream.jupiter.oauthclientsecret"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret, + "upstream-jupiter-oauth-clientsecret"sv); + LuaOptions.AddOption("cache.upstream.jupiter.openidprovider"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider, + "upstream-jupiter-openid-provider"sv); + LuaOptions.AddOption("cache.upstream.jupiter.token"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken, + "upstream-jupiter-token"sv); + LuaOptions.AddOption("cache.upstream.jupiter.namespace"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace, + "upstream-jupiter-namespace"sv); + LuaOptions.AddOption("cache.upstream.jupiter.ddcnamespace"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace, + "upstream-jupiter-namespace-ddc"sv); + + ////// cache.upstream.zen + // LuaOptions.AddOption("cache.upstream.zen"sv, ServerOptions.UpstreamCacheConfig.ZenConfig); + LuaOptions.AddOption("cache.upstream.zen.name"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Name); + LuaOptions.AddOption("cache.upstream.zen.dns"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Dns); + LuaOptions.AddOption("cache.upstream.zen.url"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Urls); + + ////// gc + LuaOptions.AddOption("gc.enabled"sv, ServerOptions.GcConfig.Enabled, "gc-enabled"sv); + LuaOptions.AddOption("gc.v2"sv, ServerOptions.GcConfig.UseGCV2, "gc-v2"sv); + + LuaOptions.AddOption("gc.monitorintervalseconds"sv, ServerOptions.GcConfig.MonitorIntervalSeconds, "gc-monitor-interval-seconds"sv); + LuaOptions.AddOption("gc.intervalseconds"sv, ServerOptions.GcConfig.IntervalSeconds, "gc-interval-seconds"sv); + LuaOptions.AddOption("gc.collectsmallobjects"sv, ServerOptions.GcConfig.CollectSmallObjects, "gc-small-objects"sv); + LuaOptions.AddOption("gc.diskreservesize"sv, ServerOptions.GcConfig.DiskReserveSize, "disk-reserve-size"sv); + LuaOptions.AddOption("gc.disksizesoftlimit"sv, ServerOptions.GcConfig.DiskSizeSoftLimit, "gc-disksize-softlimit"sv); + LuaOptions.AddOption("gc.lowdiskspacethreshold"sv, + ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, + "gc-low-diskspace-threshold"sv); + LuaOptions.AddOption("gc.lightweightintervalseconds"sv, + ServerOptions.GcConfig.LightweightIntervalSeconds, + "gc-lightweight-interval-seconds"sv); + LuaOptions.AddOption("gc.compactblockthreshold"sv, + ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, + "gc-compactblock-threshold"sv); + LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv); + LuaOptions.AddOption("gc.single-threaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv); + LuaOptions.AddOption("gc.cache.attachment.store"sv, ServerOptions.GcConfig.StoreCacheAttachmentMetaData, "gc-cache-attachment-store"); + LuaOptions.AddOption("gc.projectstore.attachment.store"sv, + ServerOptions.GcConfig.StoreProjectAttachmentMetaData, + "gc-projectstore-attachment-store"); + LuaOptions.AddOption("gc.attachment.passes"sv, ServerOptions.GcConfig.AttachmentPassCount, "gc-attachment-passes"sv); + LuaOptions.AddOption("gc.validation"sv, ServerOptions.GcConfig.EnableValidation, "gc-validation"); + + LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv); + LuaOptions.AddOption("gc.projectstore.duration.seconds"sv, + ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds, + "gc-projectstore-duration-seconds"); + LuaOptions.AddOption("gc.buildstore.duration.seconds"sv, + ServerOptions.GcConfig.BuildStore.MaxDurationSeconds, + "gc-buildstore-duration-seconds"); + + ////// security + LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); + LuaOptions.AddOption("security.encryptionaesiv"sv, ServerOptions.EncryptionIV, "encryption-aes-iv"sv); + LuaOptions.AddOption("security.openidproviders"sv, ServerOptions.AuthConfig); + + ////// workspaces + LuaOptions.AddOption("workspaces.enabled"sv, ServerOptions.WorksSpacesConfig.Enabled, "workspaces-enabled"sv); + LuaOptions.AddOption("workspaces.allowconfigchanges"sv, + ServerOptions.WorksSpacesConfig.AllowConfigurationChanges, + "workspaces-allow-changes"sv); + + LuaOptions.Parse(Path, CmdLineResult); + + // These have special command line processing so we make sure we export them if they were configured on command line + if (!ServerOptions.AuthConfig.OpenIdProviders.empty()) + { + LuaOptions.Touch("security.openidproviders"sv); + } + if (!ServerOptions.ObjectStoreConfig.Buckets.empty()) + { + LuaOptions.Touch("server.objectstore.buckets"sv); + } + if (!ServerOptions.StructuredCacheConfig.PerBucketConfigs.empty()) + { + LuaOptions.Touch("cache.buckets"sv); + } + + if (!OutputConfigFile.empty()) + { + std::filesystem::path WritePath(MakeSafeAbsolutePath(OutputConfigFile)); + ExtendableStringBuilder<512> ConfigStringBuilder; + LuaOptions.Print(ConfigStringBuilder, CmdLineResult); + BasicFile Output; + Output.Open(WritePath, BasicFile::Mode::kTruncate); + Output.Write(ConfigStringBuilder.Data(), ConfigStringBuilder.Size(), 0); + } +} + +void +ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort) +{ + using namespace std::literals; + + IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); + std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize()); + std::string JsonError; + json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError); + if (!JsonError.empty()) + { + ZEN_WARN("failed parsing plugins config file '{}'. Reason: '{}'", Path, JsonError); + return; + } + for (const json11::Json& PluginInfo : PluginsInfo.array_items()) + { + if (!PluginInfo.is_object()) + { + ZEN_WARN("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", Path, PluginInfo.dump()); + continue; + } + + HttpServerPluginConfig Config = {}; + + bool bNeedsPort = true; + + for (const std::pair<const std::string, json11::Json>& Items : PluginInfo.object_items()) + { + if (!Items.second.is_string()) + { + ZEN_WARN("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", + Path, + Items.second.dump()); + continue; + } + + const std::string& Name = Items.first; + const std::string& Value = Items.second.string_value(); + + if (Name == "name"sv) + Config.PluginName = Value; + else + { + Config.PluginOptions.push_back({Name, Value}); + + if (Name == "port"sv) + { + bNeedsPort = false; + } + } + } + + // add a default base port in case if json config didn't provide one + if (bNeedsPort) + { + Config.PluginOptions.push_back({"port", std::to_string(BasePort)}); + } + + ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config); + } +} + +void +ZenStorageServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_options()("snapshot-dir", + "Specify a snapshot of server state to mirror into the persistence root at startup", + cxxopts::value<std::string>(BaseSnapshotDir)); + options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value<std::string>(PluginsConfigFile)); + options.add_options()("scrub", + "Validate state at startup", + cxxopts::value(ServerOptions.ScrubOptions)->implicit_value("yes"), + "(nocas,nogc,nodelete,yes,no)*"); + + AddSecurityOptions(options, ServerOptions); + AddCacheOptions(options, ServerOptions); + AddGcOptions(options, ServerOptions); + AddObjectStoreOptions(options, ServerOptions); + AddBuildStoreOptions(options, ServerOptions); + AddWorkspacesOptions(options, ServerOptions); +} + +void +ZenStorageServerCmdLineOptions::AddSecurityOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("security", + "", + "encryption-aes-key", + "256 bit AES encryption key", + cxxopts::value<std::string>(ServerOptions.EncryptionKey), + ""); + + options.add_option("security", + "", + "encryption-aes-iv", + "128 bit AES encryption initialization vector", + cxxopts::value<std::string>(ServerOptions.EncryptionIV), + ""); + + options.add_option("security", + "", + "openid-provider-name", + "Open ID provider name", + cxxopts::value<std::string>(OpenIdProviderName), + "Default"); + + options.add_option("security", "", "openid-provider-url", "Open ID provider URL", cxxopts::value<std::string>(OpenIdProviderUrl), ""); + options.add_option("security", "", "openid-client-id", "Open ID client ID", cxxopts::value<std::string>(OpenIdClientId), ""); +} + +void +ZenStorageServerCmdLineOptions::AddCacheOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("cache", + "", + "upstream-cache-policy", + "", + cxxopts::value<std::string>(UpstreamCachePolicyOptions)->default_value(""), + "Upstream cache policy (readwrite|readonly|writeonly|disabled)"); + + options.add_option("cache", + "", + "upstream-jupiter-url", + "URL to a Jupiter instance", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.Url)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-oauth-url", + "URL to the OAuth provier", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-oauth-clientid", + "The OAuth client ID", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-oauth-clientsecret", + "The OAuth client secret", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-openid-provider", + "Name of a registered Open ID provider", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-token", + "A static authentication token", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-namespace", + "The Common Blob Store API namespace", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-namespace-ddc", + "The lecacy DDC namespace", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-zen-url", + "URL to remote Zen server. Use a comma separated list to choose the one with the best latency.", + cxxopts::value<std::vector<std::string>>(ServerOptions.UpstreamCacheConfig.ZenConfig.Urls), + ""); + + options.add_option("cache", + "", + "upstream-zen-dns", + "DNS that resolves to one or more Zen server instance(s)", + cxxopts::value<std::vector<std::string>>(ServerOptions.UpstreamCacheConfig.ZenConfig.Dns), + ""); + + options.add_option("cache", + "", + "upstream-thread-count", + "Number of threads used for upstream procsssing", + cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"), + ""); + + options.add_option("cache", + "", + "upstream-connect-timeout-ms", + "Connect timeout in millisecond(s). Default 5000 ms.", + cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds)->default_value("5000"), + ""); + + options.add_option("cache", + "", + "upstream-timeout-ms", + "Timeout in millisecond(s). Default 0 ms", + cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"), + ""); + + options.add_option("cache", + "", + "cache-write-log", + "Whether cache write log is enabled", + cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("false"), + ""); + + options.add_option("cache", + "", + "cache-access-log", + "Whether cache access log is enabled", + cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"), + ""); + + options.add_option( + "cache", + "", + "cache-memlayer-sizethreshold", + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory " + "caching. " + "Obsolete, replaced by `--cache-bucket-memlayer-sizethreshold`", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), + ""); + + options.add_option("cache", + "", + "cache-memlayer-targetfootprint", + "Max allowed memory used by cache memory layer per namespace in bytes. Default set to 536870912 (512 Mb).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes)->default_value("536870912"), + ""); + + options.add_option("cache", + "", + "cache-memlayer-triminterval", + "Minimum time between each attempt to trim cache memory layers in seconds. Default set to 60 (1 min).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds)->default_value("60"), + ""); + + options.add_option("cache", + "", + "cache-memlayer-maxage", + "Maximum age of payloads when trimming cache memory layers in seconds. Default set to 86400 (1 day).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"), + ""); + + options.add_option("cache", + "", + "cache-bucket-maxblocksize", + "Max size of cache bucket blocks. Default set to 1073741824 (1GB).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize)->default_value("1073741824"), + ""); + + options.add_option("cache", + "", + "cache-bucket-payloadalignment", + "Payload alignement for cache bucket blocks. Default set to 16.", + cxxopts::value<uint32_t>(ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment)->default_value("16"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-largeobjectthreshold", + "Threshold for storing cache bucket values as loose files. Default set to 131072 (128 KB).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold)->default_value("131072"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-memlayer-sizethreshold", + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory " + "caching.", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), + ""); + + options.add_option("cache", + "", + "cache-bucket-limit-overwrites", + "Whether to require policy flag pattern before allowing overwrites in cache bucket", + cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites)->default_value("false"), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddGcOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("gc", + "", + "gc-cache-attachment-store", + "Enable storing attachments referenced by a cache record in block store meta data.", + cxxopts::value<bool>(ServerOptions.GcConfig.StoreCacheAttachmentMetaData)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-projectstore-attachment-store", + "Enable storing attachments referenced by project oplogs in meta data.", + cxxopts::value<bool>(ServerOptions.GcConfig.StoreProjectAttachmentMetaData)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-validation", + "Enable validation of references after full GC.", + cxxopts::value<bool>(ServerOptions.GcConfig.EnableValidation)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-enabled", + "Whether garbage collection is enabled or not.", + cxxopts::value<bool>(ServerOptions.GcConfig.Enabled)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-v2", + "Use V2 of GC implementation or not.", + cxxopts::value<bool>(ServerOptions.GcConfig.UseGCV2)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-small-objects", + "Whether garbage collection of small objects is enabled or not.", + cxxopts::value<bool>(ServerOptions.GcConfig.CollectSmallObjects)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-interval-seconds", + "Garbage collection interval in seconds. Default set to 3600 (1 hour).", + cxxopts::value<int32_t>(ServerOptions.GcConfig.IntervalSeconds)->default_value("3600"), + ""); + + options.add_option("gc", + "", + "gc-lightweight-interval-seconds", + "Lightweight garbage collection interval in seconds. Default set to 900 (30 min).", + cxxopts::value<int32_t>(ServerOptions.GcConfig.LightweightIntervalSeconds)->default_value("900"), + ""); + + options.add_option("gc", + "", + "gc-cache-duration-seconds", + "Max duration in seconds before Z$ entries get evicted. Default set to 1209600 (2 weeks)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.Cache.MaxDurationSeconds)->default_value("1209600"), + ""); + + options.add_option("gc", + "", + "gc-projectstore-duration-seconds", + "Max duration in seconds before project store entries get evicted. Default set to 1209600 (2 weeks)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds)->default_value("1209600"), + ""); + + options.add_option("gc", + "", + "gc-buildstore-duration-seconds", + "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"), + ""); + + options.add_option("gc", + "", + "disk-reserve-size", + "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.", + cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"), + ""); + + options.add_option("gc", + "", + "gc-monitor-interval-seconds", + "Garbage collection monitoring interval in seconds. Default set to 30 (30 seconds)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"), + ""); + + options.add_option("gc", + "", + "gc-low-diskspace-threshold", + "Minimum free space on disk to allow writes to disk. Default set to 268435456 (256 Mb). Set to zero to disable.", + cxxopts::value<uint64_t>(ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites)->default_value("268435456"), + ""); + + options.add_option("gc", + "", + "gc-disksize-softlimit", + "Garbage collection disk usage soft limit. Default set to 0 (Off).", + cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskSizeSoftLimit)->default_value("0"), + ""); + + options.add_option("gc", + "", + "gc-compactblock-threshold", + "Garbage collection - how much of a compact block should be used to skip compacting the block. 0 - compact only " + "empty eligible blocks, 100 - compact all non-full eligible blocks.", + cxxopts::value<uint32_t>(ServerOptions.GcConfig.CompactBlockUsageThresholdPercent)->default_value("60"), + ""); + + options.add_option("gc", + "", + "gc-verbose", + "Enable verbose logging for GC.", + cxxopts::value<bool>(ServerOptions.GcConfig.Verbose)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-single-threaded", + "Force GC to run single threaded.", + cxxopts::value<bool>(ServerOptions.GcConfig.SingleThreaded)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-attachment-passes", + "Limit the range of unreferenced attachments included in GC check by breaking it into passes. Default is one pass " + "which includes all the attachments.", + cxxopts::value<uint16_t>(ServerOptions.GcConfig.AttachmentPassCount)->default_value("1"), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddObjectStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("objectstore", + "", + "objectstore-enabled", + "Whether the object store is enabled or not.", + cxxopts::value<bool>(ServerOptions.ObjectStoreEnabled)->default_value("false"), + ""); + + options.add_option("objectstore", + "", + "objectstore-bucket", + "Object store bucket mappings.", + cxxopts::value<std::vector<std::string>>(BucketConfigs), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddBuildStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("buildstore", + "", + "buildstore-enabled", + "Whether the builds store is enabled or not.", + cxxopts::value<bool>(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), + ""); + options.add_option("buildstore", + "", + "buildstore-disksizelimit", + "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)", + cxxopts::value<uint64_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddWorkspacesOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("workspaces", + "", + "workspaces-enabled", + "", + cxxopts::value<bool>(ServerOptions.WorksSpacesConfig.Enabled)->default_value("true"), + "Enable workspaces support with folder sharing"); + + options.add_option("workspaces", + "", + "workspaces-allow-changes", + "", + cxxopts::value<bool>(ServerOptions.WorksSpacesConfig.AllowConfigurationChanges)->default_value("false"), + "Allow adding/modifying/deleting of workspace and shares via http endpoint"); +} + +void +ZenStorageServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); + ServerOptions.PluginsConfigFile = MakeSafeAbsolutePath(PluginsConfigFile); + ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); + + if (!BaseSnapshotDir.empty()) + { + if (ServerOptions.DataDir.empty()) + throw OptionParseException("'--snapshot-dir' requires '--data-dir'", options.help()); + + if (!IsDir(ServerOptions.BaseSnapshotDir)) + throw std::runtime_error(fmt::format("'--snapshot-dir' ('{}') must be a directory", ServerOptions.BaseSnapshotDir)); + } + + if (OpenIdProviderUrl.empty() == false) + { + if (OpenIdClientId.empty()) + { + throw OptionParseException("'--openid-provider-url' requires '--openid-client-id'", options.help()); + } + + ServerOptions.AuthConfig.OpenIdProviders.push_back( + {.Name = OpenIdProviderName, .Url = OpenIdProviderUrl, .ClientId = OpenIdClientId}); + } + + ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); +} + +} // namespace zen diff --git a/src/zenserver/storageconfig.h b/src/zenserver/storageconfig.h new file mode 100644 index 000000000..2c3197226 --- /dev/null +++ b/src/zenserver/storageconfig.h @@ -0,0 +1,203 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "config.h" + +namespace zen { + +struct ZenUpstreamJupiterConfig +{ + std::string Name; + std::string Url; + std::string OAuthUrl; + std::string OAuthClientId; + std::string OAuthClientSecret; + std::string OpenIdProvider; + std::string AccessToken; + std::string Namespace; + std::string DdcNamespace; +}; + +struct ZenUpstreamZenConfig +{ + std::string Name; + std::vector<std::string> Urls; + std::vector<std::string> Dns; +}; + +enum class UpstreamCachePolicy : uint8_t +{ + Disabled = 0, + Read = 1 << 0, + Write = 1 << 1, + ReadWrite = Read | Write +}; + +struct ZenUpstreamCacheConfig +{ + ZenUpstreamJupiterConfig JupiterConfig; + ZenUpstreamZenConfig ZenConfig; + int32_t UpstreamThreadCount = 4; + int32_t ConnectTimeoutMilliseconds = 5000; + int32_t TimeoutMilliseconds = 0; + UpstreamCachePolicy CachePolicy = UpstreamCachePolicy::ReadWrite; +}; + +struct ZenCacheEvictionPolicy +{ + int32_t MaxDurationSeconds = 24 * 60 * 60; +}; + +struct ZenProjectStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 7 * 24 * 60 * 60; +}; + +struct ZenBuildStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 3 * 24 * 60 * 60; +}; + +struct ZenGcConfig +{ + // ZenCasEvictionPolicy Cas; + ZenCacheEvictionPolicy Cache; + ZenProjectStoreEvictionPolicy ProjectStore; + ZenBuildStoreEvictionPolicy BuildStore; + int32_t MonitorIntervalSeconds = 30; + int32_t IntervalSeconds = 0; + bool CollectSmallObjects = true; + bool Enabled = true; + uint64_t DiskReserveSize = 1ul << 28; + uint64_t DiskSizeSoftLimit = 0; + int32_t LightweightIntervalSeconds = 0; + uint64_t MinimumFreeDiskSpaceToAllowWrites = 1ul << 28; + bool UseGCV2 = false; + uint32_t CompactBlockUsageThresholdPercent = 90; + bool Verbose = false; + bool SingleThreaded = false; + static constexpr uint16_t GcMaxAttachmentPassCount = 256; + uint16_t AttachmentPassCount = 1; + bool StoreCacheAttachmentMetaData = false; + bool StoreProjectAttachmentMetaData = false; + bool EnableValidation = true; +}; + +struct ZenOpenIdProviderConfig +{ + std::string Name; + std::string Url; + std::string ClientId; +}; + +struct ZenAuthConfig +{ + std::vector<ZenOpenIdProviderConfig> OpenIdProviders; +}; + +struct ZenObjectStoreConfig +{ + struct BucketConfig + { + std::string Name; + std::filesystem::path Directory; + }; + + std::vector<BucketConfig> Buckets; +}; + +struct ZenStructuredCacheBucketConfig +{ + uint64_t MaxBlockSize = 1ull << 30; + uint32_t PayloadAlignment = 1u << 4; + uint64_t MemCacheSizeThreshold = 1 * 1024; + uint64_t LargeObjectThreshold = 128 * 1024; + bool LimitOverwrites = false; +}; + +struct ZenStructuredCacheConfig +{ + bool Enabled = true; + bool WriteLogEnabled = false; + bool AccessLogEnabled = false; + std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>> PerBucketConfigs; + ZenStructuredCacheBucketConfig BucketConfig; + uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024; + uint64_t MemTrimIntervalSeconds = 60; + uint64_t MemMaxAgeSeconds = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count()); +}; + +struct ZenProjectStoreConfig +{ + bool StoreCacheAttachmentMetaData = false; + bool StoreProjectAttachmentMetaData = false; +}; + +struct ZenBuildStoreConfig +{ + bool Enabled = false; + uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB +}; + +struct ZenWorkspacesConfig +{ + bool Enabled = false; + bool AllowConfigurationChanges = false; +}; + +struct ZenStorageServerOptions : public ZenServerOptions +{ + ZenUpstreamCacheConfig UpstreamCacheConfig; + ZenGcConfig GcConfig; + ZenAuthConfig AuthConfig; + ZenObjectStoreConfig ObjectStoreConfig; + ZenStructuredCacheConfig StructuredCacheConfig; + ZenProjectStoreConfig ProjectStoreConfig; + ZenBuildStoreConfig BuildStoreConfig; + ZenWorkspacesConfig WorksSpacesConfig; + std::filesystem::path PluginsConfigFile; // Path to plugins config file + std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) + bool ObjectStoreEnabled = false; + std::string ScrubOptions; +}; + +void ParseConfigFile(const std::filesystem::path& Path, + ZenStorageServerOptions& ServerOptions, + const cxxopts::ParseResult& CmdLineResult, + std::string_view OutputConfigFile); + +void ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort); +void ValidateOptions(ZenStorageServerOptions& ServerOptions); + +struct ZenStorageServerCmdLineOptions +{ + // Note to those adding future options; std::filesystem::path-type options + // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> + // stream operator to convert argv value into the options type. std::fs::path + // expects paths in streams to be quoted but argv paths are unquoted. By + // going into a std::string first, paths with whitespace parse correctly. + std::string PluginsConfigFile; + std::string BaseSnapshotDir; + + void AddCliOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + void ApplyOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + std::string OpenIdProviderName; + std::string OpenIdProviderUrl; + std::string OpenIdClientId; + + void AddSecurityOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + std::string UpstreamCachePolicyOptions; + + void AddCacheOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + void AddGcOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + std::vector<std::string> BucketConfigs; + + void AddObjectStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + void AddBuildStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + void AddWorkspacesOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); +}; + +} // namespace zen diff --git a/src/zenserver/zenstorageserver.cpp b/src/zenserver/zenstorageserver.cpp index a8dd65372..73896512d 100644 --- a/src/zenserver/zenstorageserver.cpp +++ b/src/zenserver/zenstorageserver.cpp @@ -54,8 +54,8 @@ ZEN_THIRD_PARTY_INCLUDES_END ////////////////////////////////////////////////////////////////////////// -#include "config.h" #include "diag/logging.h" +#include "storageconfig.h" #include <zencore/memory/llm.h> |