// Copyright Epic Games, Inc. All Rights Reserved. #pragma once #include "hubinstancestate.h" #include "hydrationdefaults.h" #include "resourcemetrics.h" #include "zenserver.h" #include #include namespace cxxopts { class Options; } namespace zen::LuaConfig { struct Options; } namespace zen { class HttpApiService; class HttpFrontendService; class HttpHubService; class HttpProxyHandler; struct ZenHubWatchdogConfig { uint32_t CycleIntervalMs = 3000; uint32_t CycleProcessingBudgetMs = 500; uint32_t InstanceCheckThrottleMs = 5; uint32_t ProvisionedInactivityTimeoutSeconds = 600; uint32_t HibernatedInactivityTimeoutSeconds = 1800; uint32_t InactivityCheckMarginSeconds = 60; // Activity check is triggered this far before the inactivity timeout uint32_t ActivityCheckConnectTimeoutMs = 100; uint32_t ActivityCheckRequestTimeoutMs = 200; }; struct ZenHubServerConfig : public ZenServerConfig { std::string UpstreamNotificationEndpoint; std::string InstanceId; // For use in notifications std::string ConsulEndpoint; // If set, enables Consul service registration std::string ConsulTokenEnv; // Environment variable name to read a Consul token from; defaults to CONSUL_HTTP_TOKEN if empty uint32_t ConsulHealthIntervalSeconds = 10; // Interval in seconds between Consul health checks uint32_t ConsulDeregisterAfterSeconds = 30; // Seconds before Consul deregisters an unhealthy service bool ConsulRegisterHub = true; // Whether to register the hub parent service with Consul (instance registration unaffected) uint16_t HubBasePortNumber = 21000; int HubInstanceLimit = 1000; bool HubUseJobObject = true; bool HubEnableHydration = true; // Load instance state from hydration target on provision bool HubEnableDehydration = true; // Save instance state to hydration target on deprovision bool HubHydrationPackEnabled = true; // Concatenate small files into raw CAS pack blobs during dehydrate uint64_t HubHydrationPackThresholdBytes = DefaultPackThresholdBytes; // Files strictly smaller than this are pack candidates uint64_t HubHydrationMaxPackBytes = DefaultMaxPackBytes; // Upper bound on a pack's concatenation size bool HubHydrationAsyncEnabled = true; // Route S3 hydration through AsyncHttpClient uint32_t HubHydrationAsyncMaxConcurrentRequests = 128; // Hub-wide cap on concurrent S3 hydration requests (only when HubHydrationAsyncEnabled) std::string HubInstanceHttpClass = "asio"; std::string HubInstanceMalloc; std::string HubInstanceTrace; std::string HubInstanceTraceHost; std::string HubInstanceTraceFile; uint32_t HubInstanceHttpThreadCount = 0; // Automatic uint32_t HubInstanceProvisionThreadCount = 0; // Hub-wide hydrate/dehydrate scheduling pool size uint32_t HubInstanceSpawnThreadCount = 0; // Hub-wide child process spawn/despawn pool size uint32_t HubHydrationThreadCount = 0; // Internal hydration parallelism (per-file) int HubInstanceCoreLimit = 0; // Automatic std::filesystem::path HubInstanceConfigPath; // Path to Lua config file std::string HydrationTargetSpecification; // hydration/dehydration target specification std::filesystem::path HydrationTargetConfigPath; // path to JSON config file (mutually exclusive with HydrationTargetSpecification) ZenHubWatchdogConfig WatchdogConfig; uint64_t HubProvisionDiskLimitBytes = 0; uint32_t HubProvisionDiskLimitPercent = 0; uint64_t HubProvisionMemoryLimitBytes = 0; uint32_t HubProvisionMemoryLimitPercent = 0; }; class Hub; struct HubProvisionedInstanceInfo; struct ZenHubServerConfigurator : public ZenServerConfiguratorBase { ZenHubServerConfigurator(ZenHubServerConfig& ServerOptions) : ZenServerConfiguratorBase(ServerOptions), m_ServerOptions(ServerOptions) { } ~ZenHubServerConfigurator() = default; private: virtual void AddCliOptions(cxxopts::Options& Options) override; virtual void AddConfigOptions(LuaConfig::Options& Options) override; virtual void ApplyOptions(cxxopts::Options& Options) override; virtual void OnConfigFileParsed(LuaConfig::Options& LuaOptions) override; virtual void ValidateOptions() override; ZenHubServerConfig& m_ServerOptions; }; class ZenHubServerMain : public ZenServerMain { public: ZenHubServerMain(ZenHubServerConfig& ServerOptions); virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override; ZenHubServerMain(const ZenHubServerMain&) = delete; ZenHubServerMain& operator=(const ZenHubServerMain&) = delete; typedef ZenHubServerConfig Config; typedef ZenHubServerConfigurator Configurator; private: ZenHubServerConfig& m_ServerOptions; }; class ZenHubServer : public ZenServerBase { ZenHubServer& operator=(ZenHubServer&&) = delete; ZenHubServer(ZenHubServer&&) = delete; public: ZenHubServer(); ~ZenHubServer(); int Initialize(const ZenHubServerConfig& ServerConfig, ZenServerState::ZenServerEntry* ServerEntry); void Run(); void Cleanup(); void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } private: void OnModuleStateChanged(std::string_view HubInstanceId, std::string_view ModuleId, const HubProvisionedInstanceInfo& Info, HubInstanceState PreviousState, HubInstanceState NewState); std::filesystem::path m_DataRoot; std::filesystem::path m_ContentRoot; bool m_DebugOptionForcedCrash = false; std::unique_ptr m_Proxy; std::unique_ptr m_ProvisionPool; std::unique_ptr m_SpawnPool; std::unique_ptr m_HydrationPool; std::unique_ptr m_Hub; std::unique_ptr m_HubService; std::unique_ptr m_ApiService; std::unique_ptr m_FrontendService; std::unique_ptr m_ConsulClient; std::unique_ptr m_ConsulRegistration; uint32_t m_ConsulHealthIntervalSeconds = 10; uint32_t m_ConsulDeregisterAfterSeconds = 30; static ResourceMetrics ResolveLimits(const ZenHubServerConfig& ServerConfig); void InitializeState(const ZenHubServerConfig& ServerConfig); void InitializeServices(const ZenHubServerConfig& ServerConfig); void RegisterServices(const ZenHubServerConfig& ServerConfig); void InitializeConsulRegistration(const ZenHubServerConfig& ServerConfig, int EffectivePort); }; } // namespace zen