diff options
Diffstat (limited to 'src/zenserver')
| -rw-r--r-- | src/zenserver/compute/computeserver.cpp | 1 | ||||
| -rw-r--r-- | src/zenserver/config/config.cpp | 7 | ||||
| -rw-r--r-- | src/zenserver/config/config.h | 1 | ||||
| -rw-r--r-- | src/zenserver/hub/hub.cpp | 56 | ||||
| -rw-r--r-- | src/zenserver/hub/hub.h | 5 | ||||
| -rw-r--r-- | src/zenserver/hub/storageserverinstance.cpp | 30 | ||||
| -rw-r--r-- | src/zenserver/hub/storageserverinstance.h | 26 | ||||
| -rw-r--r-- | src/zenserver/hub/zenhubserver.cpp | 48 | ||||
| -rw-r--r-- | src/zenserver/hub/zenhubserver.h | 13 | ||||
| -rw-r--r-- | src/zenserver/proxy/zenproxyserver.cpp | 1 | ||||
| -rw-r--r-- | src/zenserver/storage/zenstorageserver.cpp | 1 | ||||
| -rw-r--r-- | src/zenserver/zenserver.cpp | 1 | ||||
| -rw-r--r-- | src/zenserver/zenserver.h | 2 |
13 files changed, 151 insertions, 41 deletions
diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp index 0d8550c5b..724ef9ad2 100644 --- a/src/zenserver/compute/computeserver.cpp +++ b/src/zenserver/compute/computeserver.cpp @@ -951,6 +951,7 @@ ZenComputeServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) Server.SetContentRoot(m_ServerOptions.ContentDir); Server.SetTestMode(m_ServerOptions.IsTest); Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing); const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); if (EffectiveBasePort == -1) diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp index 60ae93853..15f6f79f3 100644 --- a/src/zenserver/config/config.cpp +++ b/src/zenserver/config/config.cpp @@ -133,6 +133,7 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions // server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); + LuaOptions.AddOption("server.allowportprobing"sv, ServerOptions.AllowPortProbing, "allow-port-probing"sv); LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); @@ -223,8 +224,11 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi #endif options.add_options()("dedicated", - "Enable dedicated server mode", + "Enable dedicated server mode, disables '--allow-port-probing' and allocates more resources.", cxxopts::value<bool>(ServerOptions.IsDedicated)->default_value("false")); + options.add_options()("allow-port-probing", + "Allow searching for an available port, disabled if '--dedicated' is enabled.", + cxxopts::value<bool>(ServerOptions.AllowPortProbing)->default_value("true")); options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(ServerOptions.IsDebug)->default_value("false")); options.add_options()("clean", "Clean out all state at startup", @@ -691,6 +695,7 @@ ZenServerConfiguratorBase::Configure(int argc, char* argv[]) } m_ServerOptions.HttpConfig.IsDedicatedServer = m_ServerOptions.IsDedicated; + m_ServerOptions.HttpConfig.AllowPortProbing = !m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing; } void diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h index e481c7225..5078fe71a 100644 --- a/src/zenserver/config/config.h +++ b/src/zenserver/config/config.h @@ -61,6 +61,7 @@ struct ZenServerConfig int CoreLimit = 0; // If set, hardware concurrency queries are capped at this number int LieCpu = 0; bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + bool AllowPortProbing = true; // Automatically false if IsDedicated is true bool ShouldCrash = false; // Option for testing crash handling bool IsFirstRun = false; std::filesystem::path ConfigFile; // Path to Lua config file diff --git a/src/zenserver/hub/hub.cpp b/src/zenserver/hub/hub.cpp index c9720b32d..3c9f40eaa 100644 --- a/src/zenserver/hub/hub.cpp +++ b/src/zenserver/hub/hub.cpp @@ -22,6 +22,8 @@ ZEN_THIRD_PARTY_INCLUDES_END # include <zencore/workthreadpool.h> #endif +#include <numeric> + namespace zen { /////////////////////////////////////////////////////////////////////////// @@ -137,10 +139,10 @@ Hub::Hub(const Configuration& Config, m_HydrationTempPath = m_RunEnvironment.CreateChildDir("hydration_temp"); ZEN_INFO("using hydration temp path: '{}'", m_HydrationTempPath); - // This is necessary to ensure the hub assigns a distinct port range. - // We need to do this primarily because otherwise automated tests will - // fail as the test runner will create processes in the default range. - m_RunEnvironment.SetNextPortNumber(m_Config.BasePortNumber); + ZEN_ASSERT(uint64_t(Config.BasePortNumber) + Config.InstanceLimit <= std::numeric_limits<uint16_t>::max()); + + m_FreePorts.resize(Config.InstanceLimit); + std::iota(m_FreePorts.begin(), m_FreePorts.end(), Config.BasePortNumber); #if ZEN_PLATFORM_WINDOWS if (m_Config.UseJobObject) @@ -199,6 +201,15 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s bool IsNewInstance = false; { RwLock::ExclusiveLockScope _(m_Lock); + uint16_t AllocatedPort = 0; + auto RestoreAllocatedPort = MakeGuard([this, &AllocatedPort]() { + if (AllocatedPort != 0) + { + m_FreePorts.push_back(AllocatedPort); + AllocatedPort = 0; + } + }); + if (auto It = m_Instances.find(std::string(ModuleId)); It == m_Instances.end()) { std::string Reason; @@ -211,9 +222,18 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s return false; } - IsNewInstance = true; - auto NewInstance = - std::make_unique<StorageServerInstance>(m_RunEnvironment, ModuleId, m_FileHydrationPath, m_HydrationTempPath); + AllocatedPort = m_FreePorts.front(); + m_FreePorts.pop_front(); + + IsNewInstance = true; + auto NewInstance = std::make_unique<StorageServerInstance>( + m_RunEnvironment, + StorageServerInstance::Configuration{.BasePort = AllocatedPort, + .HydrationTempPath = m_HydrationTempPath, + .FileHydrationPath = m_FileHydrationPath, + .HttpThreadCount = m_Config.InstanceHttpThreadCount, + .CoreLimit = m_Config.InstanceCoreLimit}, + ModuleId); #if ZEN_PLATFORM_WINDOWS if (m_JobObject.IsValid()) { @@ -222,6 +242,7 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s #endif Instance = NewInstance.get(); m_Instances.emplace(std::string(ModuleId), std::move(NewInstance)); + AllocatedPort = 0; ZEN_INFO("Created new storage server instance for module '{}'", ModuleId); } @@ -258,7 +279,13 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s { // Clean up RwLock::ExclusiveLockScope _(m_Lock); - m_Instances.erase(std::string(ModuleId)); + if (auto It = m_Instances.find(std::string(ModuleId)); It != m_Instances.end()) + { + ZEN_ASSERT(It->second != nullptr); + uint16_t BasePort = It->second->GetBasePort(); + m_FreePorts.push_back(BasePort); + m_Instances.erase(It); + } } return false; } @@ -337,6 +364,7 @@ Hub::Deprovision(const std::string& ModuleId, std::string& OutReason) auto _ = MakeGuard([&] { RwLock::ExclusiveLockScope _(m_Lock); m_DeprovisioningModules.erase(ModuleId); + m_FreePorts.push_back(BasePort); }); Instance->Deprovision(); @@ -413,7 +441,17 @@ Hub::CanProvisionInstance(std::string_view ModuleId, std::string& OutReason) if (gsl::narrow_cast<int>(m_Instances.size()) >= m_Config.InstanceLimit) { - OutReason = fmt::format("instance limit exceeded ({})", m_Config.InstanceLimit); + OutReason = fmt::format("instance limit ({}) exceeded", m_Config.InstanceLimit); + + return false; + } + + // Since deprovisioning happens outside the lock and we don't add the port back until the instance is full shut down we might be under + // the instance limit but all ports may be in use + if (m_FreePorts.empty()) + { + OutReason = fmt::format("no free ports available, deprovisioning of instances might be in flight ({})", + m_Config.InstanceLimit - m_Instances.size()); return false; } diff --git a/src/zenserver/hub/hub.h b/src/zenserver/hub/hub.h index 8b61f988d..8a84a558b 100644 --- a/src/zenserver/hub/hub.h +++ b/src/zenserver/hub/hub.h @@ -7,6 +7,7 @@ #include <zencore/system.h> #include <zenutil/zenserverprocess.h> +#include <deque> #include <filesystem> #include <functional> #include <memory> @@ -43,6 +44,9 @@ public: uint16_t BasePortNumber = 21000; int InstanceLimit = 1000; + + uint32_t InstanceHttpThreadCount = 0; // Deduce from core count + int InstanceCoreLimit = 0; // Use hardware core count }; typedef std::function<void(std::string_view ModuleId, const HubProvisionedInstanceInfo& Info)> ProvisionModuleCallbackFunc; @@ -121,6 +125,7 @@ private: ResourceMetrics m_ResourceLimits; SystemMetrics m_HostMetrics; int m_MaxInstanceCount = 0; + std::deque<uint16_t> m_FreePorts; void UpdateStats(); void UpdateCapacityMetrics(); diff --git a/src/zenserver/hub/storageserverinstance.cpp b/src/zenserver/hub/storageserverinstance.cpp index f24379715..68de5e274 100644 --- a/src/zenserver/hub/storageserverinstance.cpp +++ b/src/zenserver/hub/storageserverinstance.cpp @@ -11,16 +11,13 @@ namespace zen { -StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment, - std::string_view ModuleId, - std::filesystem::path FileHydrationPath, - std::filesystem::path HydrationTempPath) -: m_ModuleId(ModuleId) +StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment, const Configuration& Config, std::string_view ModuleId) +: m_Config(Config) +, m_ModuleId(ModuleId) , m_ServerInstance(RunEnvironment, ZenServerInstance::ServerMode::kStorageServer) -, m_HydrationPath(FileHydrationPath) { m_BaseDir = RunEnvironment.CreateChildDir(ModuleId); - m_TempDir = HydrationTempPath / ModuleId; + m_TempDir = Config.HydrationTempPath / ModuleId; } StorageServerInstance::~StorageServerInstance() @@ -37,9 +34,20 @@ StorageServerInstance::SpawnServerProcess() #if ZEN_PLATFORM_WINDOWS m_ServerInstance.SetJobObject(m_JobObject); #endif - const uint16_t BasePort = m_ServerInstance.SpawnServerAndWaitUntilReady(); - ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, BasePort); + ExtendableStringBuilder<256> AdditionalOptions; + AdditionalOptions << "--allow-port-probing=false"; + if (m_Config.HttpThreadCount != 0) + { + AdditionalOptions << " --http-threads=" << m_Config.HttpThreadCount; + } + if (m_Config.CoreLimit != 0) + { + AdditionalOptions << " --corelimit=" << m_Config.CoreLimit; + } + + m_ServerInstance.SpawnServerAndWaitUntilReady(m_Config.BasePort, AdditionalOptions.ToView()); + ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, m_Config.BasePort); m_ServerInstance.EnableShutdownOnDestroy(); } @@ -178,7 +186,7 @@ StorageServerInstance::Hydrate() HydrationConfig Config{.ServerStateDir = m_BaseDir, .TempDir = m_TempDir, .ModuleId = m_ModuleId, - .TargetSpecification = WideToUtf8(m_HydrationPath.native())}; + .TargetSpecification = WideToUtf8(m_Config.FileHydrationPath.native())}; std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator(); @@ -192,7 +200,7 @@ StorageServerInstance::Dehydrate() HydrationConfig Config{.ServerStateDir = m_BaseDir, .TempDir = m_TempDir, .ModuleId = m_ModuleId, - .TargetSpecification = WideToUtf8(m_HydrationPath.native())}; + .TargetSpecification = WideToUtf8(m_Config.FileHydrationPath.native())}; std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator(); diff --git a/src/zenserver/hub/storageserverinstance.h b/src/zenserver/hub/storageserverinstance.h index a2f3d25d7..23196d835 100644 --- a/src/zenserver/hub/storageserverinstance.h +++ b/src/zenserver/hub/storageserverinstance.h @@ -21,10 +21,16 @@ namespace zen { class StorageServerInstance { public: - StorageServerInstance(ZenServerEnvironment& RunEnvironment, - std::string_view ModuleId, - std::filesystem::path FileHydrationPath, - std::filesystem::path HydrationTempPath); + struct Configuration + { + uint16_t BasePort; + std::filesystem::path HydrationTempPath; + std::filesystem::path FileHydrationPath; + uint32_t HttpThreadCount = 0; // Deduce from core count + int CoreLimit = 0; // Use hardware core count + }; + + StorageServerInstance(ZenServerEnvironment& RunEnvironment, const Configuration& Config, std::string_view ModuleId); ~StorageServerInstance(); void Provision(); @@ -45,15 +51,17 @@ public: #endif private: - void WakeLocked(); - RwLock m_Lock; - std::string m_ModuleId; + void WakeLocked(); + RwLock m_Lock; + const Configuration m_Config; + std::string m_ModuleId; + ZenServerInstance m_ServerInstance; + std::atomic<bool> m_IsProvisioned{false}; std::atomic<bool> m_IsHibernated{false}; - ZenServerInstance m_ServerInstance; std::filesystem::path m_BaseDir; + std::filesystem::path m_TempDir; - std::filesystem::path m_HydrationPath; ResourceMetrics m_ResourceMetrics; #if ZEN_PLATFORM_WINDOWS JobObject* m_JobObject = nullptr; diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp index b0ae0a8b1..313be977c 100644 --- a/src/zenserver/hub/zenhubserver.cpp +++ b/src/zenserver/hub/zenhubserver.cpp @@ -26,6 +26,15 @@ namespace zen { void ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options) { + const char* DefaultInstanceHttp = "asio"; + +#if ZEN_WITH_HTTPSYS + if (!windows::IsRunningOnWine()) + { + DefaultInstanceHttp = "httpsys"; + } +#endif + Options.add_option("hub", "", "upstream-notification-endpoint", @@ -60,6 +69,31 @@ ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options) "Maximum number of provisioned instances for this hub", cxxopts::value<int>(m_ServerOptions.HubInstanceLimit)->default_value("1000"), ""); + + Options.add_option("hub", + "", + "hub-instance-http", + "Select HTTP server implementation for provisioned instances (asio|" +#if ZEN_WITH_HTTPSYS + "httpsys|" +#endif + "null)", + cxxopts::value<std::string>(m_ServerOptions.HubInstanceHttpClass)->default_value(DefaultInstanceHttp), + "<instance http class>"); + + Options.add_option("hub", + "", + "hub-instance-http-threads", + "Number of http server connection threads for provisioned instances", + cxxopts::value<unsigned int>(m_ServerOptions.HubInstanceHttpThreadCount), + "<instance http threads>"); + Options.add_option("hub", + "", + "hub-instance-corelimit", + "Limit concurrency of provisioned instances", + cxxopts::value(m_ServerOptions.HubInstanceCoreLimit), + "<instance core limit>"); + #if ZEN_PLATFORM_WINDOWS Options.add_option("hub", "", @@ -231,10 +265,15 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig) ZEN_INFO("instantiating Hub"); m_Hub = std::make_unique<Hub>( - Hub::Configuration{.UseJobObject = ServerConfig.HubUseJobObject, - .BasePortNumber = ServerConfig.HubBasePortNumber, - .InstanceLimit = ServerConfig.HubInstanceLimit}, - ZenServerEnvironment(ZenServerEnvironment::Hub, ServerConfig.DataDir / "hub", ServerConfig.DataDir / "servers"), + Hub::Configuration{.UseJobObject = ServerConfig.HubUseJobObject, + .BasePortNumber = ServerConfig.HubBasePortNumber, + .InstanceLimit = ServerConfig.HubInstanceLimit, + .InstanceHttpThreadCount = ServerConfig.HubInstanceHttpThreadCount, + .InstanceCoreLimit = ServerConfig.HubInstanceCoreLimit}, + ZenServerEnvironment(ZenServerEnvironment::Hub, + ServerConfig.DataDir / "hub", + ServerConfig.DataDir / "servers", + ServerConfig.HubInstanceHttpClass), m_ConsulClient ? [this, HubInstanceId = fmt::format("zen-hub-{}", ServerConfig.InstanceId)]( std::string_view ModuleId, const HubProvisionedInstanceInfo& Info) { OnProvisioned(HubInstanceId, ModuleId, Info); } @@ -405,6 +444,7 @@ ZenHubServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) Server.SetContentRoot(m_ServerOptions.ContentDir); Server.SetTestMode(m_ServerOptions.IsTest); Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing); const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); if (EffectiveBasePort == -1) diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h index f6a3eb1bc..1036598bb 100644 --- a/src/zenserver/hub/zenhubserver.h +++ b/src/zenserver/hub/zenhubserver.h @@ -24,9 +24,12 @@ struct ZenHubServerConfig : public ZenServerConfig std::string UpstreamNotificationEndpoint; std::string InstanceId; // For use in notifications std::string ConsulEndpoint; // If set, enables Consul service registration - uint16_t HubBasePortNumber = 21000; - int HubInstanceLimit = 1000; - bool HubUseJobObject = true; + uint16_t HubBasePortNumber = 21000; + int HubInstanceLimit = 1000; + bool HubUseJobObject = true; + std::string HubInstanceHttpClass = "asio"; + uint32_t HubInstanceHttpThreadCount = 0; // Deduce from core count + int HubInstanceCoreLimit = 0; // Use hardware core count }; class Hub; @@ -79,8 +82,6 @@ public: void Run(); void Cleanup(); - void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } - void SetTestMode(bool State) { m_TestMode = State; } void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } @@ -88,8 +89,6 @@ private: void OnProvisioned(std::string_view HubInstanceId, std::string_view ModuleId, const HubProvisionedInstanceInfo& Info); void OnDeprovisioned(std::string_view HubInstanceId, std::string_view ModuleId, const HubProvisionedInstanceInfo& Info); - bool m_IsDedicatedMode = false; - bool m_TestMode = false; std::filesystem::path m_DataRoot; std::filesystem::path m_ContentRoot; bool m_DebugOptionForcedCrash = false; diff --git a/src/zenserver/proxy/zenproxyserver.cpp b/src/zenserver/proxy/zenproxyserver.cpp index c768e940a..cf84c159a 100644 --- a/src/zenserver/proxy/zenproxyserver.cpp +++ b/src/zenserver/proxy/zenproxyserver.cpp @@ -454,6 +454,7 @@ ZenProxyServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) Server.SetContentRoot(m_ServerOptions.ContentDir); Server.SetTestMode(m_ServerOptions.IsTest); Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing); const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); if (EffectiveBasePort == -1) diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index bba5e0a61..f5ede5692 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -975,6 +975,7 @@ ZenStorageServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) Server.SetContentRoot(m_ServerOptions.ContentDir); Server.SetTestMode(m_ServerOptions.IsTest); Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing); int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); if (EffectiveBasePort == -1) diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 6760e0372..1cd8ed846 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -517,6 +517,7 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig) }); // clang-format on Settings.emplace_back("IsDedicated"sv, fmt::to_string(ServerConfig.IsDedicated)); + Settings.emplace_back("AllowPortProbing"sv, fmt::to_string(ServerConfig.AllowPortProbing)); Settings.emplace_back("ShouldCrash"sv, fmt::to_string(ServerConfig.ShouldCrash)); size_t MaxWidth = 0; diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 830f36e54..d6bf4454f 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -48,6 +48,7 @@ public: void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } + void SetAllowPortProbing(bool State) { m_AllowPortProbing = State; } void SetServerMode(std::string_view Mode) { m_ServerMode = Mode; } void SetTestMode(bool State) { m_TestMode = State; } @@ -66,6 +67,7 @@ protected: bool m_IsPowerCycle = false; bool m_IsDedicatedMode = false; + bool m_AllowPortProbing = true; bool m_TestMode = false; bool m_NoNetwork = false; bool m_DebugOptionForcedCrash = false; |