diff options
23 files changed, 180 insertions, 46 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 36a211d47..a2a54208e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ - Feature: Added `xmake kill` task to terminate all running zenserver instances - Feature: Support `ZEN_MALLOC` environment variable for default allocator selection; default allocator switched to rpmalloc - Feature: Added Dockerfile and extended GHA workflows to produce zenserver Docker images, primarily intended for compute nodes +- Feature: Added `--allow-port-probing` option to control whether zenserver searches for a free port on startup (default: true, automatically false when --dedicated is set) +- Feature: Added new hub options for controlling provisioned storage server instances: + - `--hub-instance-http` — HTTP server implementation for instances (asio/httpsys) + - `--hub-instance-http-threads` — Number of HTTP connection threads per instance + - `--hub-instance-corelimit` — Limit CPU concurrency per instance - Improvement: Added support for CoW block-cloning (used by build download) on Linux (tested with: btrfs/ XFS) - Improvement: Added full-file CoW copying on macOS (APFS) - Improvement: Updated asio to 1.38.0 @@ -19,6 +24,7 @@ - `--msan=[y|n]` Enable MemorySanitizer (Linux only, requires all deps instrumented) - `--asan=[y|n]` Enable AddressSanitizer (disables mimalloc and sentry) - `--tsan=[y|n]` Enable ThreadSanitizer (Linux/Mac only) +- Improvement: Hub now manages a deterministic port pool for provisioned instances allowing reuse of unused ports - Bugfix: Fixed sentry-native build to allow LTO on Windows - Bugfix: Minor test stability fixes (flaky hash collisions, per-thread RNG seeds) diff --git a/scripts/test_scripts/metadatas/PS5Client.json b/scripts/test_scripts/metadatas/PS5Client.json new file mode 100644 index 000000000..55ca338a2 --- /dev/null +++ b/scripts/test_scripts/metadatas/PS5Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 PS4Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "PS5", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 4d98e9650..e5cfbcbae 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -1033,7 +1033,7 @@ CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig ZEN_INFO("using asio HTTP server implementation") return CreateHttpAsioServer(AsioConfig { .ThreadCount = Config.ThreadCount, .ForceLoopback = Config.ForceLoopback, .IsDedicatedServer = Config.IsDedicatedServer, - .NoNetwork = Config.NoNetwork, .UnixSocketPath = PathToUtf8(Config.UnixSocketPath), + .NoNetwork = Config.NoNetwork, .AllowPortProbing = Config.AllowPortProbing, .UnixSocketPath = PathToUtf8(Config.UnixSocketPath), #if ZEN_USE_OPENSSL .HttpsPort = Config.HttpsPort, .CertFile = Config.CertFile, .KeyFile = Config.KeyFile, #endif @@ -1049,6 +1049,7 @@ CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled, .IsDedicatedServer = Config.IsDedicatedServer, .ForceLoopback = Config.ForceLoopback, + .AllowPortProbing = Config.AllowPortProbing, .HttpsPort = Config.HttpSys.HttpsPort, .CertThumbprint = Config.HttpSys.CertThumbprint, .CertStoreName = Config.HttpSys.CertStoreName, diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 2a8b2ca94..a7d7f4d9c 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -325,6 +325,7 @@ struct HttpServerPluginConfig struct HttpServerConfig { bool IsDedicatedServer = false; // Should be set to true for shared servers + bool AllowPortProbing = true; // IsDedicatedServer forces this to false std::string ServerClass; // Choice of HTTP server implementation std::vector<HttpServerPluginConfig> PluginConfigs; bool ForceLoopback = false; diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 9f4875eaf..a2cae8762 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -2182,7 +2182,7 @@ HttpAsioServerImpl::Start(uint16_t Port, const AsioConfig& Config) if (!Config.NoNetwork) { m_Acceptor.reset( - new asio_http::HttpAcceptor(*this, m_IoService, Port, Config.ForceLoopback, /*AllowPortProbing */ !Config.IsDedicatedServer)); + new asio_http::HttpAcceptor(*this, m_IoService, Port, Config.ForceLoopback, /*AllowPortProbing */ Config.AllowPortProbing)); if (!m_Acceptor->IsValid()) { @@ -2224,7 +2224,7 @@ HttpAsioServerImpl::Start(uint16_t Port, const AsioConfig& Config) *m_SslContext, gsl::narrow<uint16_t>(Config.HttpsPort), Config.ForceLoopback, - /*AllowPortProbing*/ !Config.IsDedicatedServer)); + /*AllowPortProbing*/ Config.AllowPortProbing)); if (m_HttpsAcceptor->IsValid()) { diff --git a/src/zenhttp/servers/httpasio.h b/src/zenhttp/servers/httpasio.h index 21d10170e..154c87d9c 100644 --- a/src/zenhttp/servers/httpasio.h +++ b/src/zenhttp/servers/httpasio.h @@ -12,6 +12,7 @@ struct AsioConfig bool ForceLoopback = false; bool IsDedicatedServer = false; bool NoNetwork = false; + bool AllowPortProbing = true; std::string UnixSocketPath; #if ZEN_USE_OPENSSL int HttpsPort = 0; // 0 = auto-assign; set CertFile/KeyFile to enable HTTPS diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index f8fb1c9be..9fe9a2254 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -1153,7 +1153,7 @@ HttpSysServer::RegisterHttpUrls(int BasePort) m_BaseUris.clear(); - const bool AllowPortProbing = !m_InitialConfig.IsDedicatedServer; + const bool AllowPortProbing = m_InitialConfig.AllowPortProbing; const bool AllowLocalOnly = !m_InitialConfig.IsDedicatedServer; int EffectivePort = BasePort; diff --git a/src/zenhttp/servers/httpsys.h b/src/zenhttp/servers/httpsys.h index ca465ad00..88ec13834 100644 --- a/src/zenhttp/servers/httpsys.h +++ b/src/zenhttp/servers/httpsys.h @@ -22,6 +22,7 @@ struct HttpSysConfig bool IsRequestLoggingEnabled = false; bool IsDedicatedServer = false; bool ForceLoopback = false; + bool AllowPortProbing = true; int HttpsPort = 0; // 0 = HTTPS disabled std::string CertThumbprint; // Hex SHA-1 (40 chars) for auto SSL binding std::string CertStoreName = "MY"; // Windows certificate store name diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp index 9c1669256..958a0b050 100644 --- a/src/zenserver-test/hub-tests.cpp +++ b/src/zenserver-test/hub-tests.cpp @@ -47,7 +47,7 @@ TEST_CASE("hub.lifecycle.children") { ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer); - const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(); + const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady("--hub-instance-corelimit=2 --hub-instance-http-threads=6"); REQUIRE(PortNumber != 0); SUBCASE("spawn") diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp index 0d8550c5b..724ef9ad2 100644 --- a/src/zenserver/compute/computeserver.cpp +++ b/src/zenserver/compute/computeserver.cpp @@ -951,6 +951,7 @@ ZenComputeServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) Server.SetContentRoot(m_ServerOptions.ContentDir); Server.SetTestMode(m_ServerOptions.IsTest); Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing); const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); if (EffectiveBasePort == -1) diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp index 60ae93853..15f6f79f3 100644 --- a/src/zenserver/config/config.cpp +++ b/src/zenserver/config/config.cpp @@ -133,6 +133,7 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions // server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); + LuaOptions.AddOption("server.allowportprobing"sv, ServerOptions.AllowPortProbing, "allow-port-probing"sv); LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); @@ -223,8 +224,11 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi #endif options.add_options()("dedicated", - "Enable dedicated server mode", + "Enable dedicated server mode, disables '--allow-port-probing' and allocates more resources.", cxxopts::value<bool>(ServerOptions.IsDedicated)->default_value("false")); + options.add_options()("allow-port-probing", + "Allow searching for an available port, disabled if '--dedicated' is enabled.", + cxxopts::value<bool>(ServerOptions.AllowPortProbing)->default_value("true")); options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(ServerOptions.IsDebug)->default_value("false")); options.add_options()("clean", "Clean out all state at startup", @@ -691,6 +695,7 @@ ZenServerConfiguratorBase::Configure(int argc, char* argv[]) } m_ServerOptions.HttpConfig.IsDedicatedServer = m_ServerOptions.IsDedicated; + m_ServerOptions.HttpConfig.AllowPortProbing = !m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing; } void diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h index e481c7225..5078fe71a 100644 --- a/src/zenserver/config/config.h +++ b/src/zenserver/config/config.h @@ -61,6 +61,7 @@ struct ZenServerConfig int CoreLimit = 0; // If set, hardware concurrency queries are capped at this number int LieCpu = 0; bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + bool AllowPortProbing = true; // Automatically false if IsDedicated is true bool ShouldCrash = false; // Option for testing crash handling bool IsFirstRun = false; std::filesystem::path ConfigFile; // Path to Lua config file diff --git a/src/zenserver/hub/hub.cpp b/src/zenserver/hub/hub.cpp index c9720b32d..3c9f40eaa 100644 --- a/src/zenserver/hub/hub.cpp +++ b/src/zenserver/hub/hub.cpp @@ -22,6 +22,8 @@ ZEN_THIRD_PARTY_INCLUDES_END # include <zencore/workthreadpool.h> #endif +#include <numeric> + namespace zen { /////////////////////////////////////////////////////////////////////////// @@ -137,10 +139,10 @@ Hub::Hub(const Configuration& Config, m_HydrationTempPath = m_RunEnvironment.CreateChildDir("hydration_temp"); ZEN_INFO("using hydration temp path: '{}'", m_HydrationTempPath); - // This is necessary to ensure the hub assigns a distinct port range. - // We need to do this primarily because otherwise automated tests will - // fail as the test runner will create processes in the default range. - m_RunEnvironment.SetNextPortNumber(m_Config.BasePortNumber); + ZEN_ASSERT(uint64_t(Config.BasePortNumber) + Config.InstanceLimit <= std::numeric_limits<uint16_t>::max()); + + m_FreePorts.resize(Config.InstanceLimit); + std::iota(m_FreePorts.begin(), m_FreePorts.end(), Config.BasePortNumber); #if ZEN_PLATFORM_WINDOWS if (m_Config.UseJobObject) @@ -199,6 +201,15 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s bool IsNewInstance = false; { RwLock::ExclusiveLockScope _(m_Lock); + uint16_t AllocatedPort = 0; + auto RestoreAllocatedPort = MakeGuard([this, &AllocatedPort]() { + if (AllocatedPort != 0) + { + m_FreePorts.push_back(AllocatedPort); + AllocatedPort = 0; + } + }); + if (auto It = m_Instances.find(std::string(ModuleId)); It == m_Instances.end()) { std::string Reason; @@ -211,9 +222,18 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s return false; } - IsNewInstance = true; - auto NewInstance = - std::make_unique<StorageServerInstance>(m_RunEnvironment, ModuleId, m_FileHydrationPath, m_HydrationTempPath); + AllocatedPort = m_FreePorts.front(); + m_FreePorts.pop_front(); + + IsNewInstance = true; + auto NewInstance = std::make_unique<StorageServerInstance>( + m_RunEnvironment, + StorageServerInstance::Configuration{.BasePort = AllocatedPort, + .HydrationTempPath = m_HydrationTempPath, + .FileHydrationPath = m_FileHydrationPath, + .HttpThreadCount = m_Config.InstanceHttpThreadCount, + .CoreLimit = m_Config.InstanceCoreLimit}, + ModuleId); #if ZEN_PLATFORM_WINDOWS if (m_JobObject.IsValid()) { @@ -222,6 +242,7 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s #endif Instance = NewInstance.get(); m_Instances.emplace(std::string(ModuleId), std::move(NewInstance)); + AllocatedPort = 0; ZEN_INFO("Created new storage server instance for module '{}'", ModuleId); } @@ -258,7 +279,13 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s { // Clean up RwLock::ExclusiveLockScope _(m_Lock); - m_Instances.erase(std::string(ModuleId)); + if (auto It = m_Instances.find(std::string(ModuleId)); It != m_Instances.end()) + { + ZEN_ASSERT(It->second != nullptr); + uint16_t BasePort = It->second->GetBasePort(); + m_FreePorts.push_back(BasePort); + m_Instances.erase(It); + } } return false; } @@ -337,6 +364,7 @@ Hub::Deprovision(const std::string& ModuleId, std::string& OutReason) auto _ = MakeGuard([&] { RwLock::ExclusiveLockScope _(m_Lock); m_DeprovisioningModules.erase(ModuleId); + m_FreePorts.push_back(BasePort); }); Instance->Deprovision(); @@ -413,7 +441,17 @@ Hub::CanProvisionInstance(std::string_view ModuleId, std::string& OutReason) if (gsl::narrow_cast<int>(m_Instances.size()) >= m_Config.InstanceLimit) { - OutReason = fmt::format("instance limit exceeded ({})", m_Config.InstanceLimit); + OutReason = fmt::format("instance limit ({}) exceeded", m_Config.InstanceLimit); + + return false; + } + + // Since deprovisioning happens outside the lock and we don't add the port back until the instance is full shut down we might be under + // the instance limit but all ports may be in use + if (m_FreePorts.empty()) + { + OutReason = fmt::format("no free ports available, deprovisioning of instances might be in flight ({})", + m_Config.InstanceLimit - m_Instances.size()); return false; } diff --git a/src/zenserver/hub/hub.h b/src/zenserver/hub/hub.h index 8b61f988d..8a84a558b 100644 --- a/src/zenserver/hub/hub.h +++ b/src/zenserver/hub/hub.h @@ -7,6 +7,7 @@ #include <zencore/system.h> #include <zenutil/zenserverprocess.h> +#include <deque> #include <filesystem> #include <functional> #include <memory> @@ -43,6 +44,9 @@ public: uint16_t BasePortNumber = 21000; int InstanceLimit = 1000; + + uint32_t InstanceHttpThreadCount = 0; // Deduce from core count + int InstanceCoreLimit = 0; // Use hardware core count }; typedef std::function<void(std::string_view ModuleId, const HubProvisionedInstanceInfo& Info)> ProvisionModuleCallbackFunc; @@ -121,6 +125,7 @@ private: ResourceMetrics m_ResourceLimits; SystemMetrics m_HostMetrics; int m_MaxInstanceCount = 0; + std::deque<uint16_t> m_FreePorts; void UpdateStats(); void UpdateCapacityMetrics(); diff --git a/src/zenserver/hub/storageserverinstance.cpp b/src/zenserver/hub/storageserverinstance.cpp index f24379715..68de5e274 100644 --- a/src/zenserver/hub/storageserverinstance.cpp +++ b/src/zenserver/hub/storageserverinstance.cpp @@ -11,16 +11,13 @@ namespace zen { -StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment, - std::string_view ModuleId, - std::filesystem::path FileHydrationPath, - std::filesystem::path HydrationTempPath) -: m_ModuleId(ModuleId) +StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment, const Configuration& Config, std::string_view ModuleId) +: m_Config(Config) +, m_ModuleId(ModuleId) , m_ServerInstance(RunEnvironment, ZenServerInstance::ServerMode::kStorageServer) -, m_HydrationPath(FileHydrationPath) { m_BaseDir = RunEnvironment.CreateChildDir(ModuleId); - m_TempDir = HydrationTempPath / ModuleId; + m_TempDir = Config.HydrationTempPath / ModuleId; } StorageServerInstance::~StorageServerInstance() @@ -37,9 +34,20 @@ StorageServerInstance::SpawnServerProcess() #if ZEN_PLATFORM_WINDOWS m_ServerInstance.SetJobObject(m_JobObject); #endif - const uint16_t BasePort = m_ServerInstance.SpawnServerAndWaitUntilReady(); - ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, BasePort); + ExtendableStringBuilder<256> AdditionalOptions; + AdditionalOptions << "--allow-port-probing=false"; + if (m_Config.HttpThreadCount != 0) + { + AdditionalOptions << " --http-threads=" << m_Config.HttpThreadCount; + } + if (m_Config.CoreLimit != 0) + { + AdditionalOptions << " --corelimit=" << m_Config.CoreLimit; + } + + m_ServerInstance.SpawnServerAndWaitUntilReady(m_Config.BasePort, AdditionalOptions.ToView()); + ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, m_Config.BasePort); m_ServerInstance.EnableShutdownOnDestroy(); } @@ -178,7 +186,7 @@ StorageServerInstance::Hydrate() HydrationConfig Config{.ServerStateDir = m_BaseDir, .TempDir = m_TempDir, .ModuleId = m_ModuleId, - .TargetSpecification = WideToUtf8(m_HydrationPath.native())}; + .TargetSpecification = WideToUtf8(m_Config.FileHydrationPath.native())}; std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator(); @@ -192,7 +200,7 @@ StorageServerInstance::Dehydrate() HydrationConfig Config{.ServerStateDir = m_BaseDir, .TempDir = m_TempDir, .ModuleId = m_ModuleId, - .TargetSpecification = WideToUtf8(m_HydrationPath.native())}; + .TargetSpecification = WideToUtf8(m_Config.FileHydrationPath.native())}; std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator(); diff --git a/src/zenserver/hub/storageserverinstance.h b/src/zenserver/hub/storageserverinstance.h index a2f3d25d7..23196d835 100644 --- a/src/zenserver/hub/storageserverinstance.h +++ b/src/zenserver/hub/storageserverinstance.h @@ -21,10 +21,16 @@ namespace zen { class StorageServerInstance { public: - StorageServerInstance(ZenServerEnvironment& RunEnvironment, - std::string_view ModuleId, - std::filesystem::path FileHydrationPath, - std::filesystem::path HydrationTempPath); + struct Configuration + { + uint16_t BasePort; + std::filesystem::path HydrationTempPath; + std::filesystem::path FileHydrationPath; + uint32_t HttpThreadCount = 0; // Deduce from core count + int CoreLimit = 0; // Use hardware core count + }; + + StorageServerInstance(ZenServerEnvironment& RunEnvironment, const Configuration& Config, std::string_view ModuleId); ~StorageServerInstance(); void Provision(); @@ -45,15 +51,17 @@ public: #endif private: - void WakeLocked(); - RwLock m_Lock; - std::string m_ModuleId; + void WakeLocked(); + RwLock m_Lock; + const Configuration m_Config; + std::string m_ModuleId; + ZenServerInstance m_ServerInstance; + std::atomic<bool> m_IsProvisioned{false}; std::atomic<bool> m_IsHibernated{false}; - ZenServerInstance m_ServerInstance; std::filesystem::path m_BaseDir; + std::filesystem::path m_TempDir; - std::filesystem::path m_HydrationPath; ResourceMetrics m_ResourceMetrics; #if ZEN_PLATFORM_WINDOWS JobObject* m_JobObject = nullptr; diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp index b0ae0a8b1..313be977c 100644 --- a/src/zenserver/hub/zenhubserver.cpp +++ b/src/zenserver/hub/zenhubserver.cpp @@ -26,6 +26,15 @@ namespace zen { void ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options) { + const char* DefaultInstanceHttp = "asio"; + +#if ZEN_WITH_HTTPSYS + if (!windows::IsRunningOnWine()) + { + DefaultInstanceHttp = "httpsys"; + } +#endif + Options.add_option("hub", "", "upstream-notification-endpoint", @@ -60,6 +69,31 @@ ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options) "Maximum number of provisioned instances for this hub", cxxopts::value<int>(m_ServerOptions.HubInstanceLimit)->default_value("1000"), ""); + + Options.add_option("hub", + "", + "hub-instance-http", + "Select HTTP server implementation for provisioned instances (asio|" +#if ZEN_WITH_HTTPSYS + "httpsys|" +#endif + "null)", + cxxopts::value<std::string>(m_ServerOptions.HubInstanceHttpClass)->default_value(DefaultInstanceHttp), + "<instance http class>"); + + Options.add_option("hub", + "", + "hub-instance-http-threads", + "Number of http server connection threads for provisioned instances", + cxxopts::value<unsigned int>(m_ServerOptions.HubInstanceHttpThreadCount), + "<instance http threads>"); + Options.add_option("hub", + "", + "hub-instance-corelimit", + "Limit concurrency of provisioned instances", + cxxopts::value(m_ServerOptions.HubInstanceCoreLimit), + "<instance core limit>"); + #if ZEN_PLATFORM_WINDOWS Options.add_option("hub", "", @@ -231,10 +265,15 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig) ZEN_INFO("instantiating Hub"); m_Hub = std::make_unique<Hub>( - Hub::Configuration{.UseJobObject = ServerConfig.HubUseJobObject, - .BasePortNumber = ServerConfig.HubBasePortNumber, - .InstanceLimit = ServerConfig.HubInstanceLimit}, - ZenServerEnvironment(ZenServerEnvironment::Hub, ServerConfig.DataDir / "hub", ServerConfig.DataDir / "servers"), + Hub::Configuration{.UseJobObject = ServerConfig.HubUseJobObject, + .BasePortNumber = ServerConfig.HubBasePortNumber, + .InstanceLimit = ServerConfig.HubInstanceLimit, + .InstanceHttpThreadCount = ServerConfig.HubInstanceHttpThreadCount, + .InstanceCoreLimit = ServerConfig.HubInstanceCoreLimit}, + ZenServerEnvironment(ZenServerEnvironment::Hub, + ServerConfig.DataDir / "hub", + ServerConfig.DataDir / "servers", + ServerConfig.HubInstanceHttpClass), m_ConsulClient ? [this, HubInstanceId = fmt::format("zen-hub-{}", ServerConfig.InstanceId)]( std::string_view ModuleId, const HubProvisionedInstanceInfo& Info) { OnProvisioned(HubInstanceId, ModuleId, Info); } @@ -405,6 +444,7 @@ ZenHubServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) Server.SetContentRoot(m_ServerOptions.ContentDir); Server.SetTestMode(m_ServerOptions.IsTest); Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing); const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); if (EffectiveBasePort == -1) diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h index f6a3eb1bc..1036598bb 100644 --- a/src/zenserver/hub/zenhubserver.h +++ b/src/zenserver/hub/zenhubserver.h @@ -24,9 +24,12 @@ struct ZenHubServerConfig : public ZenServerConfig std::string UpstreamNotificationEndpoint; std::string InstanceId; // For use in notifications std::string ConsulEndpoint; // If set, enables Consul service registration - uint16_t HubBasePortNumber = 21000; - int HubInstanceLimit = 1000; - bool HubUseJobObject = true; + uint16_t HubBasePortNumber = 21000; + int HubInstanceLimit = 1000; + bool HubUseJobObject = true; + std::string HubInstanceHttpClass = "asio"; + uint32_t HubInstanceHttpThreadCount = 0; // Deduce from core count + int HubInstanceCoreLimit = 0; // Use hardware core count }; class Hub; @@ -79,8 +82,6 @@ public: void Run(); void Cleanup(); - void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } - void SetTestMode(bool State) { m_TestMode = State; } void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } @@ -88,8 +89,6 @@ private: void OnProvisioned(std::string_view HubInstanceId, std::string_view ModuleId, const HubProvisionedInstanceInfo& Info); void OnDeprovisioned(std::string_view HubInstanceId, std::string_view ModuleId, const HubProvisionedInstanceInfo& Info); - bool m_IsDedicatedMode = false; - bool m_TestMode = false; std::filesystem::path m_DataRoot; std::filesystem::path m_ContentRoot; bool m_DebugOptionForcedCrash = false; diff --git a/src/zenserver/proxy/zenproxyserver.cpp b/src/zenserver/proxy/zenproxyserver.cpp index c768e940a..cf84c159a 100644 --- a/src/zenserver/proxy/zenproxyserver.cpp +++ b/src/zenserver/proxy/zenproxyserver.cpp @@ -454,6 +454,7 @@ ZenProxyServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) Server.SetContentRoot(m_ServerOptions.ContentDir); Server.SetTestMode(m_ServerOptions.IsTest); Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing); const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); if (EffectiveBasePort == -1) diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index bba5e0a61..f5ede5692 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -975,6 +975,7 @@ ZenStorageServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) Server.SetContentRoot(m_ServerOptions.ContentDir); Server.SetTestMode(m_ServerOptions.IsTest); Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing); int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); if (EffectiveBasePort == -1) diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 6760e0372..1cd8ed846 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -517,6 +517,7 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig) }); // clang-format on Settings.emplace_back("IsDedicated"sv, fmt::to_string(ServerConfig.IsDedicated)); + Settings.emplace_back("AllowPortProbing"sv, fmt::to_string(ServerConfig.AllowPortProbing)); Settings.emplace_back("ShouldCrash"sv, fmt::to_string(ServerConfig.ShouldCrash)); size_t MaxWidth = 0; diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 830f36e54..d6bf4454f 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -48,6 +48,7 @@ public: void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } + void SetAllowPortProbing(bool State) { m_AllowPortProbing = State; } void SetServerMode(std::string_view Mode) { m_ServerMode = Mode; } void SetTestMode(bool State) { m_TestMode = State; } @@ -66,6 +67,7 @@ protected: bool m_IsPowerCycle = false; bool m_IsDedicatedMode = false; + bool m_AllowPortProbing = true; bool m_TestMode = false; bool m_NoNetwork = false; bool m_DebugOptionForcedCrash = false; diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h index 2f76f0d6c..101b078e9 100644 --- a/src/zenutil/include/zenutil/zenserverprocess.h +++ b/src/zenutil/include/zenutil/zenserverprocess.h @@ -151,6 +151,11 @@ struct ZenServerInstance return GetBasePort(); } + inline void SpawnServerAndWaitUntilReady(uint16_t BasePort, std::string_view AdditionalServerArgs = std::string_view()) + { + SpawnServer(BasePort, AdditionalServerArgs, /* WaitTimeoutMs */ 100'000); + } + inline void SpawnServer(int BasePort, std::string_view AdditionalServerArgs = std::string_view()) { SpawnServer(BasePort, AdditionalServerArgs, /* WaitTimeoutMs */ 0); |