Merge branch 'main' into sb/local-crashreportssb/local-crashreports

23 files changed, 180 insertions, 46 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 36a211d47..a2a54208e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,11 @@
 - Feature: Added `xmake kill` task to terminate all running zenserver instances
 - Feature: Support `ZEN_MALLOC` environment variable for default allocator selection; default allocator switched to rpmalloc
 - Feature: Added Dockerfile and extended GHA workflows to produce zenserver Docker images, primarily intended for compute nodes
+- Feature: Added `--allow-port-probing` option to control whether zenserver searches for a free port on startup (default: true, automatically false when --dedicated is set)
+- Feature: Added new hub options for controlling provisioned storage server instances:
+  - `--hub-instance-http` — HTTP server implementation for instances (asio/httpsys)
+  - `--hub-instance-http-threads` — Number of HTTP connection threads per instance
+  - `--hub-instance-corelimit` — Limit CPU concurrency per instance
 - Improvement: Added support for CoW block-cloning (used by build download) on Linux (tested with: btrfs/ XFS)
 - Improvement: Added full-file CoW copying on macOS (APFS)
 - Improvement: Updated asio to 1.38.0
@@ -19,6 +24,7 @@
   - `--msan=[y|n]` Enable MemorySanitizer (Linux only, requires all deps instrumented)
   - `--asan=[y|n]` Enable AddressSanitizer (disables mimalloc and sentry)
   - `--tsan=[y|n]` Enable ThreadSanitizer (Linux/Mac only)
+- Improvement: Hub now manages a deterministic port pool for provisioned instances allowing reuse of unused ports
 - Bugfix: Fixed sentry-native build to allow LTO on Windows
 - Bugfix: Minor test stability fixes (flaky hash collisions, per-thread RNG seeds)
 
diff --git a/scripts/test_scripts/metadatas/PS5Client.json b/scripts/test_scripts/metadatas/PS5Client.json
new file mode 100644
index 000000000..55ca338a2
--- /dev/null
+++ b/scripts/test_scripts/metadatas/PS5Client.json
@@ -0,0 +1,9 @@
+{
+  "name": "++Fortnite+Main-CL-50966326 PS4Client",
+  "branch": "ZenBuildTest2",
+  "baselineBranch": "ZenBuildTest2",
+  "platform": "PS5",
+  "project": "Fortnite",
+  "changelist": 50966326,
+  "buildType": "staged-build"
+}
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index 4d98e9650..e5cfbcbae 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -1033,7 +1033,7 @@ CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig
 		ZEN_INFO("using asio HTTP server implementation")
 		return CreateHttpAsioServer(AsioConfig {
 			.ThreadCount = Config.ThreadCount, .ForceLoopback = Config.ForceLoopback, .IsDedicatedServer = Config.IsDedicatedServer,
-			.NoNetwork = Config.NoNetwork, .UnixSocketPath = PathToUtf8(Config.UnixSocketPath),
+			.NoNetwork = Config.NoNetwork, .AllowPortProbing = Config.AllowPortProbing, .UnixSocketPath = PathToUtf8(Config.UnixSocketPath),
 #if ZEN_USE_OPENSSL
 			.HttpsPort = Config.HttpsPort, .CertFile = Config.CertFile, .KeyFile = Config.KeyFile,
 #endif
@@ -1049,6 +1049,7 @@ CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig
 													.IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled,
 													.IsDedicatedServer		 = Config.IsDedicatedServer,
 													.ForceLoopback			 = Config.ForceLoopback,
+													.AllowPortProbing		 = Config.AllowPortProbing,
 													.HttpsPort				 = Config.HttpSys.HttpsPort,
 													.CertThumbprint			 = Config.HttpSys.CertThumbprint,
 													.CertStoreName			 = Config.HttpSys.CertStoreName,
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
index 2a8b2ca94..a7d7f4d9c 100644
--- a/src/zenhttp/include/zenhttp/httpserver.h
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -325,6 +325,7 @@ struct HttpServerPluginConfig
 struct HttpServerConfig
 {
 	bool								IsDedicatedServer = false;	// Should be set to true for shared servers
+	bool								AllowPortProbing  = true;	// IsDedicatedServer forces this to false
 	std::string							ServerClass;				// Choice of HTTP server implementation
 	std::vector<HttpServerPluginConfig> PluginConfigs;
 	bool								ForceLoopback = false;
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index 9f4875eaf..a2cae8762 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -2182,7 +2182,7 @@ HttpAsioServerImpl::Start(uint16_t Port, const AsioConfig& Config)
 	if (!Config.NoNetwork)
 	{
 		m_Acceptor.reset(
-			new asio_http::HttpAcceptor(*this, m_IoService, Port, Config.ForceLoopback, /*AllowPortProbing */ !Config.IsDedicatedServer));
+			new asio_http::HttpAcceptor(*this, m_IoService, Port, Config.ForceLoopback, /*AllowPortProbing */ Config.AllowPortProbing));
 
 		if (!m_Acceptor->IsValid())
 		{
@@ -2224,7 +2224,7 @@ HttpAsioServerImpl::Start(uint16_t Port, const AsioConfig& Config)
 														   *m_SslContext,
 														   gsl::narrow<uint16_t>(Config.HttpsPort),
 														   Config.ForceLoopback,
-														   /*AllowPortProbing*/ !Config.IsDedicatedServer));
+														   /*AllowPortProbing*/ Config.AllowPortProbing));
 
 		if (m_HttpsAcceptor->IsValid())
 		{
diff --git a/src/zenhttp/servers/httpasio.h b/src/zenhttp/servers/httpasio.h
index 21d10170e..154c87d9c 100644
--- a/src/zenhttp/servers/httpasio.h
+++ b/src/zenhttp/servers/httpasio.h
@@ -12,6 +12,7 @@ struct AsioConfig
 	bool		 ForceLoopback	   = false;
 	bool		 IsDedicatedServer = false;
 	bool		 NoNetwork		   = false;
+	bool		 AllowPortProbing  = true;
 	std::string	 UnixSocketPath;
 #if ZEN_USE_OPENSSL
 	int			HttpsPort = 0;	// 0 = auto-assign; set CertFile/KeyFile to enable HTTPS
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index f8fb1c9be..9fe9a2254 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -1153,7 +1153,7 @@ HttpSysServer::RegisterHttpUrls(int BasePort)
 
 	m_BaseUris.clear();
 
-	const bool AllowPortProbing = !m_InitialConfig.IsDedicatedServer;
+	const bool AllowPortProbing = m_InitialConfig.AllowPortProbing;
 	const bool AllowLocalOnly	= !m_InitialConfig.IsDedicatedServer;
 
 	int EffectivePort = BasePort;
diff --git a/src/zenhttp/servers/httpsys.h b/src/zenhttp/servers/httpsys.h
index ca465ad00..88ec13834 100644
--- a/src/zenhttp/servers/httpsys.h
+++ b/src/zenhttp/servers/httpsys.h
@@ -22,6 +22,7 @@ struct HttpSysConfig
 	bool		 IsRequestLoggingEnabled = false;
 	bool		 IsDedicatedServer		 = false;
 	bool		 ForceLoopback			 = false;
+	bool		 AllowPortProbing		 = true;
 	int			 HttpsPort				 = 0;  // 0 = HTTPS disabled
 	std::string	 CertThumbprint;			   // Hex SHA-1 (40 chars) for auto SSL binding
 	std::string	 CertStoreName = "MY";		   // Windows certificate store name
diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp
index 9c1669256..958a0b050 100644
--- a/src/zenserver-test/hub-tests.cpp
+++ b/src/zenserver-test/hub-tests.cpp
@@ -47,7 +47,7 @@ TEST_CASE("hub.lifecycle.children")
 {
 	ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer);
 
-	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady();
+	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady("--hub-instance-corelimit=2 --hub-instance-http-threads=6");
 	REQUIRE(PortNumber != 0);
 
 	SUBCASE("spawn")
diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp
index 0d8550c5b..724ef9ad2 100644
--- a/src/zenserver/compute/computeserver.cpp
+++ b/src/zenserver/compute/computeserver.cpp
@@ -951,6 +951,7 @@ ZenComputeServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
 	Server.SetContentRoot(m_ServerOptions.ContentDir);
 	Server.SetTestMode(m_ServerOptions.IsTest);
 	Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+	Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing);
 
 	const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
 	if (EffectiveBasePort == -1)
diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp
index 60ae93853..15f6f79f3 100644
--- a/src/zenserver/config/config.cpp
+++ b/src/zenserver/config/config.cpp
@@ -133,6 +133,7 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions
 	// server
 
 	LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv);
+	LuaOptions.AddOption("server.allowportprobing"sv, ServerOptions.AllowPortProbing, "allow-port-probing"sv);
 	LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv);
 	LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
 	LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
@@ -223,8 +224,11 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi
 #endif
 
 	options.add_options()("dedicated",
-						  "Enable dedicated server mode",
+						  "Enable dedicated server mode, disables '--allow-port-probing' and allocates more resources.",
 						  cxxopts::value<bool>(ServerOptions.IsDedicated)->default_value("false"));
+	options.add_options()("allow-port-probing",
+						  "Allow searching for an available port, disabled if '--dedicated' is enabled.",
+						  cxxopts::value<bool>(ServerOptions.AllowPortProbing)->default_value("true"));
 	options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(ServerOptions.IsDebug)->default_value("false"));
 	options.add_options()("clean",
 						  "Clean out all state at startup",
@@ -691,6 +695,7 @@ ZenServerConfiguratorBase::Configure(int argc, char* argv[])
 	}
 
 	m_ServerOptions.HttpConfig.IsDedicatedServer = m_ServerOptions.IsDedicated;
+	m_ServerOptions.HttpConfig.AllowPortProbing	 = !m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing;
 }
 
 void
diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h
index e481c7225..5078fe71a 100644
--- a/src/zenserver/config/config.h
+++ b/src/zenserver/config/config.h
@@ -61,6 +61,7 @@ struct ZenServerConfig
 	int					  CoreLimit		   = 0;		// If set, hardware concurrency queries are capped at this number
 	int					  LieCpu		   = 0;
 	bool				  IsDedicated	   = false;	 // Indicates a dedicated/shared instance, with larger resource requirements
+	bool				  AllowPortProbing = true;	 // Automatically false if IsDedicated is true
 	bool				  ShouldCrash	   = false;	 // Option for testing crash handling
 	bool				  IsFirstRun	   = false;
 	std::filesystem::path ConfigFile;		   // Path to Lua config file
diff --git a/src/zenserver/hub/hub.cpp b/src/zenserver/hub/hub.cpp
index c9720b32d..3c9f40eaa 100644
--- a/src/zenserver/hub/hub.cpp
+++ b/src/zenserver/hub/hub.cpp
@@ -22,6 +22,8 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #	include <zencore/workthreadpool.h>
 #endif
 
+#include <numeric>
+
 namespace zen {
 
 ///////////////////////////////////////////////////////////////////////////
@@ -137,10 +139,10 @@ Hub::Hub(const Configuration&		   Config,
 	m_HydrationTempPath = m_RunEnvironment.CreateChildDir("hydration_temp");
 	ZEN_INFO("using hydration temp path: '{}'", m_HydrationTempPath);
 
-	// This is necessary to ensure the hub assigns a distinct port range.
-	// We need to do this primarily because otherwise automated tests will
-	// fail as the test runner will create processes in the default range.
-	m_RunEnvironment.SetNextPortNumber(m_Config.BasePortNumber);
+	ZEN_ASSERT(uint64_t(Config.BasePortNumber) + Config.InstanceLimit <= std::numeric_limits<uint16_t>::max());
+
+	m_FreePorts.resize(Config.InstanceLimit);
+	std::iota(m_FreePorts.begin(), m_FreePorts.end(), Config.BasePortNumber);
 
 #if ZEN_PLATFORM_WINDOWS
 	if (m_Config.UseJobObject)
@@ -199,6 +201,15 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 	bool				   IsNewInstance = false;
 	{
 		RwLock::ExclusiveLockScope _(m_Lock);
+		uint16_t				   AllocatedPort		= 0;
+		auto					   RestoreAllocatedPort = MakeGuard([this, &AllocatedPort]() {
+			  if (AllocatedPort != 0)
+			  {
+				  m_FreePorts.push_back(AllocatedPort);
+				  AllocatedPort = 0;
+			  }
+		  });
+
 		if (auto It = m_Instances.find(std::string(ModuleId)); It == m_Instances.end())
 		{
 			std::string Reason;
@@ -211,9 +222,18 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 				return false;
 			}
 
-			IsNewInstance = true;
-			auto NewInstance =
-				std::make_unique<StorageServerInstance>(m_RunEnvironment, ModuleId, m_FileHydrationPath, m_HydrationTempPath);
+			AllocatedPort = m_FreePorts.front();
+			m_FreePorts.pop_front();
+
+			IsNewInstance	 = true;
+			auto NewInstance = std::make_unique<StorageServerInstance>(
+				m_RunEnvironment,
+				StorageServerInstance::Configuration{.BasePort			= AllocatedPort,
+													 .HydrationTempPath = m_HydrationTempPath,
+													 .FileHydrationPath = m_FileHydrationPath,
+													 .HttpThreadCount	= m_Config.InstanceHttpThreadCount,
+													 .CoreLimit			= m_Config.InstanceCoreLimit},
+				ModuleId);
 #if ZEN_PLATFORM_WINDOWS
 			if (m_JobObject.IsValid())
 			{
@@ -222,6 +242,7 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 #endif
 			Instance = NewInstance.get();
 			m_Instances.emplace(std::string(ModuleId), std::move(NewInstance));
+			AllocatedPort = 0;
 
 			ZEN_INFO("Created new storage server instance for module '{}'", ModuleId);
 		}
@@ -258,7 +279,13 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 		{
 			// Clean up
 			RwLock::ExclusiveLockScope _(m_Lock);
-			m_Instances.erase(std::string(ModuleId));
+			if (auto It = m_Instances.find(std::string(ModuleId)); It != m_Instances.end())
+			{
+				ZEN_ASSERT(It->second != nullptr);
+				uint16_t BasePort = It->second->GetBasePort();
+				m_FreePorts.push_back(BasePort);
+				m_Instances.erase(It);
+			}
 		}
 		return false;
 	}
@@ -337,6 +364,7 @@ Hub::Deprovision(const std::string& ModuleId, std::string& OutReason)
 	auto _ = MakeGuard([&] {
 		RwLock::ExclusiveLockScope _(m_Lock);
 		m_DeprovisioningModules.erase(ModuleId);
+		m_FreePorts.push_back(BasePort);
 	});
 
 	Instance->Deprovision();
@@ -413,7 +441,17 @@ Hub::CanProvisionInstance(std::string_view ModuleId, std::string& OutReason)
 
 	if (gsl::narrow_cast<int>(m_Instances.size()) >= m_Config.InstanceLimit)
 	{
-		OutReason = fmt::format("instance limit exceeded ({})", m_Config.InstanceLimit);
+		OutReason = fmt::format("instance limit ({}) exceeded", m_Config.InstanceLimit);
+
+		return false;
+	}
+
+	// Since deprovisioning happens outside the lock and we don't add the port back until the instance is full shut down we might be under
+	// the instance limit but all ports may be in use
+	if (m_FreePorts.empty())
+	{
+		OutReason = fmt::format("no free ports available, deprovisioning of instances might be in flight ({})",
+								m_Config.InstanceLimit - m_Instances.size());
 
 		return false;
 	}
diff --git a/src/zenserver/hub/hub.h b/src/zenserver/hub/hub.h
index 8b61f988d..8a84a558b 100644
--- a/src/zenserver/hub/hub.h
+++ b/src/zenserver/hub/hub.h
@@ -7,6 +7,7 @@
 #include <zencore/system.h>
 #include <zenutil/zenserverprocess.h>
 
+#include <deque>
 #include <filesystem>
 #include <functional>
 #include <memory>
@@ -43,6 +44,9 @@ public:
 		uint16_t BasePortNumber = 21000;
 
 		int InstanceLimit = 1000;
+
+		uint32_t InstanceHttpThreadCount = 0;  // Deduce from core count
+		int		 InstanceCoreLimit		 = 0;  // Use hardware core count
 	};
 
 	typedef std::function<void(std::string_view ModuleId, const HubProvisionedInstanceInfo& Info)> ProvisionModuleCallbackFunc;
@@ -121,6 +125,7 @@ private:
 	ResourceMetrics															m_ResourceLimits;
 	SystemMetrics															m_HostMetrics;
 	int																		m_MaxInstanceCount = 0;
+	std::deque<uint16_t>													m_FreePorts;
 
 	void UpdateStats();
 	void UpdateCapacityMetrics();
diff --git a/src/zenserver/hub/storageserverinstance.cpp b/src/zenserver/hub/storageserverinstance.cpp
index f24379715..68de5e274 100644
--- a/src/zenserver/hub/storageserverinstance.cpp
+++ b/src/zenserver/hub/storageserverinstance.cpp
@@ -11,16 +11,13 @@
 
 namespace zen {
 
-StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment,
-											 std::string_view	   ModuleId,
-											 std::filesystem::path FileHydrationPath,
-											 std::filesystem::path HydrationTempPath)
-: m_ModuleId(ModuleId)
+StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment, const Configuration& Config, std::string_view ModuleId)
+: m_Config(Config)
+, m_ModuleId(ModuleId)
 , m_ServerInstance(RunEnvironment, ZenServerInstance::ServerMode::kStorageServer)
-, m_HydrationPath(FileHydrationPath)
 {
 	m_BaseDir = RunEnvironment.CreateChildDir(ModuleId);
-	m_TempDir = HydrationTempPath / ModuleId;
+	m_TempDir = Config.HydrationTempPath / ModuleId;
 }
 
 StorageServerInstance::~StorageServerInstance()
@@ -37,9 +34,20 @@ StorageServerInstance::SpawnServerProcess()
 #if ZEN_PLATFORM_WINDOWS
 	m_ServerInstance.SetJobObject(m_JobObject);
 #endif
-	const uint16_t BasePort = m_ServerInstance.SpawnServerAndWaitUntilReady();
 
-	ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, BasePort);
+	ExtendableStringBuilder<256> AdditionalOptions;
+	AdditionalOptions << "--allow-port-probing=false";
+	if (m_Config.HttpThreadCount != 0)
+	{
+		AdditionalOptions << " --http-threads=" << m_Config.HttpThreadCount;
+	}
+	if (m_Config.CoreLimit != 0)
+	{
+		AdditionalOptions << " --corelimit=" << m_Config.CoreLimit;
+	}
+
+	m_ServerInstance.SpawnServerAndWaitUntilReady(m_Config.BasePort, AdditionalOptions.ToView());
+	ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, m_Config.BasePort);
 
 	m_ServerInstance.EnableShutdownOnDestroy();
 }
@@ -178,7 +186,7 @@ StorageServerInstance::Hydrate()
 	HydrationConfig Config{.ServerStateDir		= m_BaseDir,
 						   .TempDir				= m_TempDir,
 						   .ModuleId			= m_ModuleId,
-						   .TargetSpecification = WideToUtf8(m_HydrationPath.native())};
+						   .TargetSpecification = WideToUtf8(m_Config.FileHydrationPath.native())};
 
 	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator();
 
@@ -192,7 +200,7 @@ StorageServerInstance::Dehydrate()
 	HydrationConfig Config{.ServerStateDir		= m_BaseDir,
 						   .TempDir				= m_TempDir,
 						   .ModuleId			= m_ModuleId,
-						   .TargetSpecification = WideToUtf8(m_HydrationPath.native())};
+						   .TargetSpecification = WideToUtf8(m_Config.FileHydrationPath.native())};
 
 	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator();
 
diff --git a/src/zenserver/hub/storageserverinstance.h b/src/zenserver/hub/storageserverinstance.h
index a2f3d25d7..23196d835 100644
--- a/src/zenserver/hub/storageserverinstance.h
+++ b/src/zenserver/hub/storageserverinstance.h
@@ -21,10 +21,16 @@ namespace zen {
 class StorageServerInstance
 {
 public:
-	StorageServerInstance(ZenServerEnvironment& RunEnvironment,
-						  std::string_view		ModuleId,
-						  std::filesystem::path FileHydrationPath,
-						  std::filesystem::path HydrationTempPath);
+	struct Configuration
+	{
+		uint16_t			  BasePort;
+		std::filesystem::path HydrationTempPath;
+		std::filesystem::path FileHydrationPath;
+		uint32_t			  HttpThreadCount = 0;	// Deduce from core count
+		int					  CoreLimit		  = 0;	// Use hardware core count
+	};
+
+	StorageServerInstance(ZenServerEnvironment& RunEnvironment, const Configuration& Config, std::string_view ModuleId);
 	~StorageServerInstance();
 
 	void Provision();
@@ -45,15 +51,17 @@ public:
 #endif
 
 private:
-	void				  WakeLocked();
-	RwLock				  m_Lock;
-	std::string			  m_ModuleId;
+	void				WakeLocked();
+	RwLock				m_Lock;
+	const Configuration m_Config;
+	std::string			m_ModuleId;
+	ZenServerInstance	m_ServerInstance;
+
 	std::atomic<bool>	  m_IsProvisioned{false};
 	std::atomic<bool>	  m_IsHibernated{false};
-	ZenServerInstance	  m_ServerInstance;
 	std::filesystem::path m_BaseDir;
+
 	std::filesystem::path m_TempDir;
-	std::filesystem::path m_HydrationPath;
 	ResourceMetrics		  m_ResourceMetrics;
 #if ZEN_PLATFORM_WINDOWS
 	JobObject* m_JobObject = nullptr;
diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
index b0ae0a8b1..313be977c 100644
--- a/src/zenserver/hub/zenhubserver.cpp
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -26,6 +26,15 @@ namespace zen {
 void
 ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options)
 {
+	const char* DefaultInstanceHttp = "asio";
+
+#if ZEN_WITH_HTTPSYS
+	if (!windows::IsRunningOnWine())
+	{
+		DefaultInstanceHttp = "httpsys";
+	}
+#endif
+
 	Options.add_option("hub",
 					   "",
 					   "upstream-notification-endpoint",
@@ -60,6 +69,31 @@ ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options)
 					   "Maximum number of provisioned instances for this hub",
 					   cxxopts::value<int>(m_ServerOptions.HubInstanceLimit)->default_value("1000"),
 					   "");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-instance-http",
+					   "Select HTTP server implementation for provisioned instances (asio|"
+#if ZEN_WITH_HTTPSYS
+					   "httpsys|"
+#endif
+					   "null)",
+					   cxxopts::value<std::string>(m_ServerOptions.HubInstanceHttpClass)->default_value(DefaultInstanceHttp),
+					   "<instance http class>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-instance-http-threads",
+					   "Number of http server connection threads for provisioned instances",
+					   cxxopts::value<unsigned int>(m_ServerOptions.HubInstanceHttpThreadCount),
+					   "<instance http threads>");
+	Options.add_option("hub",
+					   "",
+					   "hub-instance-corelimit",
+					   "Limit concurrency of provisioned instances",
+					   cxxopts::value(m_ServerOptions.HubInstanceCoreLimit),
+					   "<instance core limit>");
+
 #if ZEN_PLATFORM_WINDOWS
 	Options.add_option("hub",
 					   "",
@@ -231,10 +265,15 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig)
 
 	ZEN_INFO("instantiating Hub");
 	m_Hub = std::make_unique<Hub>(
-		Hub::Configuration{.UseJobObject   = ServerConfig.HubUseJobObject,
-						   .BasePortNumber = ServerConfig.HubBasePortNumber,
-						   .InstanceLimit  = ServerConfig.HubInstanceLimit},
-		ZenServerEnvironment(ZenServerEnvironment::Hub, ServerConfig.DataDir / "hub", ServerConfig.DataDir / "servers"),
+		Hub::Configuration{.UseJobObject			= ServerConfig.HubUseJobObject,
+						   .BasePortNumber			= ServerConfig.HubBasePortNumber,
+						   .InstanceLimit			= ServerConfig.HubInstanceLimit,
+						   .InstanceHttpThreadCount = ServerConfig.HubInstanceHttpThreadCount,
+						   .InstanceCoreLimit		= ServerConfig.HubInstanceCoreLimit},
+		ZenServerEnvironment(ZenServerEnvironment::Hub,
+							 ServerConfig.DataDir / "hub",
+							 ServerConfig.DataDir / "servers",
+							 ServerConfig.HubInstanceHttpClass),
 		m_ConsulClient ? [this, HubInstanceId = fmt::format("zen-hub-{}", ServerConfig.InstanceId)](
 							 std::string_view				   ModuleId,
 							 const HubProvisionedInstanceInfo& Info) { OnProvisioned(HubInstanceId, ModuleId, Info); }
@@ -405,6 +444,7 @@ ZenHubServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
 	Server.SetContentRoot(m_ServerOptions.ContentDir);
 	Server.SetTestMode(m_ServerOptions.IsTest);
 	Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+	Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing);
 
 	const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
 	if (EffectiveBasePort == -1)
diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h
index f6a3eb1bc..1036598bb 100644
--- a/src/zenserver/hub/zenhubserver.h
+++ b/src/zenserver/hub/zenhubserver.h
@@ -24,9 +24,12 @@ struct ZenHubServerConfig : public ZenServerConfig
 	std::string UpstreamNotificationEndpoint;
 	std::string InstanceId;		 // For use in notifications
 	std::string ConsulEndpoint;	 // If set, enables Consul service registration
-	uint16_t	HubBasePortNumber = 21000;
-	int			HubInstanceLimit  = 1000;
-	bool		HubUseJobObject	  = true;
+	uint16_t	HubBasePortNumber		   = 21000;
+	int			HubInstanceLimit		   = 1000;
+	bool		HubUseJobObject			   = true;
+	std::string HubInstanceHttpClass	   = "asio";
+	uint32_t	HubInstanceHttpThreadCount = 0;	 // Deduce from core count
+	int			HubInstanceCoreLimit	   = 0;	 // Use hardware core count
 };
 
 class Hub;
@@ -79,8 +82,6 @@ public:
 	void Run();
 	void Cleanup();
 
-	void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; }
-	void SetTestMode(bool State) { m_TestMode = State; }
 	void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; }
 	void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; }
 
@@ -88,8 +89,6 @@ private:
 	void OnProvisioned(std::string_view HubInstanceId, std::string_view ModuleId, const HubProvisionedInstanceInfo& Info);
 	void OnDeprovisioned(std::string_view HubInstanceId, std::string_view ModuleId, const HubProvisionedInstanceInfo& Info);
 
-	bool				  m_IsDedicatedMode = false;
-	bool				  m_TestMode		= false;
 	std::filesystem::path m_DataRoot;
 	std::filesystem::path m_ContentRoot;
 	bool				  m_DebugOptionForcedCrash = false;
diff --git a/src/zenserver/proxy/zenproxyserver.cpp b/src/zenserver/proxy/zenproxyserver.cpp
index c768e940a..cf84c159a 100644
--- a/src/zenserver/proxy/zenproxyserver.cpp
+++ b/src/zenserver/proxy/zenproxyserver.cpp
@@ -454,6 +454,7 @@ ZenProxyServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
 	Server.SetContentRoot(m_ServerOptions.ContentDir);
 	Server.SetTestMode(m_ServerOptions.IsTest);
 	Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+	Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing);
 
 	const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
 	if (EffectiveBasePort == -1)
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index bba5e0a61..f5ede5692 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -975,6 +975,7 @@ ZenStorageServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
 	Server.SetContentRoot(m_ServerOptions.ContentDir);
 	Server.SetTestMode(m_ServerOptions.IsTest);
 	Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+	Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing);
 
 	int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
 	if (EffectiveBasePort == -1)
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 6760e0372..1cd8ed846 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -517,6 +517,7 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig)
 	});
 	// clang-format on
 	Settings.emplace_back("IsDedicated"sv, fmt::to_string(ServerConfig.IsDedicated));
+	Settings.emplace_back("AllowPortProbing"sv, fmt::to_string(ServerConfig.AllowPortProbing));
 	Settings.emplace_back("ShouldCrash"sv, fmt::to_string(ServerConfig.ShouldCrash));
 
 	size_t MaxWidth = 0;
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 830f36e54..d6bf4454f 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -48,6 +48,7 @@ public:
 	void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; }
 	void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; }
 	void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; }
+	void SetAllowPortProbing(bool State) { m_AllowPortProbing = State; }
 	void SetServerMode(std::string_view Mode) { m_ServerMode = Mode; }
 	void SetTestMode(bool State) { m_TestMode = State; }
 
@@ -66,6 +67,7 @@ protected:
 	bool							m_IsPowerCycle = false;
 
 	bool		m_IsDedicatedMode		 = false;
+	bool		m_AllowPortProbing		 = true;
 	bool		m_TestMode				 = false;
 	bool		m_NoNetwork				 = false;
 	bool		m_DebugOptionForcedCrash = false;
diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h
index 2f76f0d6c..101b078e9 100644
--- a/src/zenutil/include/zenutil/zenserverprocess.h
+++ b/src/zenutil/include/zenutil/zenserverprocess.h
@@ -151,6 +151,11 @@ struct ZenServerInstance
 		return GetBasePort();
 	}
 
+	inline void SpawnServerAndWaitUntilReady(uint16_t BasePort, std::string_view AdditionalServerArgs = std::string_view())
+	{
+		SpawnServer(BasePort, AdditionalServerArgs, /* WaitTimeoutMs */ 100'000);
+	}
+
 	inline void SpawnServer(int BasePort, std::string_view AdditionalServerArgs = std::string_view())
 	{
 		SpawnServer(BasePort, AdditionalServerArgs, /* WaitTimeoutMs */ 0);