Merge branch 'main' into zs/long-filename-improvement

34 files changed, 465 insertions, 116 deletions

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index bc621ad23..d19b006b7 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -36,7 +36,7 @@ jobs:
         scripts\sentry-cli --auth-token ${{ secrets.SENTRY_API_KEY }} debug-files upload --org to --project zen-server build/windows/x64/release/zenserver.exe build/windows/x64/release/zen.exe build/windows/x64/release/zenserver.pdb build/windows/x64/release/zen.pdb build/windows/x64/release/zenserver.src.zip build/windows/x64/release/zen.src.zip
 
     - name: Upload zenserver-win64
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3-node20
       with:
         name: zenserver-win64
         path: build/zenserver-win64.zip
@@ -79,7 +79,7 @@ jobs:
         scripts/sentry-cli --auth-token ${{ secrets.SENTRY_API_KEY }} debug-files upload --org to --project zen-server build/linux/x86_64/release/zenserver build/linux/x86_64/release/zen build/linux/x86_64/release/zenserver.sym build/linux/x86_64/release/zen.sym build/linux/x86_64/release/zenserver.src.zip build/linux/x86_64/release/zen.src.zip
 
     - name: Upload zenserver-linux
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3-node20
       with:
         name: zenserver-linux
         path: build/zenserver-linux.zip
@@ -122,7 +122,7 @@ jobs:
         scripts/sentry-cli --auth-token ${{ secrets.SENTRY_API_KEY }} debug-files upload --org to --project zen-server build/macosx/x86_64/release/zenserver build/macosx/x86_64/release/zen build/macosx/x86_64/release/zenserver.dSYM build/macosx/x86_64/release/zen.dSYM build/macosx/x86_64/release/zenserver.src.zip build/macosx/x86_64/release/zen.src.zip
 
     - name: Upload zenserver-macos
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3-node20
       with:
         name: zenserver-macos
         path: build/zenserver-macos.zip
@@ -141,13 +141,13 @@ jobs:
       run: echo "content=$(cat VERSION.txt)" >> "$GITHUB_OUTPUT"
 
     - name: Download Linux bundle
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v3-node20
       with:
         name: zenserver-linux
         path: artifacts/linux
 
     - name: Download Windows bundle
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v3-node20
       with:
         name: zenserver-win64
         path: artifacts/win64
@@ -207,19 +207,19 @@ jobs:
         scripts/sentry-cli --auth-token ${{ secrets.SENTRY_API_KEY }} releases new --org to --project zen-server ${{steps.read_version.outputs.content}}
 
     - name: Download Linux artifacts
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v3-node20
       with:
         name: zenserver-linux
         path: linux
 
     - name: Download MacOS artifacts
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v3-node20
       with:
         name: zenserver-macos
         path: macos
 
     - name: Download Windows artifacts
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v3-node20
       with:
         name: zenserver-win64
         path: win64
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index 2c99e8a46..932020337 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -91,7 +91,7 @@ jobs:
 
     - name: Upload report
       if: ${{ (failure() || success()) && (matrix.config == 'debug') }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3-node20
       with:
         name: reports-win64
         path: build/reports/*.xml
@@ -112,7 +112,7 @@ jobs:
 
     - name: Upload zenserver-win64
       if: ${{ github.ref_name == 'main' && matrix.config == 'release' }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3-node20
       with:
         name: zenserver-win64
         path: build/zenserver-win64.zip
@@ -159,7 +159,7 @@ jobs:
 
     - name: Upload report
       if: ${{ (failure() || success()) && (matrix.config == 'debug') }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3-node20
       with:
         name: reports-linux
         path: build/reports/*.xml
@@ -180,7 +180,7 @@ jobs:
 
     - name: Upload zenserver-linux
       if: ${{ github.ref_name == 'main' && matrix.config == 'release' }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3-node20
       with:
         name: zenserver-linux
         path: build/zenserver-linux.zip
@@ -222,7 +222,7 @@ jobs:
 
     - name: Upload report
       if: ${{ (failure() || success()) && (matrix.config == 'debug') }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3-node20
       with:
         name: reports-macos
         path: build/reports/*.xml
@@ -243,7 +243,7 @@ jobs:
 
     - name: Upload zenserver-macos
       if: ${{ github.ref_name == 'main' && matrix.config == 'release' }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3-node20
       with:
         name: zenserver-macos
         path: build/zenserver-macos.zip
@@ -259,13 +259,13 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Download Linux bundle
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v3-node20
       with:
         name: zenserver-linux
         path: artifacts/linux
 
     - name: Download Windows bundle
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v3-node20
       with:
         name: zenserver-win64
         path: artifacts/win64
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cad801247..5918c2328 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,11 +10,21 @@
 - Feature: Added `xmake kill` task to terminate all running zenserver instances
 - Feature: Support `ZEN_MALLOC` environment variable for default allocator selection; default allocator switched to rpmalloc
 - Feature: Added Dockerfile and extended GHA workflows to produce zenserver Docker images, primarily intended for compute nodes
+- Feature: Added `--allow-port-probing` option to control whether zenserver searches for a free port on startup (default: true, automatically false when --dedicated is set)
+- Feature: Added new hub options for controlling provisioned storage server instances:
+  - `--hub-instance-http` — HTTP server implementation for instances (asio/httpsys)
+  - `--hub-instance-http-threads` — Number of HTTP connection threads per instance
+  - `--hub-instance-corelimit` — Limit CPU concurrency per instance
 - Improvement: Added support for CoW block-cloning (used by build download) on Linux (tested with: btrfs/ XFS)
 - Improvement: Added full-file CoW copying on macOS (APFS)
 - Improvement: Updated asio to 1.38.0
 - Improvement: Updated fmt to 1.12.1
 - Improvement: Fixed issue where oplog upload could create blocks larger than the max limit (64Mb)
+- Improvement: Add easy access options for sanitizers with `xmake config` and `xmake test` as options
+  - `--msan=[y|n]` Enable MemorySanitizer (Linux only, requires all deps instrumented)
+  - `--asan=[y|n]` Enable AddressSanitizer (disables mimalloc and sentry)
+  - `--tsan=[y|n]` Enable ThreadSanitizer (Linux/Mac only)
+- Improvement: Hub now manages a deterministic port pool for provisioned instances allowing reuse of unused ports
 - Bugfix: Fixed sentry-native build to allow LTO on Windows
 - Bugfix: Minor test stability fixes (flaky hash collisions, per-thread RNG seeds)
 - Bugfix: Handle long paths in the project store
diff --git a/asan.supp b/asan.supp
new file mode 100644
index 000000000..e784c3fb8
--- /dev/null
+++ b/asan.supp
@@ -0,0 +1,18 @@
+# AddressSanitizer options for Zen
+#
+# IMPORTANT: There is no native "options from file" mechanism for ASAN on any
+# platform or compiler. The sanitizer.options rule in xmake.lua parses this
+# file and sets ASAN_OPTIONS directly. If you run a binary without xmake you
+# must set ASAN_OPTIONS manually, e.g.:
+#   Linux/Mac:  ASAN_OPTIONS="new_delete_type_mismatch=0" ./zenserver
+#   Windows:    set ASAN_OPTIONS=new_delete_type_mismatch=0
+#
+# Format:
+#   key=value   ASAN runtime option
+#   #           comment line (the whole line must start with #)
+#   <blank>     ignored
+
+# EASTL's allocator stores a size_t header in every allocation, but the
+# deallocation path passes the element size without the header. ASAN reports
+# this as a new/delete type mismatch. This is a false positive.
+new_delete_type_mismatch=0
diff --git a/msan.supp b/msan.supp
new file mode 100644
index 000000000..1131a7915
--- /dev/null
+++ b/msan.supp
@@ -0,0 +1,14 @@
+# MemorySanitizer options for Zen
+#
+# IMPORTANT: This file is NOT passed directly to the MSAN runtime. The
+# sanitizer.msan_options rule in xmake.lua parses it and constructs
+# MSAN_OPTIONS. If you run a binary directly (not via xmake run / xmake test)
+# you must set MSAN_OPTIONS manually.
+#
+# MSAN has no native suppression file mechanism; only key=value options are
+# supported here.
+#
+# Format:
+#   key=value   MSAN runtime option; xmake sets it directly in MSAN_OPTIONS
+#   #           comment line (the whole line must start with #)
+#   <blank>     ignored
diff --git a/scripts/test.lua b/scripts/test.lua
index df1218ce8..3c18225fb 100644
--- a/scripts/test.lua
+++ b/scripts/test.lua
@@ -86,11 +86,22 @@ function main()
         arch = "x86_64"
     end
 
+    local want_asan = option.get("asan") == true
+    local have_asan = config.get("asan") == true
+    local want_tsan = option.get("tsan") == true
+    local have_tsan = config.get("tsan") == true
+    local want_msan = option.get("msan") == true
+    local have_msan = config.get("msan") == true
+
     -- Only reconfigure if current config doesn't already match
-    if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch then
+    if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch
+            or want_asan ~= have_asan or want_tsan ~= have_tsan or want_msan ~= have_msan then
         local toolchain_flag = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or ""
         local sdk_flag = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or ""
-        os.exec("xmake config -y -c -m debug -p %s -a %s %s %s", plat, arch, toolchain_flag, sdk_flag)
+        local asan_flag = want_asan and "--asan=y" or ""
+        local tsan_flag = want_tsan and "--tsan=y" or ""
+        local msan_flag = want_msan and "--msan=y" or ""
+        os.exec("xmake config -y -c -m debug -p %s -a %s %s %s %s %s %s", plat, arch, toolchain_flag, sdk_flag, asan_flag, tsan_flag, msan_flag)
     end
 
     -- Build targets we're going to run
@@ -104,7 +115,7 @@ function main()
 
     local use_junit_reporting = option.get("junit")
     local use_noskip = option.get("noskip")
-    local use_verbose = option.get("verbose")
+    local use_verbose = option.get("output")
     local repeat_count = tonumber(option.get("repeat")) or 1
     local extra_args = option.get("arguments") or {}
     local junit_report_files = {}
diff --git a/scripts/test_scripts/metadatas/PS5Client.json b/scripts/test_scripts/metadatas/PS5Client.json
new file mode 100644
index 000000000..55ca338a2
--- /dev/null
+++ b/scripts/test_scripts/metadatas/PS5Client.json
@@ -0,0 +1,9 @@
+{
+  "name": "++Fortnite+Main-CL-50966326 PS4Client",
+  "branch": "ZenBuildTest2",
+  "baselineBranch": "ZenBuildTest2",
+  "platform": "PS5",
+  "project": "Fortnite",
+  "changelist": 50966326,
+  "buildType": "staged-build"
+}
diff --git a/src/zencore/include/zencore/memory/fmalloc.h b/src/zencore/include/zencore/memory/fmalloc.h
index 0c183cfd0..c50a9729c 100644
--- a/src/zencore/include/zencore/memory/fmalloc.h
+++ b/src/zencore/include/zencore/memory/fmalloc.h
@@ -9,6 +9,7 @@
 // Detect if any sanitizers are enabled.
 
 #if defined(__has_feature)
+// Clang: query each sanitizer individually
 #	if __has_feature(address_sanitizer)
 #		define ZEN_ADDRESS_SANITIZER 1
 #	endif
@@ -21,8 +22,14 @@
 #	if __has_feature(leak_sanitizer)
 #		define ZEN_LEAK_SANITIZER 1
 #	endif
-#elif defined(__SANITIZE_ADDRESS__)	 // For Windows
-#	define ZEN_ADDRESS_SANITIZER 1
+#else
+// MSVC and GCC: check predefined macros set by the compiler when sanitizers are active
+#	if defined(__SANITIZE_ADDRESS__)  // MSVC (ASAN only), GCC
+#		define ZEN_ADDRESS_SANITIZER 1
+#	endif
+#	if defined(__SANITIZE_THREAD__)  // GCC
+#		define ZEN_THREAD_SANITIZER 1
+#	endif
 #endif
 
 #if !defined(ZEN_ADDRESS_SANITIZER)
diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua
index 171f4c533..b08975df1 100644
--- a/src/zencore/xmake.lua
+++ b/src/zencore/xmake.lua
@@ -21,7 +21,7 @@ target('zencore')
         add_deps("rpmalloc")
     end
 
-    if has_config("zenmimalloc") then
+    if has_config("zenmimalloc") and not use_asan then
         add_packages("mimalloc")
     end
 
@@ -47,7 +47,7 @@ target('zencore')
         {public=true}
     )
 
-    if has_config("zensentry") then
+    if has_config("zensentry") and not use_asan then
         add_packages("sentry-native")
 
         if is_os("windows") then
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index 4d98e9650..e5cfbcbae 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -1033,7 +1033,7 @@ CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig
 		ZEN_INFO("using asio HTTP server implementation")
 		return CreateHttpAsioServer(AsioConfig {
 			.ThreadCount = Config.ThreadCount, .ForceLoopback = Config.ForceLoopback, .IsDedicatedServer = Config.IsDedicatedServer,
-			.NoNetwork = Config.NoNetwork, .UnixSocketPath = PathToUtf8(Config.UnixSocketPath),
+			.NoNetwork = Config.NoNetwork, .AllowPortProbing = Config.AllowPortProbing, .UnixSocketPath = PathToUtf8(Config.UnixSocketPath),
 #if ZEN_USE_OPENSSL
 			.HttpsPort = Config.HttpsPort, .CertFile = Config.CertFile, .KeyFile = Config.KeyFile,
 #endif
@@ -1049,6 +1049,7 @@ CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig
 													.IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled,
 													.IsDedicatedServer		 = Config.IsDedicatedServer,
 													.ForceLoopback			 = Config.ForceLoopback,
+													.AllowPortProbing		 = Config.AllowPortProbing,
 													.HttpsPort				 = Config.HttpSys.HttpsPort,
 													.CertThumbprint			 = Config.HttpSys.CertThumbprint,
 													.CertStoreName			 = Config.HttpSys.CertStoreName,
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
index 2a8b2ca94..a7d7f4d9c 100644
--- a/src/zenhttp/include/zenhttp/httpserver.h
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -325,6 +325,7 @@ struct HttpServerPluginConfig
 struct HttpServerConfig
 {
 	bool								IsDedicatedServer = false;	// Should be set to true for shared servers
+	bool								AllowPortProbing  = true;	// IsDedicatedServer forces this to false
 	std::string							ServerClass;				// Choice of HTTP server implementation
 	std::vector<HttpServerPluginConfig> PluginConfigs;
 	bool								ForceLoopback = false;
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index 9f4875eaf..a2cae8762 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -2182,7 +2182,7 @@ HttpAsioServerImpl::Start(uint16_t Port, const AsioConfig& Config)
 	if (!Config.NoNetwork)
 	{
 		m_Acceptor.reset(
-			new asio_http::HttpAcceptor(*this, m_IoService, Port, Config.ForceLoopback, /*AllowPortProbing */ !Config.IsDedicatedServer));
+			new asio_http::HttpAcceptor(*this, m_IoService, Port, Config.ForceLoopback, /*AllowPortProbing */ Config.AllowPortProbing));
 
 		if (!m_Acceptor->IsValid())
 		{
@@ -2224,7 +2224,7 @@ HttpAsioServerImpl::Start(uint16_t Port, const AsioConfig& Config)
 														   *m_SslContext,
 														   gsl::narrow<uint16_t>(Config.HttpsPort),
 														   Config.ForceLoopback,
-														   /*AllowPortProbing*/ !Config.IsDedicatedServer));
+														   /*AllowPortProbing*/ Config.AllowPortProbing));
 
 		if (m_HttpsAcceptor->IsValid())
 		{
diff --git a/src/zenhttp/servers/httpasio.h b/src/zenhttp/servers/httpasio.h
index 21d10170e..154c87d9c 100644
--- a/src/zenhttp/servers/httpasio.h
+++ b/src/zenhttp/servers/httpasio.h
@@ -12,6 +12,7 @@ struct AsioConfig
 	bool		 ForceLoopback	   = false;
 	bool		 IsDedicatedServer = false;
 	bool		 NoNetwork		   = false;
+	bool		 AllowPortProbing  = true;
 	std::string	 UnixSocketPath;
 #if ZEN_USE_OPENSSL
 	int			HttpsPort = 0;	// 0 = auto-assign; set CertFile/KeyFile to enable HTTPS
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index f8fb1c9be..9fe9a2254 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -1153,7 +1153,7 @@ HttpSysServer::RegisterHttpUrls(int BasePort)
 
 	m_BaseUris.clear();
 
-	const bool AllowPortProbing = !m_InitialConfig.IsDedicatedServer;
+	const bool AllowPortProbing = m_InitialConfig.AllowPortProbing;
 	const bool AllowLocalOnly	= !m_InitialConfig.IsDedicatedServer;
 
 	int EffectivePort = BasePort;
diff --git a/src/zenhttp/servers/httpsys.h b/src/zenhttp/servers/httpsys.h
index ca465ad00..88ec13834 100644
--- a/src/zenhttp/servers/httpsys.h
+++ b/src/zenhttp/servers/httpsys.h
@@ -22,6 +22,7 @@ struct HttpSysConfig
 	bool		 IsRequestLoggingEnabled = false;
 	bool		 IsDedicatedServer		 = false;
 	bool		 ForceLoopback			 = false;
+	bool		 AllowPortProbing		 = true;
 	int			 HttpsPort				 = 0;  // 0 = HTTPS disabled
 	std::string	 CertThumbprint;			   // Hex SHA-1 (40 chars) for auto SSL binding
 	std::string	 CertStoreName = "MY";		   // Windows certificate store name
diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp
index 9c1669256..958a0b050 100644
--- a/src/zenserver-test/hub-tests.cpp
+++ b/src/zenserver-test/hub-tests.cpp
@@ -47,7 +47,7 @@ TEST_CASE("hub.lifecycle.children")
 {
 	ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer);
 
-	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady();
+	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady("--hub-instance-corelimit=2 --hub-instance-http-threads=6");
 	REQUIRE(PortNumber != 0);
 
 	SUBCASE("spawn")
diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp
index 0d8550c5b..724ef9ad2 100644
--- a/src/zenserver/compute/computeserver.cpp
+++ b/src/zenserver/compute/computeserver.cpp
@@ -951,6 +951,7 @@ ZenComputeServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
 	Server.SetContentRoot(m_ServerOptions.ContentDir);
 	Server.SetTestMode(m_ServerOptions.IsTest);
 	Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+	Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing);
 
 	const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
 	if (EffectiveBasePort == -1)
diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp
index 60ae93853..15f6f79f3 100644
--- a/src/zenserver/config/config.cpp
+++ b/src/zenserver/config/config.cpp
@@ -133,6 +133,7 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions
 	// server
 
 	LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv);
+	LuaOptions.AddOption("server.allowportprobing"sv, ServerOptions.AllowPortProbing, "allow-port-probing"sv);
 	LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv);
 	LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
 	LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
@@ -223,8 +224,11 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi
 #endif
 
 	options.add_options()("dedicated",
-						  "Enable dedicated server mode",
+						  "Enable dedicated server mode, disables '--allow-port-probing' and allocates more resources.",
 						  cxxopts::value<bool>(ServerOptions.IsDedicated)->default_value("false"));
+	options.add_options()("allow-port-probing",
+						  "Allow searching for an available port, disabled if '--dedicated' is enabled.",
+						  cxxopts::value<bool>(ServerOptions.AllowPortProbing)->default_value("true"));
 	options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(ServerOptions.IsDebug)->default_value("false"));
 	options.add_options()("clean",
 						  "Clean out all state at startup",
@@ -691,6 +695,7 @@ ZenServerConfiguratorBase::Configure(int argc, char* argv[])
 	}
 
 	m_ServerOptions.HttpConfig.IsDedicatedServer = m_ServerOptions.IsDedicated;
+	m_ServerOptions.HttpConfig.AllowPortProbing	 = !m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing;
 }
 
 void
diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h
index e481c7225..5078fe71a 100644
--- a/src/zenserver/config/config.h
+++ b/src/zenserver/config/config.h
@@ -61,6 +61,7 @@ struct ZenServerConfig
 	int					  CoreLimit		   = 0;		// If set, hardware concurrency queries are capped at this number
 	int					  LieCpu		   = 0;
 	bool				  IsDedicated	   = false;	 // Indicates a dedicated/shared instance, with larger resource requirements
+	bool				  AllowPortProbing = true;	 // Automatically false if IsDedicated is true
 	bool				  ShouldCrash	   = false;	 // Option for testing crash handling
 	bool				  IsFirstRun	   = false;
 	std::filesystem::path ConfigFile;		   // Path to Lua config file
diff --git a/src/zenserver/hub/hub.cpp b/src/zenserver/hub/hub.cpp
index c9720b32d..3c9f40eaa 100644
--- a/src/zenserver/hub/hub.cpp
+++ b/src/zenserver/hub/hub.cpp
@@ -22,6 +22,8 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #	include <zencore/workthreadpool.h>
 #endif
 
+#include <numeric>
+
 namespace zen {
 
 ///////////////////////////////////////////////////////////////////////////
@@ -137,10 +139,10 @@ Hub::Hub(const Configuration&		   Config,
 	m_HydrationTempPath = m_RunEnvironment.CreateChildDir("hydration_temp");
 	ZEN_INFO("using hydration temp path: '{}'", m_HydrationTempPath);
 
-	// This is necessary to ensure the hub assigns a distinct port range.
-	// We need to do this primarily because otherwise automated tests will
-	// fail as the test runner will create processes in the default range.
-	m_RunEnvironment.SetNextPortNumber(m_Config.BasePortNumber);
+	ZEN_ASSERT(uint64_t(Config.BasePortNumber) + Config.InstanceLimit <= std::numeric_limits<uint16_t>::max());
+
+	m_FreePorts.resize(Config.InstanceLimit);
+	std::iota(m_FreePorts.begin(), m_FreePorts.end(), Config.BasePortNumber);
 
 #if ZEN_PLATFORM_WINDOWS
 	if (m_Config.UseJobObject)
@@ -199,6 +201,15 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 	bool				   IsNewInstance = false;
 	{
 		RwLock::ExclusiveLockScope _(m_Lock);
+		uint16_t				   AllocatedPort		= 0;
+		auto					   RestoreAllocatedPort = MakeGuard([this, &AllocatedPort]() {
+			  if (AllocatedPort != 0)
+			  {
+				  m_FreePorts.push_back(AllocatedPort);
+				  AllocatedPort = 0;
+			  }
+		  });
+
 		if (auto It = m_Instances.find(std::string(ModuleId)); It == m_Instances.end())
 		{
 			std::string Reason;
@@ -211,9 +222,18 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 				return false;
 			}
 
-			IsNewInstance = true;
-			auto NewInstance =
-				std::make_unique<StorageServerInstance>(m_RunEnvironment, ModuleId, m_FileHydrationPath, m_HydrationTempPath);
+			AllocatedPort = m_FreePorts.front();
+			m_FreePorts.pop_front();
+
+			IsNewInstance	 = true;
+			auto NewInstance = std::make_unique<StorageServerInstance>(
+				m_RunEnvironment,
+				StorageServerInstance::Configuration{.BasePort			= AllocatedPort,
+													 .HydrationTempPath = m_HydrationTempPath,
+													 .FileHydrationPath = m_FileHydrationPath,
+													 .HttpThreadCount	= m_Config.InstanceHttpThreadCount,
+													 .CoreLimit			= m_Config.InstanceCoreLimit},
+				ModuleId);
 #if ZEN_PLATFORM_WINDOWS
 			if (m_JobObject.IsValid())
 			{
@@ -222,6 +242,7 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 #endif
 			Instance = NewInstance.get();
 			m_Instances.emplace(std::string(ModuleId), std::move(NewInstance));
+			AllocatedPort = 0;
 
 			ZEN_INFO("Created new storage server instance for module '{}'", ModuleId);
 		}
@@ -258,7 +279,13 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 		{
 			// Clean up
 			RwLock::ExclusiveLockScope _(m_Lock);
-			m_Instances.erase(std::string(ModuleId));
+			if (auto It = m_Instances.find(std::string(ModuleId)); It != m_Instances.end())
+			{
+				ZEN_ASSERT(It->second != nullptr);
+				uint16_t BasePort = It->second->GetBasePort();
+				m_FreePorts.push_back(BasePort);
+				m_Instances.erase(It);
+			}
 		}
 		return false;
 	}
@@ -337,6 +364,7 @@ Hub::Deprovision(const std::string& ModuleId, std::string& OutReason)
 	auto _ = MakeGuard([&] {
 		RwLock::ExclusiveLockScope _(m_Lock);
 		m_DeprovisioningModules.erase(ModuleId);
+		m_FreePorts.push_back(BasePort);
 	});
 
 	Instance->Deprovision();
@@ -413,7 +441,17 @@ Hub::CanProvisionInstance(std::string_view ModuleId, std::string& OutReason)
 
 	if (gsl::narrow_cast<int>(m_Instances.size()) >= m_Config.InstanceLimit)
 	{
-		OutReason = fmt::format("instance limit exceeded ({})", m_Config.InstanceLimit);
+		OutReason = fmt::format("instance limit ({}) exceeded", m_Config.InstanceLimit);
+
+		return false;
+	}
+
+	// Since deprovisioning happens outside the lock and we don't add the port back until the instance is full shut down we might be under
+	// the instance limit but all ports may be in use
+	if (m_FreePorts.empty())
+	{
+		OutReason = fmt::format("no free ports available, deprovisioning of instances might be in flight ({})",
+								m_Config.InstanceLimit - m_Instances.size());
 
 		return false;
 	}
diff --git a/src/zenserver/hub/hub.h b/src/zenserver/hub/hub.h
index 8b61f988d..8a84a558b 100644
--- a/src/zenserver/hub/hub.h
+++ b/src/zenserver/hub/hub.h
@@ -7,6 +7,7 @@
 #include <zencore/system.h>
 #include <zenutil/zenserverprocess.h>
 
+#include <deque>
 #include <filesystem>
 #include <functional>
 #include <memory>
@@ -43,6 +44,9 @@ public:
 		uint16_t BasePortNumber = 21000;
 
 		int InstanceLimit = 1000;
+
+		uint32_t InstanceHttpThreadCount = 0;  // Deduce from core count
+		int		 InstanceCoreLimit		 = 0;  // Use hardware core count
 	};
 
 	typedef std::function<void(std::string_view ModuleId, const HubProvisionedInstanceInfo& Info)> ProvisionModuleCallbackFunc;
@@ -121,6 +125,7 @@ private:
 	ResourceMetrics															m_ResourceLimits;
 	SystemMetrics															m_HostMetrics;
 	int																		m_MaxInstanceCount = 0;
+	std::deque<uint16_t>													m_FreePorts;
 
 	void UpdateStats();
 	void UpdateCapacityMetrics();
diff --git a/src/zenserver/hub/storageserverinstance.cpp b/src/zenserver/hub/storageserverinstance.cpp
index f24379715..68de5e274 100644
--- a/src/zenserver/hub/storageserverinstance.cpp
+++ b/src/zenserver/hub/storageserverinstance.cpp
@@ -11,16 +11,13 @@
 
 namespace zen {
 
-StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment,
-											 std::string_view	   ModuleId,
-											 std::filesystem::path FileHydrationPath,
-											 std::filesystem::path HydrationTempPath)
-: m_ModuleId(ModuleId)
+StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment, const Configuration& Config, std::string_view ModuleId)
+: m_Config(Config)
+, m_ModuleId(ModuleId)
 , m_ServerInstance(RunEnvironment, ZenServerInstance::ServerMode::kStorageServer)
-, m_HydrationPath(FileHydrationPath)
 {
 	m_BaseDir = RunEnvironment.CreateChildDir(ModuleId);
-	m_TempDir = HydrationTempPath / ModuleId;
+	m_TempDir = Config.HydrationTempPath / ModuleId;
 }
 
 StorageServerInstance::~StorageServerInstance()
@@ -37,9 +34,20 @@ StorageServerInstance::SpawnServerProcess()
 #if ZEN_PLATFORM_WINDOWS
 	m_ServerInstance.SetJobObject(m_JobObject);
 #endif
-	const uint16_t BasePort = m_ServerInstance.SpawnServerAndWaitUntilReady();
 
-	ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, BasePort);
+	ExtendableStringBuilder<256> AdditionalOptions;
+	AdditionalOptions << "--allow-port-probing=false";
+	if (m_Config.HttpThreadCount != 0)
+	{
+		AdditionalOptions << " --http-threads=" << m_Config.HttpThreadCount;
+	}
+	if (m_Config.CoreLimit != 0)
+	{
+		AdditionalOptions << " --corelimit=" << m_Config.CoreLimit;
+	}
+
+	m_ServerInstance.SpawnServerAndWaitUntilReady(m_Config.BasePort, AdditionalOptions.ToView());
+	ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, m_Config.BasePort);
 
 	m_ServerInstance.EnableShutdownOnDestroy();
 }
@@ -178,7 +186,7 @@ StorageServerInstance::Hydrate()
 	HydrationConfig Config{.ServerStateDir		= m_BaseDir,
 						   .TempDir				= m_TempDir,
 						   .ModuleId			= m_ModuleId,
-						   .TargetSpecification = WideToUtf8(m_HydrationPath.native())};
+						   .TargetSpecification = WideToUtf8(m_Config.FileHydrationPath.native())};
 
 	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator();
 
@@ -192,7 +200,7 @@ StorageServerInstance::Dehydrate()
 	HydrationConfig Config{.ServerStateDir		= m_BaseDir,
 						   .TempDir				= m_TempDir,
 						   .ModuleId			= m_ModuleId,
-						   .TargetSpecification = WideToUtf8(m_HydrationPath.native())};
+						   .TargetSpecification = WideToUtf8(m_Config.FileHydrationPath.native())};
 
 	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator();
 
diff --git a/src/zenserver/hub/storageserverinstance.h b/src/zenserver/hub/storageserverinstance.h
index a2f3d25d7..23196d835 100644
--- a/src/zenserver/hub/storageserverinstance.h
+++ b/src/zenserver/hub/storageserverinstance.h
@@ -21,10 +21,16 @@ namespace zen {
 class StorageServerInstance
 {
 public:
-	StorageServerInstance(ZenServerEnvironment& RunEnvironment,
-						  std::string_view		ModuleId,
-						  std::filesystem::path FileHydrationPath,
-						  std::filesystem::path HydrationTempPath);
+	struct Configuration
+	{
+		uint16_t			  BasePort;
+		std::filesystem::path HydrationTempPath;
+		std::filesystem::path FileHydrationPath;
+		uint32_t			  HttpThreadCount = 0;	// Deduce from core count
+		int					  CoreLimit		  = 0;	// Use hardware core count
+	};
+
+	StorageServerInstance(ZenServerEnvironment& RunEnvironment, const Configuration& Config, std::string_view ModuleId);
 	~StorageServerInstance();
 
 	void Provision();
@@ -45,15 +51,17 @@ public:
 #endif
 
 private:
-	void				  WakeLocked();
-	RwLock				  m_Lock;
-	std::string			  m_ModuleId;
+	void				WakeLocked();
+	RwLock				m_Lock;
+	const Configuration m_Config;
+	std::string			m_ModuleId;
+	ZenServerInstance	m_ServerInstance;
+
 	std::atomic<bool>	  m_IsProvisioned{false};
 	std::atomic<bool>	  m_IsHibernated{false};
-	ZenServerInstance	  m_ServerInstance;
 	std::filesystem::path m_BaseDir;
+
 	std::filesystem::path m_TempDir;
-	std::filesystem::path m_HydrationPath;
 	ResourceMetrics		  m_ResourceMetrics;
 #if ZEN_PLATFORM_WINDOWS
 	JobObject* m_JobObject = nullptr;
diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
index b0ae0a8b1..313be977c 100644
--- a/src/zenserver/hub/zenhubserver.cpp
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -26,6 +26,15 @@ namespace zen {
 void
 ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options)
 {
+	const char* DefaultInstanceHttp = "asio";
+
+#if ZEN_WITH_HTTPSYS
+	if (!windows::IsRunningOnWine())
+	{
+		DefaultInstanceHttp = "httpsys";
+	}
+#endif
+
 	Options.add_option("hub",
 					   "",
 					   "upstream-notification-endpoint",
@@ -60,6 +69,31 @@ ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options)
 					   "Maximum number of provisioned instances for this hub",
 					   cxxopts::value<int>(m_ServerOptions.HubInstanceLimit)->default_value("1000"),
 					   "");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-instance-http",
+					   "Select HTTP server implementation for provisioned instances (asio|"
+#if ZEN_WITH_HTTPSYS
+					   "httpsys|"
+#endif
+					   "null)",
+					   cxxopts::value<std::string>(m_ServerOptions.HubInstanceHttpClass)->default_value(DefaultInstanceHttp),
+					   "<instance http class>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-instance-http-threads",
+					   "Number of http server connection threads for provisioned instances",
+					   cxxopts::value<unsigned int>(m_ServerOptions.HubInstanceHttpThreadCount),
+					   "<instance http threads>");
+	Options.add_option("hub",
+					   "",
+					   "hub-instance-corelimit",
+					   "Limit concurrency of provisioned instances",
+					   cxxopts::value(m_ServerOptions.HubInstanceCoreLimit),
+					   "<instance core limit>");
+
 #if ZEN_PLATFORM_WINDOWS
 	Options.add_option("hub",
 					   "",
@@ -231,10 +265,15 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig)
 
 	ZEN_INFO("instantiating Hub");
 	m_Hub = std::make_unique<Hub>(
-		Hub::Configuration{.UseJobObject   = ServerConfig.HubUseJobObject,
-						   .BasePortNumber = ServerConfig.HubBasePortNumber,
-						   .InstanceLimit  = ServerConfig.HubInstanceLimit},
-		ZenServerEnvironment(ZenServerEnvironment::Hub, ServerConfig.DataDir / "hub", ServerConfig.DataDir / "servers"),
+		Hub::Configuration{.UseJobObject			= ServerConfig.HubUseJobObject,
+						   .BasePortNumber			= ServerConfig.HubBasePortNumber,
+						   .InstanceLimit			= ServerConfig.HubInstanceLimit,
+						   .InstanceHttpThreadCount = ServerConfig.HubInstanceHttpThreadCount,
+						   .InstanceCoreLimit		= ServerConfig.HubInstanceCoreLimit},
+		ZenServerEnvironment(ZenServerEnvironment::Hub,
+							 ServerConfig.DataDir / "hub",
+							 ServerConfig.DataDir / "servers",
+							 ServerConfig.HubInstanceHttpClass),
 		m_ConsulClient ? [this, HubInstanceId = fmt::format("zen-hub-{}", ServerConfig.InstanceId)](
 							 std::string_view				   ModuleId,
 							 const HubProvisionedInstanceInfo& Info) { OnProvisioned(HubInstanceId, ModuleId, Info); }
@@ -405,6 +444,7 @@ ZenHubServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
 	Server.SetContentRoot(m_ServerOptions.ContentDir);
 	Server.SetTestMode(m_ServerOptions.IsTest);
 	Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+	Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing);
 
 	const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
 	if (EffectiveBasePort == -1)
diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h
index f6a3eb1bc..1036598bb 100644
--- a/src/zenserver/hub/zenhubserver.h
+++ b/src/zenserver/hub/zenhubserver.h
@@ -24,9 +24,12 @@ struct ZenHubServerConfig : public ZenServerConfig
 	std::string UpstreamNotificationEndpoint;
 	std::string InstanceId;		 // For use in notifications
 	std::string ConsulEndpoint;	 // If set, enables Consul service registration
-	uint16_t	HubBasePortNumber = 21000;
-	int			HubInstanceLimit  = 1000;
-	bool		HubUseJobObject	  = true;
+	uint16_t	HubBasePortNumber		   = 21000;
+	int			HubInstanceLimit		   = 1000;
+	bool		HubUseJobObject			   = true;
+	std::string HubInstanceHttpClass	   = "asio";
+	uint32_t	HubInstanceHttpThreadCount = 0;	 // Deduce from core count
+	int			HubInstanceCoreLimit	   = 0;	 // Use hardware core count
 };
 
 class Hub;
@@ -79,8 +82,6 @@ public:
 	void Run();
 	void Cleanup();
 
-	void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; }
-	void SetTestMode(bool State) { m_TestMode = State; }
 	void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; }
 	void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; }
 
@@ -88,8 +89,6 @@ private:
 	void OnProvisioned(std::string_view HubInstanceId, std::string_view ModuleId, const HubProvisionedInstanceInfo& Info);
 	void OnDeprovisioned(std::string_view HubInstanceId, std::string_view ModuleId, const HubProvisionedInstanceInfo& Info);
 
-	bool				  m_IsDedicatedMode = false;
-	bool				  m_TestMode		= false;
 	std::filesystem::path m_DataRoot;
 	std::filesystem::path m_ContentRoot;
 	bool				  m_DebugOptionForcedCrash = false;
diff --git a/src/zenserver/proxy/zenproxyserver.cpp b/src/zenserver/proxy/zenproxyserver.cpp
index c768e940a..cf84c159a 100644
--- a/src/zenserver/proxy/zenproxyserver.cpp
+++ b/src/zenserver/proxy/zenproxyserver.cpp
@@ -454,6 +454,7 @@ ZenProxyServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
 	Server.SetContentRoot(m_ServerOptions.ContentDir);
 	Server.SetTestMode(m_ServerOptions.IsTest);
 	Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+	Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing);
 
 	const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
 	if (EffectiveBasePort == -1)
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index bba5e0a61..f5ede5692 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -975,6 +975,7 @@ ZenStorageServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
 	Server.SetContentRoot(m_ServerOptions.ContentDir);
 	Server.SetTestMode(m_ServerOptions.IsTest);
 	Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+	Server.SetAllowPortProbing(!m_ServerOptions.IsDedicated && m_ServerOptions.AllowPortProbing);
 
 	int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
 	if (EffectiveBasePort == -1)
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index 6b29dadfb..fe279ebb2 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -39,11 +39,11 @@ target("zenserver")
     add_packages("oidctoken")
     add_packages("nomad")
 
-    if has_config("zenmimalloc") then
+    if has_config("zenmimalloc") and not use_asan then
         add_packages("mimalloc")
     end
 
-    if has_config("zensentry") then
+    if has_config("zensentry") and not use_asan then
         add_packages("sentry-native")
     end
 
@@ -140,11 +140,19 @@ target("zenserver")
 
         table.insert(args, "--detach=false")
 
+        -- On Windows the ASAN runtime is a DLL whose directory must be on PATH;
+        -- runenvs.make collects that (and any other Windows package run envs).
+        local addenvs, setenvs
+        if is_host("windows") then
+            import("private.action.run.runenvs")
+            addenvs, setenvs = runenvs.make(target)
+        end
+
         -- debugging?
         if option.get("debug") then
-            debugger.run(targetfile, args, {curdir = rundir})
+            debugger.run(targetfile, args, {curdir = rundir, addenvs = addenvs, setenvs = setenvs})
         else
-            os.execv(targetfile, args, {curdir = rundir, detach = option.get("detach")})
+            os.execv(targetfile, args, {curdir = rundir, detach = option.get("detach"), addenvs = addenvs, setenvs = setenvs})
         end
     end)
 
@@ -175,7 +183,7 @@ target("zenserver")
             end
         end
 
-        if has_config("zensentry") then
+        if has_config("zensentry") and not target:policy("build.sanitizer.address") then
             local pkg = target:pkg("sentry-native")
             if pkg then
                 local installdir = pkg:installdir()
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 6760e0372..1cd8ed846 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -517,6 +517,7 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig)
 	});
 	// clang-format on
 	Settings.emplace_back("IsDedicated"sv, fmt::to_string(ServerConfig.IsDedicated));
+	Settings.emplace_back("AllowPortProbing"sv, fmt::to_string(ServerConfig.AllowPortProbing));
 	Settings.emplace_back("ShouldCrash"sv, fmt::to_string(ServerConfig.ShouldCrash));
 
 	size_t MaxWidth = 0;
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 830f36e54..d6bf4454f 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -48,6 +48,7 @@ public:
 	void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; }
 	void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; }
 	void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; }
+	void SetAllowPortProbing(bool State) { m_AllowPortProbing = State; }
 	void SetServerMode(std::string_view Mode) { m_ServerMode = Mode; }
 	void SetTestMode(bool State) { m_TestMode = State; }
 
@@ -66,6 +67,7 @@ protected:
 	bool							m_IsPowerCycle = false;
 
 	bool		m_IsDedicatedMode		 = false;
+	bool		m_AllowPortProbing		 = true;
 	bool		m_TestMode				 = false;
 	bool		m_NoNetwork				 = false;
 	bool		m_DebugOptionForcedCrash = false;
diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h
index 2f76f0d6c..101b078e9 100644
--- a/src/zenutil/include/zenutil/zenserverprocess.h
+++ b/src/zenutil/include/zenutil/zenserverprocess.h
@@ -151,6 +151,11 @@ struct ZenServerInstance
 		return GetBasePort();
 	}
 
+	inline void SpawnServerAndWaitUntilReady(uint16_t BasePort, std::string_view AdditionalServerArgs = std::string_view())
+	{
+		SpawnServer(BasePort, AdditionalServerArgs, /* WaitTimeoutMs */ 100'000);
+	}
+
 	inline void SpawnServer(int BasePort, std::string_view AdditionalServerArgs = std::string_view())
 	{
 		SpawnServer(BasePort, AdditionalServerArgs, /* WaitTimeoutMs */ 0);
diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua
index e8832d50a..7377cf0f6 100644
--- a/thirdparty/xmake.lua
+++ b/thirdparty/xmake.lua
@@ -38,6 +38,7 @@ target('rpmalloc')
     set_languages("c17", "cxx20")
     if is_os("windows") and not (get_config("toolchain") or ""):find("clang") then
         add_cflags("/experimental:c11atomics", {force=true, tools="cl"})
+        add_cflags("/wd5105", {tools="cl"})
     end
     add_defines("RPMALLOC_FIRST_CLASS_HEAPS=1", "ENABLE_STATISTICS=1", "ENABLE_OVERRIDE=0")
     add_files("rpmalloc/rpmalloc.c")
diff --git a/tsan.supp b/tsan.supp
index 93398da67..6dfe4e393 100644
--- a/tsan.supp
+++ b/tsan.supp
@@ -1,17 +1,42 @@
-# TSAN suppression / options file for zenserver
+# ThreadSanitizer options and suppression patterns for Zen
 #
-# Usage:
-#   TSAN_OPTIONS="detect_deadlocks=0 suppressions=$(pwd)/tsan.supp" ./zenserver
+# This is a mixed-format file:
 #
-# NOTE: detect_deadlocks=0 is required because the GC's LockState() acquires shared
-# lock scopes on every named cache bucket (m_IndexLock) and every oplog
-# (GcReferenceLocker) simultaneously. With enough buckets/projects/oplogs this
-# easily exceeds TSAN's hard per-thread limit of 128 simultaneously-held locks
-# (all_locks_with_contexts_[128] in sanitizer_deadlock_detector.h:67), causing a
-# CHECK abort. This is a known TSAN limitation, not a real deadlock risk.
-# The long-term fix is to replace the N per-bucket shared-lock pattern in
-# ZenCacheStore::LockState / ProjectStore::LockState with a single coarser
-# "GC epoch" RwLock at the disk-layer / project-store level.
+#   key=value    TSAN runtime option; parsed by xmake's sanitizer.options
+#                rule and set directly in TSAN_OPTIONS. When this file is also
+#                passed as suppressions=<path>, the TSAN runtime silently skips
+#                these lines (they don't match the type:pattern format).
+#   race:...     race condition suppression pattern
+#   mutex:...    mutex suppression pattern
+#   signal:...   signal suppression pattern
+#   thread:...   thread suppression pattern
+#   #            comment line
+#   <blank>      ignored
+#
+# xmake passes the file via TSAN_OPTIONS=suppressions=<path> when both the
+# compiler is Clang or GCC (not MSVC/clang-cl) and the platform is Linux or
+# macOS. This is handled by the sanitizer.options rule in xmake.lua.
+#
+# If you run a binary directly (not via xmake run / xmake test) set
+# TSAN_OPTIONS manually, e.g.:
+#   TSAN_OPTIONS="detect_deadlocks=0:suppressions=$(pwd)/tsan.supp" ./zenserver
+
+# Required because GC's LockState() acquires shared lock scopes on every named
+# cache bucket (m_IndexLock) and every oplog (GcReferenceLocker) simultaneously.
+# With enough buckets/projects/oplogs this easily exceeds TSAN's hard per-thread
+# limit of 128 simultaneously-held locks, causing a CHECK abort. This is a known
+# TSAN limitation, not a real deadlock risk. The long-term fix is to replace the
+# N per-bucket shared-lock pattern in ZenCacheStore::LockState /
+# ProjectStore::LockState with a single coarser "GC epoch" RwLock.
+#
+# NOTE: this line will produce a TSAN warning along the lines of:
+#   WARNING: failed to parse suppression 'detect_deadlocks=0'
+# This is expected and harmless. xmake's sanitizer.options rule parses
+# key=value lines from this file and injects them into TSAN_OPTIONS directly.
+# When the file is subsequently passed to the TSAN runtime as suppressions=<path>,
+# the runtime does not understand key=value syntax, prints the warning, and skips
+# the line. The option is already active via TSAN_OPTIONS; nothing is lost.
+detect_deadlocks=0
 
 # EASTL's hashtable uses a global gpEmptyBucketArray[2] sentinel shared by all
 # empty hash tables (mnBucketCount == 1). DoFreeNodes unconditionally writes NULL
diff --git a/xmake.lua b/xmake.lua
index 846bc8f1d..d40131e35 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -19,6 +19,13 @@ end
 set_allowedmodes("debug", "release")
 add_rules("mode.debug", "mode.release")
 
+-- Disable xmake's run.autobuild (on by default in xmake 3.x). Auto-rebuild
+-- before run is dangerous: if the current config differs from the build that
+-- produced the binary (e.g. ASAN was toggled), xmake can rebuild with wrong
+-- flags, causing linker errors (e.g. MSVC annotate_string mismatch). Users
+-- are expected to run `xmake build` explicitly before `xmake run`.
+set_policy("run.autobuild", false)
+
 if is_plat("windows") then
     if false then
         -- DLL runtime
@@ -37,44 +44,161 @@ if is_plat("windows") then
     end
 end
 
---------------------------------------------------------------------------
 -- Sanitizers
 --
 -- https://xmake.io/api/description/builtin-policies.html#build-sanitizer-address
 --
--- When using sanitizers, it may be necessary to change some configuration
--- options. In particular, you may want to use `--zensentry=no` to disable
--- Sentry support as it may not be compatible with some sanitizers. Also,
--- it may be necessary to disable mimalloc by using `--zenmimalloc=no`.
-
--- AddressSanitizer is supported on Windows (MSVC 2019+), Linux, and MacOS
+-- Each sanitizer has a .supp file (asan.supp, tsan.supp, msan.supp). A single
+-- sanitizer.options rule handles all three in one before_run callback.
+--
+-- asan.supp and msan.supp contain key=value runtime options (no native
+-- "options from file" mechanism exists for either sanitizer on any platform).
+--
+-- tsan.supp is a mixed-format file: key=value runtime options (parsed by xmake
+-- and set in TSAN_OPTIONS) plus native TSAN suppression patterns (race:...,
+-- mutex:...) that are passed via TSAN_OPTIONS=suppressions=<path> on Linux/Mac.
+-- The TSAN runtime silently skips key=value lines when parsing the suppression
+-- file (they don't match type:pattern; TSAN emits a verbose-level warning).
+--
+-- IMPORTANT: The env vars are set by xmake's before_run hook. If you run a
+-- binary directly (not via xmake run / xmake test) you must set them manually.
 
-local use_asan = false       -- Automatically disables Sentry when set to true
-set_policy("build.sanitizer.address", use_asan)
+--------------------------------------------------------------------------
 
--- ThreadSanitizer, MemorySanitizer, LeakSanitizer, and UndefinedBehaviorSanitizer
--- are supported on Linux and MacOS only.
+-- AddressSanitizer is supported on Windows (MSVC 2019+), Linux, and MacOS.
+-- Enable with: `xmake config --asan=y` or `xmake test --asan`
+-- (automatically disables mimalloc and sentry, which are incompatible with ASAN)
 --
--- You can enable these by editing the xmake.lua directly, or by passing the 
--- appropriate flags on the command line:
+-- Options live in asan.supp. For manual runs:
+--   Linux/Mac:  ASAN_OPTIONS="new_delete_type_mismatch=0" ./zenserver
+--   Windows:    set ASAN_OPTIONS=new_delete_type_mismatch=0
+
+option("asan")
+    set_default(false)
+    set_showmenu(true)
+    set_description("Enable AddressSanitizer (disables mimalloc and sentry)")
+option_end()
+
+-- Global flag: read by sub-target xmake.lua files (like enable_unity)
+use_asan = has_config("asan")
+
+if use_asan then
+    set_policy("build.sanitizer.address", true)
+    -- On Windows/MSVC, explicitly define __SANITIZE_ADDRESS__ so that
+    -- IntelliSense in VS sees it (XmakeDefines -> PreprocessorDefinitions).
+    -- MSVC sets it implicitly at compile time via /fsanitize=address, but
+    -- vsxmake IntelliSense needs it in the explicit defines list.
+    -- Not needed on Clang/GCC: they set it implicitly and adding it via -D
+    -- would cause a macro-redefinition warning (error with -Werror).
+    if is_os("windows") then
+        add_defines("__SANITIZE_ADDRESS__=1")
+    end
+end
+
+--------------------------------------------------------------------------
+
+-- ThreadSanitizer is supported on Linux and MacOS only.
+-- Enable with: `xmake config --tsan=y` or `xmake test --tsan`
 --
--- `xmake --policies=build.sanitizer.thread:y` for ThreadSanitizer, 
--- `xmake --policies=build.sanitizer.memory:y` for MemorySanitizer, etc.
+-- Key=value options and suppression patterns both live in tsan.supp. For
+-- manual runs: TSAN_OPTIONS="detect_deadlocks=0:suppressions=$(pwd)/tsan.supp"
+
+option("tsan")
+    set_default(false)
+    set_showmenu(true)
+    set_description("Enable ThreadSanitizer (Linux/Mac only)")
+option_end()
 
--- When using TSAN you will want to also use the suppression tile to silence
--- known benign races. You do this by ensuring the the TSAN_OPTIONS environment
--- vriable is set to something like `TSAN_OPTIONS="suppressions=$(projectdir)/tsan.supp"`
+use_tsan = has_config("tsan")
+
+if use_tsan then
+    if is_os("windows") then
+        raise("TSAN is not supported on Windows. Use Linux or macOS.")
+    end
+    set_policy("build.sanitizer.thread", true)
+end
+
+--------------------------------------------------------------------------
+
+-- MemorySanitizer is supported on Linux only. In practice it rarely works
+-- because all dependencies must also be compiled with MSan instrumentation.
+-- Enable with: `xmake config --msan=y` or `xmake test --msan`
 --
--- `prompt> TSAN_OPTIONS="detect_deadlocks=0 suppressions=$(projectdir)/tsan.supp" xmake run zenserver`
+-- Options live in msan.supp. For manual runs: set MSAN_OPTIONS manually.
+
+option("msan")
+    set_default(false)
+    set_showmenu(true)
+    set_description("Enable MemorySanitizer (Linux only, requires all deps instrumented)")
+option_end()
+
+use_msan = has_config("msan")
+
+if use_msan then
+    set_policy("build.sanitizer.memory", true)
+end
+
+--------------------------------------------------------------------------
+
+-- Single rule handles all three sanitizer env vars. parse_supp_opts is defined
+-- once inside before_run (scripting scope) where io is a valid global.
+
+rule("sanitizer.options")
+    before_run(function(target)
+        -- Parses key=value lines from a .supp file. Blank lines and lines whose
+        -- first non-whitespace character is '#' are ignored. Suppression-pattern
+        -- lines (race:..., mutex:...) don't match ^[%w_]+= and are skipped.
+        local function parse_supp_opts(filepath)
+            local opts = {}
+            local content = os.isfile(filepath) and io.readfile(filepath) or ""
+            for line in content:gmatch("[^\r\n]+") do
+                line = line:match("^%s*(.-)%s*$")
+                if line ~= "" and not line:match("^#") and line:match("^[%w_]+=") then
+                    table.insert(opts, line)
+                end
+            end
+            return opts
+        end
+
+        if target:policy("build.sanitizer.address") and not os.getenv("ASAN_OPTIONS") then
+            local opts = parse_supp_opts(path.join(os.projectdir(), "asan.supp"))
+            if #opts > 0 then
+                os.setenv("ASAN_OPTIONS", table.concat(opts, ":"))
+            end
+        end
+
+        if target:policy("build.sanitizer.thread") and not os.getenv("TSAN_OPTIONS") then
+            local suppfile = path.join(os.projectdir(), "tsan.supp")
+            local opts = parse_supp_opts(suppfile)
+            -- Also pass the suppression patterns via the native file mechanism when
+            -- both conditions hold:
+            --   1. Not clang-cl: file path support requires the LLVM/GCC TSAN runtime.
+            --   2. Platform is Linux or macOS: TSAN is unavailable on Windows
+            --      (config-time raise() above prevents reaching here on Windows).
+            local tc_name = get_config("toolchain") or ""
+            local is_not_clang_cl = tc_name ~= "clang-cl"
+            if os.isfile(suppfile) and is_not_clang_cl
+                    and (target:is_plat("linux") or target:is_plat("macosx")) then
+                table.insert(opts, "suppressions=" .. suppfile)
+            end
+            if #opts > 0 then
+                os.setenv("TSAN_OPTIONS", table.concat(opts, ":"))
+            end
+        end
+
+        if target:policy("build.sanitizer.memory") and not os.getenv("MSAN_OPTIONS") then
+            local opts = parse_supp_opts(path.join(os.projectdir(), "msan.supp"))
+            if #opts > 0 then
+                os.setenv("MSAN_OPTIONS", table.concat(opts, ":"))
+            end
+        end
+    end)
+rule_end()
+add_rules("sanitizer.options")
 
---set_policy("build.sanitizer.thread", true) 
 --set_policy("build.sanitizer.leak", true)
 --set_policy("build.sanitizer.undefined", true)
 
--- In practice, this does not work because of the difficulty of compiling 
--- dependencies with MemorySanitizer. 
---set_policy("build.sanitizer.memory", true) 
-
 --------------------------------------------------------------------------
 -- Dependencies
 
@@ -259,16 +383,16 @@ end
 option("zensentry")
     set_default(true)
     set_showmenu(true)
-    set_description("Enables Sentry support")
+    set_description("Enables Sentry support (incompatible with ASAN)")
 option_end()
-add_define_by_config("ZEN_USE_SENTRY", "zensentry")
+add_defines("ZEN_USE_SENTRY=" .. ((has_config("zensentry") and not use_asan) and "1" or "0"))
 
 option("zenmimalloc")
-    set_default(not use_asan)
+    set_default(true)
     set_showmenu(true)
-    set_description("Use MiMalloc for faster memory management")
+    set_description("Use MiMalloc for faster memory management (incompatible with ASAN)")
 option_end()
-add_define_by_config("ZEN_USE_MIMALLOC", "zenmimalloc")
+add_defines("ZEN_USE_MIMALLOC=" .. ((has_config("zenmimalloc") and not use_asan) and "1" or "0"))
 
 option("zenrpmalloc")
     set_default(true)
@@ -486,7 +610,10 @@ task("test")
             {'j', "junit", "k", nil, "Enable junit report output"},
             {'n', "noskip", "k", nil, "Run skipped tests (passes --no-skip to doctest)"},
             {nil, "repeat", "kv", nil, "Repeat tests N times (stops on first failure)"},
-            {'v', "verbose", "k", nil, "Route child process output to stdout (zenserver-test)"},
+            {'V', "output", "k", nil, "Route child process output to stdout (zenserver-test)"},
+            {nil, "asan", "k", nil, "Enable AddressSanitizer (disables mimalloc and sentry)"},
+            {nil, "tsan", "k", nil, "Enable ThreadSanitizer (Linux/Mac only)"},
+            {nil, "msan", "k", nil, "Enable MemorySanitizer (Linux only, requires all deps instrumented)"},
             {nil, "arguments", "vs", nil, "Extra arguments passed to test runners (after --)"}
         }
     }