use dynamic port assignment for tests (#545)

this change replaces hard-coded port numbers in tests with dynamically assigned ports, to avoid potential issues around socket lifetimes and re-use policies

3 files changed, 216 insertions, 215 deletions

diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 6aa58ee14..3cf6e2976 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -153,11 +153,7 @@ TEST_CASE("default.single")
 
 	ZenServerInstance Instance(TestEnv);
 	Instance.SetTestDir(TestDir);
-	Instance.SpawnServer(13337);
-
-	ZEN_INFO("Waiting...");
-
-	Instance.WaitUntilReady();
+	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady();
 
 	std::atomic<uint64_t> RequestCount{0};
 	std::atomic<uint64_t> BatchCounter{0};
@@ -170,7 +166,7 @@ TEST_CASE("default.single")
 
 		ZEN_INFO("query batch {} started (thread {})", BatchNo, ThreadId);
 		cpr::Session cli;
-		cli.SetUrl(cpr::Url{"http://localhost:13337/test/hello"});
+		cli.SetUrl(cpr::Url{fmt::format("http://localhost:{}/test/hello", PortNumber)});
 
 		for (int i = 0; i < 10000; ++i)
 		{
@@ -206,17 +202,17 @@ TEST_CASE("multi.basic")
 	ZenServerInstance	  Instance1(TestEnv);
 	std::filesystem::path TestDir1 = TestEnv.CreateNewTestDir();
 	Instance1.SetTestDir(TestDir1);
-	Instance1.SpawnServer(13337);
+	Instance1.SpawnServer();
 
 	ZenServerInstance	  Instance2(TestEnv);
 	std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir();
 	Instance2.SetTestDir(TestDir2);
-	Instance2.SpawnServer(13338);
+	Instance2.SpawnServer();
 
 	ZEN_INFO("Waiting...");
 
-	Instance1.WaitUntilReady();
-	Instance2.WaitUntilReady();
+	const uint16_t PortNum1 = Instance1.WaitUntilReady();
+	const uint16_t PortNum2 = Instance2.WaitUntilReady();
 
 	std::atomic<uint64_t> RequestCount{0};
 	std::atomic<uint64_t> BatchCounter{0};
@@ -242,10 +238,10 @@ TEST_CASE("multi.basic")
 
 	ZEN_INFO("Running multi-server test...");
 
-	Concurrency::parallel_invoke([&] { IssueTestRequests(13337); },
-								 [&] { IssueTestRequests(13338); },
-								 [&] { IssueTestRequests(13337); },
-								 [&] { IssueTestRequests(13338); });
+	Concurrency::parallel_invoke([&] { IssueTestRequests(PortNum1); },
+								 [&] { IssueTestRequests(PortNum2); },
+								 [&] { IssueTestRequests(PortNum1); },
+								 [&] { IssueTestRequests(PortNum2); });
 
 	uint64_t Elapsed = timer.GetElapsedTimeMs();
 
@@ -261,12 +257,10 @@ TEST_CASE("project.basic")
 
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
-	const uint16_t PortNumber = 13337;
-
 	ZenServerInstance Instance1(TestEnv);
 	Instance1.SetTestDir(TestDir);
-	Instance1.SpawnServer(PortNumber);
-	Instance1.WaitUntilReady();
+
+	const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady();
 
 	std::atomic<uint64_t> RequestCount{0};
 
@@ -423,29 +417,6 @@ TEST_CASE("project.basic")
 			 zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req"));
 }
 
-#	if 0  // this is extremely WIP
-TEST_CASE("project.pipe")
-{
-	using namespace std::literals;
-
-	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
-
-	const uint16_t PortNumber = 13337;
-
-	ZenServerInstance Instance1(TestEnv);
-	Instance1.SetTestDir(TestDir);
-	Instance1.SpawnServer(PortNumber);
-	Instance1.WaitUntilReady();
-
-	zen::LocalProjectClient LocalClient(PortNumber);
-
-	zen::CbObjectWriter Cbow;
-	Cbow << "hey" << 42;
-
-	zen::CbObject Response = LocalClient.MessageTransaction(Cbow.Save());
-}
-#	endif
-
 namespace utils {
 
 	struct ZenConfig
@@ -455,43 +426,46 @@ namespace utils {
 		std::string			  BaseUri;
 		std::string			  Args;
 
-		static ZenConfig New(uint16_t Port = 13337, std::string Args = "")
+		static ZenConfig New(std::string Args = "")
+		{
+			return ZenConfig{.DataDir = TestEnv.CreateNewTestDir(), .Port = TestEnv.GetNewPortNumber(), .Args = std::move(Args)};
+		}
+
+		static ZenConfig New(uint16_t Port, std::string Args = "")
 		{
-			return ZenConfig{.DataDir = TestEnv.CreateNewTestDir(),
-							 .Port	  = Port,
-							 .BaseUri = fmt::format("http://localhost:{}/z$", Port),
-							 .Args	  = std::move(Args)};
+			return ZenConfig{.DataDir = TestEnv.CreateNewTestDir(), .Port = Port, .Args = std::move(Args)};
 		}
 
-		static ZenConfig NewWithUpstream(uint16_t UpstreamPort)
+		static ZenConfig NewWithUpstream(uint16_t Port, uint16_t UpstreamPort)
 		{
-			return New(13337, fmt::format("--debug --upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", UpstreamPort));
+			return New(Port, fmt::format("--debug --upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", UpstreamPort));
 		}
 
-		static ZenConfig NewWithThreadedUpstreams(std::span<uint16_t> UpstreamPorts, bool Debug)
+		static ZenConfig NewWithThreadedUpstreams(uint16_t NewPort, std::span<uint16_t> UpstreamPorts, bool Debug)
 		{
 			std::string Args = Debug ? "--debug" : "";
 			for (uint16_t Port : UpstreamPorts)
 			{
 				Args = fmt::format("{}{}--upstream-zen-url=http://localhost:{}", Args, Args.length() > 0 ? " " : "", Port);
 			}
-			return New(13337, Args);
+			return New(NewPort, Args);
 		}
 
 		void Spawn(ZenServerInstance& Inst)
 		{
 			Inst.SetTestDir(DataDir);
 			Inst.SpawnServer(Port, Args);
-			Inst.WaitUntilReady();
+			const uint16_t InstancePort = Inst.WaitUntilReady();
+
+			if (Port != InstancePort)
+				ZEN_DEBUG("relocation detected from {} to {}", Port, InstancePort);
+
+			Port	= InstancePort;
+			BaseUri = fmt::format("http://localhost:{}/z$", Port);
 		}
 	};
 
-	void SpawnServer(ZenServerInstance& Server, ZenConfig& Cfg)
-	{
-		Server.SetTestDir(Cfg.DataDir);
-		Server.SpawnServer(Cfg.Port, Cfg.Args);
-		Server.WaitUntilReady();
-	}
+	void SpawnServer(ZenServerInstance& Server, ZenConfig& Cfg) { Cfg.Spawn(Server); }
 
 }  // namespace utils
 
@@ -501,18 +475,16 @@ TEST_CASE("zcache.basic")
 
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
-	const uint16_t PortNumber = 13337;
-
-	const int  kIterationCount = 100;
-	const auto BaseUri		   = fmt::format("http://localhost:{}/z$", PortNumber);
+	const int kIterationCount = 100;
 
 	auto HashKey = [](int i) -> zen::IoHash { return zen::IoHash::HashBuffer(&i, sizeof i); };
 
 	{
 		ZenServerInstance Instance1(TestEnv);
 		Instance1.SetTestDir(TestDir);
-		Instance1.SpawnServer(PortNumber);
-		Instance1.WaitUntilReady();
+
+		const uint16_t	  PortNumber = Instance1.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
 
 		// Populate with some simple data
 
@@ -569,8 +541,9 @@ TEST_CASE("zcache.basic")
 	{
 		ZenServerInstance Instance1(TestEnv);
 		Instance1.SetTestDir(TestDir);
-		Instance1.SpawnServer(PortNumber);
-		Instance1.WaitUntilReady();
+		const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady();
+
+		const std::string BaseUri = fmt::format("http://localhost:{}/z$", PortNumber);
 
 		// Retrieve data again
 
@@ -647,14 +620,12 @@ TEST_CASE("zcache.cbpackage")
 
 	SUBCASE("PUT/GET returns correct package")
 	{
-		std::filesystem::path TestDir	 = TestEnv.CreateNewTestDir();
-		const uint16_t		  PortNumber = 13337;
-		const auto			  BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Instance1(TestEnv);
 		Instance1.SetTestDir(TestDir);
-		Instance1.SpawnServer(PortNumber);
-		Instance1.WaitUntilReady();
+		const uint16_t	  PortNumber = Instance1.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
 
 		const std::string_view Bucket = "mosdef"sv;
 		zen::IoHash			   Key;
@@ -687,24 +658,21 @@ TEST_CASE("zcache.cbpackage")
 	SUBCASE("PUT propagates upstream")
 	{
 		// Setup local and remote server
-		std::filesystem::path LocalDataDir	   = TestEnv.CreateNewTestDir();
-		std::filesystem::path RemoteDataDir	   = TestEnv.CreateNewTestDir();
-		const uint16_t		  LocalPortNumber  = 13337;
-		const uint16_t		  RemotePortNumber = 13338;
-
-		const auto LocalBaseUri	 = fmt::format("http://localhost:{}/z$", LocalPortNumber);
-		const auto RemoteBaseUri = fmt::format("http://localhost:{}/z$", RemotePortNumber);
+		std::filesystem::path LocalDataDir	= TestEnv.CreateNewTestDir();
+		std::filesystem::path RemoteDataDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance RemoteInstance(TestEnv);
 		RemoteInstance.SetTestDir(RemoteDataDir);
-		RemoteInstance.SpawnServer(RemotePortNumber);
-		RemoteInstance.WaitUntilReady();
+		const uint16_t RemotePortNumber = RemoteInstance.SpawnServerAndWaitUntilReady();
 
 		ZenServerInstance LocalInstance(TestEnv);
 		LocalInstance.SetTestDir(LocalDataDir);
-		LocalInstance.SpawnServer(LocalPortNumber,
+		LocalInstance.SpawnServer(TestEnv.GetNewPortNumber(),
 								  fmt::format("--upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", RemotePortNumber));
-		LocalInstance.WaitUntilReady();
+		const uint16_t LocalPortNumber = LocalInstance.WaitUntilReady();
+
+		const auto LocalBaseUri	 = fmt::format("http://localhost:{}/z$", LocalPortNumber);
+		const auto RemoteBaseUri = fmt::format("http://localhost:{}/z$", RemotePortNumber);
 
 		const std::string_view Bucket = "mosdef"sv;
 		zen::IoHash			   Key;
@@ -750,24 +718,21 @@ TEST_CASE("zcache.cbpackage")
 	SUBCASE("GET finds upstream when missing in local")
 	{
 		// Setup local and remote server
-		std::filesystem::path LocalDataDir	   = TestEnv.CreateNewTestDir();
-		std::filesystem::path RemoteDataDir	   = TestEnv.CreateNewTestDir();
-		const uint16_t		  LocalPortNumber  = 13337;
-		const uint16_t		  RemotePortNumber = 13338;
-
-		const auto LocalBaseUri	 = fmt::format("http://localhost:{}/z$", LocalPortNumber);
-		const auto RemoteBaseUri = fmt::format("http://localhost:{}/z$", RemotePortNumber);
+		std::filesystem::path LocalDataDir	= TestEnv.CreateNewTestDir();
+		std::filesystem::path RemoteDataDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance RemoteInstance(TestEnv);
 		RemoteInstance.SetTestDir(RemoteDataDir);
-		RemoteInstance.SpawnServer(RemotePortNumber);
-		RemoteInstance.WaitUntilReady();
+		const uint16_t RemotePortNumber = RemoteInstance.SpawnServerAndWaitUntilReady();
 
 		ZenServerInstance LocalInstance(TestEnv);
 		LocalInstance.SetTestDir(LocalDataDir);
-		LocalInstance.SpawnServer(LocalPortNumber,
+		LocalInstance.SpawnServer(TestEnv.GetNewPortNumber(),
 								  fmt::format("--upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", RemotePortNumber));
-		LocalInstance.WaitUntilReady();
+		const uint16_t LocalPortNumber = LocalInstance.WaitUntilReady();
+
+		const auto LocalBaseUri	 = fmt::format("http://localhost:{}/z$", LocalPortNumber);
+		const auto RemoteBaseUri = fmt::format("http://localhost:{}/z$", RemotePortNumber);
 
 		const std::string_view Bucket = "mosdef"sv;
 		zen::IoHash			   Key;
@@ -843,15 +808,17 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("query - 'local' does not query upstream (binary)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+		const uint16_t UpstreamPort = UpstreamCfg.Port;
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamPort);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const std::string_view Bucket = "legacy"sv;
+
 		zen::IoHash Key;
 		auto		BinaryValue = GenerateData(1024, Key);
 
@@ -878,15 +845,17 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("store - 'local' does not store upstream (binary)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+		const uint16_t UpstreamPort = UpstreamCfg.Port;
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamPort);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash Key;
 		auto		BinaryValue = GenerateData(1024, Key);
 
@@ -913,15 +882,16 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("store - 'local/remote' stores local and upstream (binary)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash Key;
 		auto		BinaryValue = GenerateData(1024, Key);
 
@@ -948,15 +918,16 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("query - 'local' does not query upstream (cppackage)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash	   Key;
 		zen::IoHash	   PayloadId;
 		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
@@ -985,15 +956,16 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("store - 'local' does not store upstream (cbpackge)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash	   Key;
 		zen::IoHash	   PayloadId;
 		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
@@ -1022,15 +994,16 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("store - 'local/remote' stores local and upstream (cbpackage)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash	   Key;
 		zen::IoHash	   PayloadId;
 		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
@@ -1059,12 +1032,12 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("skip - 'data' returns cache record without attachments/empty payload")
 	{
-		ZenConfig		  Cfg = ZenConfig::New();
+		ZenConfig		  Cfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance Instance(TestEnv);
-		const auto		  Bucket = "test"sv;
-
 		Cfg.Spawn(Instance);
 
+		const auto Bucket = "test"sv;
+
 		zen::IoHash	   Key;
 		zen::IoHash	   PayloadId;
 		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
@@ -1111,12 +1084,12 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("skip - 'data' returns empty binary value")
 	{
-		ZenConfig		  Cfg = ZenConfig::New();
+		ZenConfig		  Cfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance Instance(TestEnv);
-		const auto		  Bucket = "test"sv;
-
 		Cfg.Spawn(Instance);
 
+		const auto Bucket = "test"sv;
+
 		zen::IoHash Key;
 		auto		BinaryValue = GenerateData(1024, Key);
 
@@ -1245,14 +1218,13 @@ TEST_CASE("zcache.rpc")
 
 	SUBCASE("get cache records")
 	{
-		std::filesystem::path TestDir	 = TestEnv.CreateNewTestDir();
-		const uint16_t		  PortNumber = 13337;
-		const auto			  BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Inst(TestEnv);
 		Inst.SetTestDir(TestDir);
-		Inst.SpawnServer(PortNumber);
-		Inst.WaitUntilReady();
+
+		const uint16_t	  BasePort = Inst.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri  = fmt::format("http://localhost:{}/z$", BasePort);
 
 		CachePolicy				   Policy = CachePolicy::Default;
 		std::vector<zen::CacheKey> Keys	  = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 128);
@@ -1276,14 +1248,12 @@ TEST_CASE("zcache.rpc")
 
 	SUBCASE("get missing cache records")
 	{
-		std::filesystem::path TestDir	 = TestEnv.CreateNewTestDir();
-		const uint16_t		  PortNumber = 13337;
-		const auto			  BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Inst(TestEnv);
 		Inst.SetTestDir(TestDir);
-		Inst.SpawnServer(PortNumber);
-		Inst.WaitUntilReady();
+		const uint16_t	  BasePort = Inst.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri  = fmt::format("http://localhost:{}/z$", BasePort);
 
 		CachePolicy				   Policy		= CachePolicy::Default;
 		std::vector<zen::CacheKey> ExistingKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 128);
@@ -1324,12 +1294,12 @@ TEST_CASE("zcache.rpc")
 	{
 		using namespace utils;
 
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamServer(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalServer(TestEnv);
-
 		SpawnServer(UpstreamServer, UpstreamCfg);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalServer(TestEnv);
 		SpawnServer(LocalServer, LocalCfg);
 
 		std::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "ue4.ddc"sv, "mastodon"sv, 4);
@@ -1349,12 +1319,12 @@ TEST_CASE("zcache.rpc")
 	{
 		using namespace utils;
 
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamServer(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalServer(TestEnv);
-
 		SpawnServer(UpstreamServer, UpstreamCfg);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalServer(TestEnv);
 		SpawnServer(LocalServer, LocalCfg);
 
 		std::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "ue4.ddc"sv, "mastodon"sv, 4);
@@ -1376,14 +1346,13 @@ TEST_CASE("zcache.rpc")
 	{
 		using namespace utils;
 
-		std::filesystem::path TestDir	 = TestEnv.CreateNewTestDir();
-		const uint16_t		  PortNumber = 13337;
-		const auto			  BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Inst(TestEnv);
 		Inst.SetTestDir(TestDir);
-		Inst.SpawnServer(PortNumber);
-		Inst.WaitUntilReady();
+
+		const uint16_t	  BasePort = Inst.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri  = fmt::format("http://localhost:{}/z$", BasePort);
 
 		std::vector<zen::CacheKey> SmallKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 4, 1024);
 		std::vector<zen::CacheKey> LargeKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 4, 1024 * 1024 * 16, SmallKeys.size());
@@ -1525,26 +1494,25 @@ TEST_CASE("zcache.failing.upstream")
 	using namespace std::literals;
 	using namespace utils;
 
-	const uint16_t	  Upstream1PortNumber = 13338;
-	ZenConfig		  Upstream1Cfg		  = ZenConfig::New(Upstream1PortNumber);
+	ZenConfig		  Upstream1Cfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 	ZenServerInstance Upstream1Server(TestEnv);
+	SpawnServer(Upstream1Server, Upstream1Cfg);
 
-	const uint16_t	  Upstream2PortNumber = 13339;
-	ZenConfig		  Upstream2Cfg		  = ZenConfig::New(Upstream2PortNumber);
+	ZenConfig		  Upstream2Cfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 	ZenServerInstance Upstream2Server(TestEnv);
+	SpawnServer(Upstream2Server, Upstream2Cfg);
 
-	std::vector<std::uint16_t> UpstreamPorts = {Upstream1PortNumber, Upstream2PortNumber};
-	ZenConfig				   LocalCfg		 = ZenConfig::NewWithThreadedUpstreams(UpstreamPorts, false);
+	std::vector<std::uint16_t> UpstreamPorts = {Upstream1Cfg.Port, Upstream2Cfg.Port};
+	ZenConfig				   LocalCfg		 = ZenConfig::NewWithThreadedUpstreams(TestEnv.GetNewPortNumber(), UpstreamPorts, false);
 	LocalCfg.Args += (" --upstream-thread-count 2");
 	ZenServerInstance LocalServer(TestEnv);
-	const uint16_t	  LocalPortNumber = 13337;
-	const auto		  LocalUri		  = fmt::format("http://localhost:{}/z$", LocalPortNumber);
-	const auto		  Upstream1Uri	  = fmt::format("http://localhost:{}/z$", Upstream1PortNumber);
-	const auto		  Upstream2Uri	  = fmt::format("http://localhost:{}/z$", Upstream2PortNumber);
-
-	SpawnServer(Upstream1Server, Upstream1Cfg);
-	SpawnServer(Upstream2Server, Upstream2Cfg);
 	SpawnServer(LocalServer, LocalCfg);
+
+	const uint16_t LocalPortNumber = LocalCfg.Port;
+	const auto	   LocalUri		   = fmt::format("http://localhost:{}/z$", LocalPortNumber);
+	const auto	   Upstream1Uri	   = fmt::format("http://localhost:{}/z$", Upstream1Cfg.Port);
+	const auto	   Upstream2Uri	   = fmt::format("http://localhost:{}/z$", Upstream2Cfg.Port);
+
 	bool Upstream1Running = true;
 	bool Upstream2Running = true;
 
@@ -1785,16 +1753,16 @@ TEST_CASE("zcache.rpc.allpolicies")
 	using namespace std::literals;
 	using namespace utils;
 
-	ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+	ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 	ZenServerInstance UpstreamServer(TestEnv);
-	ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-	ZenServerInstance LocalServer(TestEnv);
-	const uint16_t	  LocalPortNumber = 13337;
-	const auto		  BaseUri		  = fmt::format("http://localhost:{}/z$", LocalPortNumber);
-
 	SpawnServer(UpstreamServer, UpstreamCfg);
+
+	ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+	ZenServerInstance LocalServer(TestEnv);
 	SpawnServer(LocalServer, LocalCfg);
 
+	const auto BaseUri = fmt::format("http://localhost:{}/z$", LocalServer.GetBasePort());
+
 	std::string_view TestVersion   = "F72150A02AE34B57A9EC91D36BA1CE08"sv;
 	std::string_view TestBucket	   = "allpoliciestest"sv;
 	std::string_view TestNamespace = "ue4.ddc"sv;
@@ -2321,7 +2289,7 @@ public:
 
 			Callback(*Instance);
 
-			Instance->SpawnServer(13337 + i, AdditionalServerArgs);
+			Instance->SpawnServer(TestEnv.GetNewPortNumber(), AdditionalServerArgs);
 		}
 
 		for (int i = 0; i < m_ServerCount; ++i)
diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h
index 14d19ad39..ce6a990f8 100644
--- a/src/zenutil/include/zenutil/zenserverprocess.h
+++ b/src/zenutil/include/zenutil/zenserverprocess.h
@@ -38,6 +38,7 @@ public:
 	inline bool				IsInitialized() const { return m_IsInitialized; }
 	inline bool				IsTestEnvironment() const { return m_IsTestInstance; }
 	inline std::string_view GetServerClass() const { return m_ServerClass; }
+	inline uint16_t			GetNewPortNumber() { return m_NextPortNumber.fetch_add(1); }
 
 private:
 	std::filesystem::path m_ProgramBaseDir;
@@ -45,6 +46,7 @@ private:
 	bool				  m_IsInitialized  = false;
 	bool				  m_IsTestInstance = false;
 	std::string			  m_ServerClass;
+	std::atomic_uint16_t  m_NextPortNumber{20000};
 };
 
 /** Zen Server Instance management
@@ -63,7 +65,7 @@ struct ZenServerInstance
 
 	void			   Shutdown();
 	void			   SignalShutdown();
-	void			   WaitUntilReady();
+	uint16_t		   WaitUntilReady();
 	[[nodiscard]] bool WaitUntilReady(int Timeout);
 	void			   EnableTermination() { m_Terminate = true; }
 	void			   DisableShutdownOnDestroy() { m_ShutdownOnDestroy = false; }
@@ -72,8 +74,20 @@ struct ZenServerInstance
 	inline void		   SetOwnerPid(int Pid) { m_OwnerPid = Pid; }
 	bool			   IsRunning();
 
-	void		SetTestDir(std::filesystem::path TestDir);
-	inline void SpawnServer(int BasePort = 0, std::string_view AdditionalServerArgs = std::string_view())
+	void SetTestDir(std::filesystem::path TestDir);
+
+	inline void SpawnServer(std::string_view AdditionalServerArgs = std::string_view())
+	{
+		SpawnServer(m_Env.GetNewPortNumber(), AdditionalServerArgs, /* WaitTimeoutMs */ 0);
+	}
+
+	inline uint16_t SpawnServerAndWaitUntilReady(std::string_view AdditionalServerArgs = std::string_view())
+	{
+		SpawnServer(m_Env.GetNewPortNumber(), AdditionalServerArgs, /* WaitTimeoutMs */ 100'000);
+		return GetBasePort();
+	}
+
+	inline void SpawnServer(int BasePort, std::string_view AdditionalServerArgs = std::string_view())
 	{
 		SpawnServer(BasePort, AdditionalServerArgs, /* WaitTimeoutMs */ 0);
 	}
@@ -85,6 +99,7 @@ struct ZenServerInstance
 
 	void		AttachToRunningServer(int BasePort = 0);
 	std::string GetBaseUri() const;
+	uint16_t	GetBasePort() const { return m_BasePort; }
 
 private:
 	ZenServerEnvironment& m_Env;
@@ -94,12 +109,13 @@ private:
 	bool				  m_Terminate		  = false;
 	bool				  m_ShutdownOnDestroy = true;
 	std::filesystem::path m_TestDir;
-	int					  m_BasePort = 0;
+	uint16_t			  m_BasePort = 0;
 	std::optional<int>	  m_OwnerPid;
 	std::string			  m_Name;
 
 	void CreateShutdownEvent(int BasePort);
 	void SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs);
+	void OnServerReady();
 };
 
 /** Shared system state
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 2971b0fab..909692fbc 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -12,6 +12,8 @@
 
 #include <atomic>
 
+#include <gsl/gsl-lite.hpp>
+
 #if ZEN_PLATFORM_WINDOWS
 #	include <zencore/windows.h>
 #else
@@ -530,8 +532,6 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	ChildEventName << "Zen_Child_" << ChildId;
 	NamedEvent ChildEvent{ChildEventName};
 
-	CreateShutdownEvent(BasePort);
-
 	ExtendableStringBuilder<32> LogId;
 	LogId << "Zen" << ChildId;
 	m_Name = LogId.ToString();
@@ -567,7 +567,7 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	if (BasePort)
 	{
 		CommandLine << " --port " << BasePort;
-		m_BasePort = BasePort;
+		m_BasePort = gsl::narrow_cast<uint16_t>(BasePort);
 	}
 
 	if (!m_TestDir.empty())
@@ -629,39 +629,6 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 		{
 			throw std::runtime_error(fmt::format("server start of {} timeout after {}", m_Name, NiceTimeSpanMs(WaitTimeoutMs)));
 		}
-
-		// Determine effective base port
-
-		ZenServerState State;
-		if (!State.InitializeReadOnly())
-		{
-			// TODO: return success/error code instead?
-			throw std::runtime_error("no zen state found");
-		}
-
-		const ZenServerState::ZenServerEntry* Entry = nullptr;
-
-		if (BasePort)
-		{
-			Entry = State.Lookup(BasePort);
-		}
-		else
-		{
-			State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) {
-				if (InEntry.Pid == static_cast<uint32_t>(GetProcessId(ChildPid)))
-				{
-					Entry = &InEntry;
-				}
-			});
-		}
-
-		if (!Entry)
-		{
-			// TODO: return success/error code instead?
-			throw std::runtime_error("no server entry found");
-		}
-
-		m_BasePort = Entry->EffectiveListenPort;
 	}
 }
 
@@ -717,23 +684,73 @@ ZenServerInstance::Detach()
 	}
 }
 
-void
+uint16_t
 ZenServerInstance::WaitUntilReady()
 {
 	while (m_ReadyEvent.Wait(100) == false)
 	{
 		if (!m_Process.IsRunning() || !m_Process.IsValid())
 		{
-			ZEN_INFO("Wait abandoned by invalid process (running={})", m_Process.IsRunning());
-			return;
+			ZEN_WARN("Wait abandoned by invalid process (running={})", m_Process.IsRunning());
+
+			return 0;
 		}
 	}
+
+	OnServerReady();
+
+	return m_BasePort;
 }
 
 bool
 ZenServerInstance::WaitUntilReady(int Timeout)
 {
-	return m_ReadyEvent.Wait(Timeout);
+	if (m_ReadyEvent.Wait(Timeout))
+	{
+		OnServerReady();
+
+		return true;
+	}
+
+	return false;
+}
+
+void
+ZenServerInstance::OnServerReady()
+{
+	// Determine effective base port
+
+	ZenServerState State;
+	if (!State.InitializeReadOnly())
+	{
+		// TODO: return success/error code instead?
+		throw std::runtime_error("no zen state found");
+	}
+
+	const ZenServerState::ZenServerEntry* Entry = nullptr;
+
+	if (m_BasePort)
+	{
+		Entry = State.Lookup(m_BasePort);
+	}
+	else
+	{
+		State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) {
+			if (InEntry.Pid == (uint32_t)m_Process.Pid())
+			{
+				Entry = &InEntry;
+			}
+		});
+	}
+
+	if (!Entry)
+	{
+		// TODO: return success/error code instead?
+		throw std::runtime_error("no server entry found");
+	}
+
+	m_BasePort = Entry->EffectiveListenPort;
+	CreateShutdownEvent(m_BasePort);
 }
 
 std::string