ZenServerProcess API changes (#719)

This refactor aims to improve the `ZenServerProcess` classes by making them useful for managing child zenserver instances in more scenarios than just automated tests. This involves changing some functions to not talk about "test directory" and instead use "data directory" etc

As a consequence of the API changes, some tests have changed accordingly.

The code includes som reference to the "hub" mode but there is not yet any other code using this mode, it's just included in this PR to simplify future merges.

9 files changed, 177 insertions, 53 deletions

diff --git a/README.md b/README.md
index de65348a2..0bc708ff6 100644
--- a/README.md
+++ b/README.md
@@ -51,13 +51,14 @@ from a Command Prompt window or a Terminal instance
 
 ### Building with xmake on the command line
 
-* configure xmake: `xmake config -m debug|release -a x64`
+* configure xmake: `xmake config -m debug`
 * build zenserver: `xmake build zenserver`
   * build all targets at once: `xmake`
 
-Compiled binaries are located in the `build` directory
+Compiled binaries end up in the `build` directory. xmake uses the `.xmake` directory to store
+project specific state. For more tips on working with xmake, see [xmake notes](docs/xmake.md)
 
-#### Installing pre-commit (optional)
+### Installing pre-commit (optional but recommended)
 
 This is necessary to run pre-commit locally, which is useful in particular to run clang-format prior to
 commit.
@@ -65,7 +66,12 @@ commit.
 * Make sure python3 is installed. Version 3.11 or later should work
   * You can install using `winget install python3`
 * Run `pip install pre-commit==3.2.0` (later versions may or may not work)
-* If you want the pre-commit steps to be run at commit time you can run `pre-commit install` within your local repo.
+* If you want the pre-commit steps to be run at commit time (which is recommended) you can 
+  run `pre-commit install` within your local repo.
+
+Once you have pre-commit installed, it's convenient to use the provided xmake task to trigger it 
+manually to apply clang-format rules. You can do this by running `xmake precommit` on the command 
+line.
 
 ## Building on Linux
 
@@ -163,7 +169,11 @@ brew install pkgconfig
 # Implementation Notes
 
 * The implementation currently depends only on a few libraries including the C++ standard library
-* It uses exceptions for certain types of unexpected error conditions
+  * In a few places we also use EASTL, mostly to eliminate memory allocations by using the excellent
+    `eastl::fixed_vector` et al containers which contain an embedded memory buffer which is used 
+    until capacity is exceeded at which point it switches to the heap
+* It uses exceptions for certain types of unexpected error conditions. Exceptions should not be 
+  used for other flow control.
 
 # Contributing Code
 
diff --git a/src/zen/cmds/service_cmd.cpp b/src/zen/cmds/service_cmd.cpp
index aa6bfb436..a781dc340 100644
--- a/src/zen/cmds/service_cmd.cpp
+++ b/src/zen/cmds/service_cmd.cpp
@@ -13,6 +13,7 @@
 #	include <zencore/windows.h>
 #	include <shellapi.h>
 #	include <Shlwapi.h>
+#	pragma comment(lib, "Shlwapi.lib")
 #endif
 
 #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
diff --git a/src/zenserver-test/cache-tests.cpp b/src/zenserver-test/cache-tests.cpp
index 854590987..0272d3797 100644
--- a/src/zenserver-test/cache-tests.cpp
+++ b/src/zenserver-test/cache-tests.cpp
@@ -35,7 +35,7 @@ TEST_CASE("zcache.basic")
 
 	{
 		ZenServerInstance Instance1(TestEnv);
-		Instance1.SetTestDir(TestDir);
+		Instance1.SetDataDir(TestDir);
 
 		const uint16_t	  PortNumber = Instance1.SpawnServerAndWaitUntilReady();
 		const std::string BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
@@ -91,7 +91,7 @@ TEST_CASE("zcache.basic")
 
 	{
 		ZenServerInstance Instance1(TestEnv);
-		Instance1.SetTestDir(TestDir);
+		Instance1.SetDataDir(TestDir);
 		const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady();
 
 		const std::string BaseUri = fmt::format("http://localhost:{}/z$", PortNumber);
@@ -167,7 +167,7 @@ TEST_CASE("zcache.cbpackage")
 		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Instance1(TestEnv);
-		Instance1.SetTestDir(TestDir);
+		Instance1.SetDataDir(TestDir);
 		const uint16_t	  PortNumber = Instance1.SpawnServerAndWaitUntilReady();
 		const std::string BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
 
@@ -203,11 +203,11 @@ TEST_CASE("zcache.cbpackage")
 		std::filesystem::path RemoteDataDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance RemoteInstance(TestEnv);
-		RemoteInstance.SetTestDir(RemoteDataDir);
+		RemoteInstance.SetDataDir(RemoteDataDir);
 		const uint16_t RemotePortNumber = RemoteInstance.SpawnServerAndWaitUntilReady();
 
 		ZenServerInstance LocalInstance(TestEnv);
-		LocalInstance.SetTestDir(LocalDataDir);
+		LocalInstance.SetDataDir(LocalDataDir);
 		LocalInstance.SpawnServer(TestEnv.GetNewPortNumber(),
 								  fmt::format("--upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", RemotePortNumber));
 		const uint16_t LocalPortNumber = LocalInstance.WaitUntilReady();
@@ -261,11 +261,11 @@ TEST_CASE("zcache.cbpackage")
 		std::filesystem::path RemoteDataDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance RemoteInstance(TestEnv);
-		RemoteInstance.SetTestDir(RemoteDataDir);
+		RemoteInstance.SetDataDir(RemoteDataDir);
 		const uint16_t RemotePortNumber = RemoteInstance.SpawnServerAndWaitUntilReady();
 
 		ZenServerInstance LocalInstance(TestEnv);
-		LocalInstance.SetTestDir(LocalDataDir);
+		LocalInstance.SetDataDir(LocalDataDir);
 		LocalInstance.SpawnServer(TestEnv.GetNewPortNumber(),
 								  fmt::format("--upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", RemotePortNumber));
 		const uint16_t LocalPortNumber = LocalInstance.WaitUntilReady();
@@ -756,7 +756,7 @@ TEST_CASE("zcache.rpc")
 		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Inst(TestEnv);
-		Inst.SetTestDir(TestDir);
+		Inst.SetDataDir(TestDir);
 
 		const uint16_t	  BasePort = Inst.SpawnServerAndWaitUntilReady();
 		const std::string BaseUri  = fmt::format("http://localhost:{}/z$", BasePort);
@@ -786,7 +786,7 @@ TEST_CASE("zcache.rpc")
 		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Inst(TestEnv);
-		Inst.SetTestDir(TestDir);
+		Inst.SetDataDir(TestDir);
 		const uint16_t	  BasePort = Inst.SpawnServerAndWaitUntilReady();
 		const std::string BaseUri  = fmt::format("http://localhost:{}/z$", BasePort);
 
@@ -1239,7 +1239,7 @@ TEST_CASE("zcache.rpc")
 		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Inst(TestEnv);
-		Inst.SetTestDir(TestDir);
+		Inst.SetDataDir(TestDir);
 
 		const uint16_t	  BasePort = Inst.SpawnServerAndWaitUntilReady();
 		const std::string BaseUri  = fmt::format("http://localhost:{}/z$", BasePort);
diff --git a/src/zenserver-test/projectstore-tests.cpp b/src/zenserver-test/projectstore-tests.cpp
index c8c96dbbb..735aef159 100644
--- a/src/zenserver-test/projectstore-tests.cpp
+++ b/src/zenserver-test/projectstore-tests.cpp
@@ -34,7 +34,7 @@ TEST_CASE("project.basic")
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 	ZenServerInstance Instance1(TestEnv);
-	Instance1.SetTestDir(TestDir);
+	Instance1.SetDataDir(TestDir);
 
 	const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady();
 
diff --git a/src/zenserver-test/workspace-tests.cpp b/src/zenserver-test/workspace-tests.cpp
index f299b6dcf..7595d790a 100644
--- a/src/zenserver-test/workspace-tests.cpp
+++ b/src/zenserver-test/workspace-tests.cpp
@@ -81,7 +81,7 @@ TEST_CASE("workspaces.create")
 
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 	ZenServerInstance	  Instance(TestEnv);
-	Instance.SetTestDir(TestDir);
+	Instance.SetDataDir(TestDir);
 	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(
 		fmt::format("--workspaces-enabled --workspaces-allow-changes --system-dir {}", SystemRootPath));
 	CHECK(PortNumber != 0);
@@ -214,7 +214,7 @@ TEST_CASE("workspaces.restricted")
 
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 	ZenServerInstance	  Instance(TestEnv);
-	Instance.SetTestDir(TestDir);
+	Instance.SetDataDir(TestDir);
 	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath));
 	CHECK(PortNumber != 0);
 
@@ -319,7 +319,7 @@ TEST_CASE("workspaces.lifetimes")
 	{
 		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 		ZenServerInstance	  Instance(TestEnv);
-		Instance.SetTestDir(TestDir);
+		Instance.SetDataDir(TestDir);
 		const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(
 			fmt::format("--workspaces-enabled --workspaces-allow-changes --system-dir {}", SystemRootPath));
 		CHECK(PortNumber != 0);
@@ -343,7 +343,7 @@ TEST_CASE("workspaces.lifetimes")
 	{
 		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 		ZenServerInstance	  Instance(TestEnv);
-		Instance.SetTestDir(TestDir);
+		Instance.SetDataDir(TestDir);
 		const uint16_t PortNumber =
 			Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath));
 		CHECK(PortNumber != 0);
@@ -362,7 +362,7 @@ TEST_CASE("workspaces.lifetimes")
 	{
 		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 		ZenServerInstance	  Instance(TestEnv);
-		Instance.SetTestDir(TestDir);
+		Instance.SetDataDir(TestDir);
 		const uint16_t PortNumber =
 			Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath));
 		CHECK(PortNumber != 0);
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 418fc7978..6f207b184 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -122,7 +122,7 @@ TEST_CASE("default.single")
 {
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 	ZenServerInstance	  Instance(TestEnv);
-	Instance.SetTestDir(TestDir);
+	Instance.SetDataDir(TestDir);
 	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady();
 
 	std::atomic<uint64_t> RequestCount{0};
@@ -172,7 +172,7 @@ TEST_CASE("default.loopback")
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 	ZenServerInstance Instance(TestEnv);
-	Instance.SetTestDir(TestDir);
+	Instance.SetDataDir(TestDir);
 	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady("--http-forceloopback");
 
 	ZEN_INFO("Running loopback server test...");
@@ -200,12 +200,12 @@ TEST_CASE("multi.basic")
 {
 	ZenServerInstance	  Instance1(TestEnv);
 	std::filesystem::path TestDir1 = TestEnv.CreateNewTestDir();
-	Instance1.SetTestDir(TestDir1);
+	Instance1.SetDataDir(TestDir1);
 	Instance1.SpawnServer();
 
 	ZenServerInstance	  Instance2(TestEnv);
 	std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir();
-	Instance2.SetTestDir(TestDir2);
+	Instance2.SetDataDir(TestDir2);
 	Instance2.SpawnServer();
 
 	ZEN_INFO("Waiting...");
@@ -336,14 +336,14 @@ TEST_CASE("lifetime.owner")
 
 	ZenServerInstance	  Zen1(TestEnv);
 	std::filesystem::path TestDir1 = TestEnv.CreateNewTestDir();
-	Zen1.SetTestDir(TestDir1);
+	Zen1.SetDataDir(TestDir1);
 	Zen1.SpawnServer(PortNumber);
 	Zen1.WaitUntilReady();
 	Zen1.Detach();
 
 	ZenServerInstance	  Zen2(TestEnv);
 	std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir();
-	Zen2.SetTestDir(TestDir2);
+	Zen2.SetDataDir(TestDir2);
 	Zen2.SpawnServer(PortNumber);
 	Zen2.WaitUntilReady();
 	Zen2.Detach();
@@ -362,24 +362,24 @@ TEST_CASE("lifetime.owner.2")
 	std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir();
 
 	ZenServerInstance Zen1(TestEnv);
-	Zen1.SetTestDir(TestDir1);
+	Zen1.SetDataDir(TestDir1);
 	Zen1.SpawnServer(PortNumber);
 	Zen1.WaitUntilReady();
 
 	ZenServerInstance Zen2(TestEnv);
-	Zen2.SetTestDir(TestDir2);
+	Zen2.SetDataDir(TestDir2);
 	Zen2.SetOwnerPid(Zen1.GetPid());
 	Zen2.SpawnServer(PortNumber + 1);
 	Zen2.Detach();
 
 	ZenServerInstance Zen3(TestEnv);
-	Zen3.SetTestDir(TestDir2);
+	Zen3.SetDataDir(TestDir2);
 	Zen3.SetOwnerPid(Zen1.GetPid());
 	Zen3.SpawnServer(PortNumber + 1);
 	Zen3.Detach();
 
 	ZenServerInstance Zen4(TestEnv);
-	Zen4.SetTestDir(TestDir2);
+	Zen4.SetDataDir(TestDir2);
 	Zen4.SetOwnerPid(Zen1.GetPid());
 	Zen4.SpawnServer(PortNumber + 1);
 	Zen4.Detach();
diff --git a/src/zenserver-test/zenserver-test.h b/src/zenserver-test/zenserver-test.h
index e7cee3f94..8d83285e6 100644
--- a/src/zenserver-test/zenserver-test.h
+++ b/src/zenserver-test/zenserver-test.h
@@ -82,7 +82,7 @@ namespace utils {
 
 		void Spawn(ZenServerInstance& Inst)
 		{
-			Inst.SetTestDir(DataDir);
+			Inst.SetDataDir(DataDir);
 			Inst.SpawnServer(Port, Args);
 			const uint16_t InstancePort = Inst.WaitUntilReady();
 			CHECK_MESSAGE(InstancePort != 0, Inst.GetLogOutput());
@@ -163,7 +163,7 @@ public:
 		{
 			auto& Instance = m_Instances[i];
 			Instance	   = std::make_unique<ZenServerInstance>(TestEnv);
-			Instance->SetTestDir(TestEnv.CreateNewTestDir());
+			Instance->SetDataDir(TestEnv.CreateNewTestDir());
 		}
 
 		for (int i = 0; i < m_ServerCount; ++i)
diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h
index 0da63285b..d0402640b 100644
--- a/src/zenutil/include/zenutil/zenserverprocess.h
+++ b/src/zenutil/include/zenutil/zenserverprocess.h
@@ -34,8 +34,10 @@ public:
 
 	void Initialize(std::filesystem::path ProgramBaseDir);
 	void InitializeForTest(std::filesystem::path ProgramBaseDir, std::filesystem::path TestBaseDir, std::string_view ServerClass = "");
+	void InitializeForHub(std::filesystem::path ProgramBaseDir, std::filesystem::path TestBaseDir, std::string_view ServerClass = "");
 
 	std::filesystem::path	CreateNewTestDir();
+	std::filesystem::path	CreateChildDir(std::string_view ChildName);
 	std::filesystem::path	ProgramBaseDir() const { return m_ProgramBaseDir; }
 	std::filesystem::path	GetTestRootDir(std::string_view Path);
 	inline bool				IsInitialized() const { return m_IsInitialized; }
@@ -43,11 +45,18 @@ public:
 	inline std::string_view GetServerClass() const { return m_ServerClass; }
 	inline uint16_t			GetNewPortNumber() { return m_NextPortNumber.fetch_add(1); }
 
+	// The defaults will work for a single root process only. For hierarchical
+	// setups (e.g., hub managing storage servers), we need to be able to
+	// allocate distinct child IDs and ports to avoid overlap/conflicts.
+	static void SetBaseChildId(int InitialValue);
+	void		SetNextPortNumber(uint16_t NewValue) { m_NextPortNumber = NewValue; }
+
 private:
 	std::filesystem::path m_ProgramBaseDir;
-	std::filesystem::path m_TestBaseDir;
+	std::filesystem::path m_ChildProcessBaseDir;
 	bool				  m_IsInitialized  = false;
 	bool				  m_IsTestInstance = false;
+	bool				  m_IsHubInstance  = false;
 	std::string			  m_ServerClass;
 	std::atomic_uint16_t  m_NextPortNumber{20000};
 };
@@ -60,10 +69,19 @@ private:
 	Especially useful for automated testing but can also be used for
 	management tools.
 
+	This is also used by zenserver in hub mode, for managing storage
+	server instances.
+
   */
 struct ZenServerInstance
 {
-	ZenServerInstance(ZenServerEnvironment& TestEnvironment);
+	enum class ServerMode
+	{
+		kStorageServer,	 // default
+		kHubServer,
+	};
+
+	ZenServerInstance(ZenServerEnvironment& TestEnvironment, ServerMode Mode = ServerMode::kStorageServer);
 	~ZenServerInstance();
 
 	int				   Shutdown();
@@ -72,6 +90,7 @@ struct ZenServerInstance
 	[[nodiscard]] bool WaitUntilReady(int Timeout);
 	[[nodiscard]] bool WaitUntilExited(int Timeout, std::error_code& OutEc);
 	void			   EnableTermination() { m_Terminate = true; }
+	void			   EnableShutdownOnDestroy() { m_ShutdownOnDestroy = true; }
 	void			   DisableShutdownOnDestroy() { m_ShutdownOnDestroy = false; }
 	void			   Detach();
 	inline int		   GetPid() const { return m_Process.Pid(); }
@@ -81,7 +100,11 @@ struct ZenServerInstance
 	bool			   Terminate();
 	std::string		   GetLogOutput() const;
 
-	void SetTestDir(std::filesystem::path TestDir);
+	inline ServerMode GetServerMode() const { return m_ServerMode; }
+
+	inline void SetServerExecutablePath(std::filesystem::path ExecutablePath) { m_ServerExecutablePath = ExecutablePath; }
+
+	void SetDataDir(std::filesystem::path TestDir);
 
 	inline void SpawnServer(std::string_view AdditionalServerArgs = std::string_view())
 	{
@@ -117,11 +140,13 @@ private:
 	std::unique_ptr<NamedEvent> m_ShutdownEvent;
 	bool						m_Terminate			= false;
 	bool						m_ShutdownOnDestroy = true;
-	std::filesystem::path		m_TestDir;
-	uint16_t					m_BasePort = 0;
+	std::filesystem::path		m_DataDir;
+	uint16_t					m_BasePort	 = 0;
+	ServerMode					m_ServerMode = ServerMode::kStorageServer;
 	std::optional<int>			m_OwnerPid;
 	std::string					m_Name;
 	std::filesystem::path		m_OutputCapturePath;
+	std::filesystem::path		m_ServerExecutablePath;
 
 	void CreateShutdownEvent(int BasePort);
 	void SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs);
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 6a93f0c63..b56df400d 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -31,6 +31,19 @@
 
 namespace zen {
 
+// this needs to key off the current process child-id, in order to avoid conflicts
+// in situations where we have a tree of zenserver child processes (such as in hub
+// tests)
+
+std::atomic<int> ChildIdCounter{0};
+
+void
+ZenServerEnvironment::SetBaseChildId(int InitialValue)
+{
+	ZEN_ASSERT(ChildIdCounter == 0);
+	ChildIdCounter = InitialValue;
+}
+
 namespace zenutil {
 #if ZEN_PLATFORM_WINDOWS
 	class SecurityAttributes
@@ -507,8 +520,8 @@ ZenServerEnvironment::InitializeForTest(std::filesystem::path ProgramBaseDir,
 {
 	using namespace std::literals;
 
-	m_ProgramBaseDir = ProgramBaseDir;
-	m_TestBaseDir	 = TestBaseDir;
+	m_ProgramBaseDir	  = ProgramBaseDir;
+	m_ChildProcessBaseDir = TestBaseDir;
 
 	ZEN_INFO("Program base dir is '{}'", ProgramBaseDir);
 	ZEN_INFO("Cleaning test base dir '{}'", TestBaseDir);
@@ -536,6 +549,59 @@ ZenServerEnvironment::InitializeForTest(std::filesystem::path ProgramBaseDir,
 	}
 }
 
+void
+ZenServerEnvironment::InitializeForHub(std::filesystem::path ProgramBaseDir,
+									   std::filesystem::path ChildBaseDir,
+									   std::string_view		 ServerClass)
+{
+	using namespace std::literals;
+
+	m_ProgramBaseDir	  = ProgramBaseDir;
+	m_ChildProcessBaseDir = ChildBaseDir;
+
+	ZEN_INFO("Program base dir is '{}'", m_ProgramBaseDir);
+	ZEN_INFO("Cleaning child base dir '{}'", m_ChildProcessBaseDir);
+	DeleteDirectories(m_ChildProcessBaseDir.c_str());
+
+	m_IsHubInstance = true;
+	m_IsInitialized = true;
+
+	if (ServerClass.empty())
+	{
+#if ZEN_WITH_HTTPSYS
+		if (!zen::windows::IsRunningOnWine())
+		{
+			m_ServerClass = "httpsys"sv;
+
+			return;
+		}
+#endif
+
+		m_ServerClass = "asio"sv;
+	}
+	else
+	{
+		m_ServerClass = ServerClass;
+	}
+}
+
+std::filesystem::path
+ZenServerEnvironment::CreateChildDir(std::string_view ChildName)
+{
+	using namespace std::literals;
+
+	std::filesystem::path ChildPath = m_ChildProcessBaseDir / ChildName;
+
+	if (!IsDir(ChildPath))
+	{
+		ZEN_INFO("Creating new test dir @ '{}'", ChildPath);
+
+		CreateDirectories(ChildPath);
+	}
+
+	return ChildPath;
+}
+
 std::filesystem::path
 ZenServerEnvironment::CreateNewTestDir()
 {
@@ -544,7 +610,7 @@ ZenServerEnvironment::CreateNewTestDir()
 	ExtendableWideStringBuilder<256> TestDir;
 	TestDir << "test"sv << int64_t(ZenServerTestCounter.fetch_add(1));
 
-	std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str();
+	std::filesystem::path TestPath = m_ChildProcessBaseDir / TestDir.c_str();
 	ZEN_ASSERT(!IsDir(TestPath));
 
 	ZEN_INFO("Creating new test dir @ '{}'", TestPath);
@@ -566,11 +632,11 @@ ZenServerEnvironment::GetTestRootDir(std::string_view Path)
 
 //////////////////////////////////////////////////////////////////////////
 
-std::atomic<int> ChildIdCounter{0};
-
-ZenServerInstance::ZenServerInstance(ZenServerEnvironment& TestEnvironment) : m_Env(TestEnvironment)
+ZenServerInstance::ZenServerInstance(ZenServerEnvironment& TestEnvironment, ServerMode Mode) : m_Env(TestEnvironment), m_ServerMode(Mode)
 {
 	ZEN_ASSERT(TestEnvironment.IsInitialized());
+
+	m_ServerMode = Mode;
 }
 
 ZenServerInstance::~ZenServerInstance()
@@ -710,6 +776,22 @@ ZenServerInstance::SpawnServer(std::string_view ServerArgs, bool OpenConsole, in
 	SpawnServerInternal(ChildId, ServerArgs, OpenConsole, WaitTimeoutMs);
 }
 
+std::string_view
+ToString(ZenServerInstance::ServerMode Mode)
+{
+	using namespace std::literals;
+
+	switch (Mode)
+	{
+		case ZenServerInstance::ServerMode::kStorageServer:
+			return "storage"sv;
+		case ZenServerInstance::ServerMode::kHubServer:
+			return "hub"sv;
+		default:
+			return "invalid"sv;
+	}
+}
+
 void
 ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, bool OpenConsole, int WaitTimeoutMs)
 {
@@ -721,6 +803,12 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs,
 
 	ExtendableStringBuilder<512> CommandLine;
 	CommandLine << "zenserver" ZEN_EXE_SUFFIX_LITERAL;	// see CreateProc() call for actual binary path
+
+	if (m_ServerMode == ServerMode::kHubServer)
+	{
+		CommandLine << " hub";
+	}
+
 	CommandLine << " --child-id " << ChildEventName;
 
 	if (!ServerArgs.empty())
@@ -730,7 +818,7 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs,
 
 	std::filesystem::path CurrentDirectory = std::filesystem::current_path();
 
-	ZEN_DEBUG("Spawning server '{}'", m_Name);
+	ZEN_DEBUG("Spawning {} server '{}'", ToString(m_ServerMode), m_Name);
 
 	uint32_t CreationFlags = 0;
 	if (OpenConsole)
@@ -738,8 +826,9 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs,
 		CreationFlags |= CreateProcOptions::Flag_NewConsole;
 	}
 
-	const std::filesystem::path BaseDir	   = m_Env.ProgramBaseDir();
-	const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
+	const std::filesystem::path BaseDir = m_Env.ProgramBaseDir();
+	const std::filesystem::path Executable =
+		m_ServerExecutablePath.empty() ? (BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL) : m_ServerExecutablePath;
 	const std::filesystem::path OutputPath =
 		OpenConsole ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log");
 	CreateProcOptions CreateOptions = {.WorkingDirectory = &CurrentDirectory, .Flags = CreationFlags, .StdoutFile = OutputPath};
@@ -799,8 +888,7 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	const int ChildId = AssignName();
 
 	ExtendableStringBuilder<512> CommandLine;
-
-	const bool IsTest = m_Env.IsTestEnvironment();
+	const bool					 IsTest = m_Env.IsTestEnvironment();
 
 	if (IsTest)
 	{
@@ -835,10 +923,10 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 		m_BasePort = gsl::narrow_cast<uint16_t>(BasePort);
 	}
 
-	if (!m_TestDir.empty())
+	if (!m_DataDir.empty())
 	{
 		CommandLine << " --data-dir ";
-		PathToUtf8(m_TestDir.c_str(), CommandLine);
+		PathToUtf8(m_DataDir.c_str(), CommandLine);
 	}
 
 	if (!AdditionalServerArgs.empty())
@@ -1028,10 +1116,10 @@ ZenServerInstance::GetBaseUri() const
 }
 
 void
-ZenServerInstance::SetTestDir(std::filesystem::path TestDir)
+ZenServerInstance::SetDataDir(std::filesystem::path TestDir)
 {
 	ZEN_ASSERT(!m_Process.IsValid());
-	m_TestDir = TestDir;
+	m_DataDir = TestDir;
 }
 
 bool