add tests for s3 and file hydrators (#886)

2 files changed, 699 insertions, 4 deletions

diff --git a/src/zenserver/hub/hydration.cpp b/src/zenserver/hub/hydration.cpp
index 4998ef1af..4a44da052 100644
--- a/src/zenserver/hub/hydration.cpp
+++ b/src/zenserver/hub/hydration.cpp
@@ -17,6 +17,16 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <json11.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
+#if ZEN_WITH_TESTS
+#	include <zencore/parallelwork.h>
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+#	include <zencore/thread.h>
+#	include <zencore/workthreadpool.h>
+#	include <zenutil/cloud/minioprocess.h>
+#	include <cstring>
+#endif	// ZEN_WITH_TESTS
+
 namespace zen {
 
 namespace {
@@ -110,8 +120,6 @@ FileHydrator::Hydrate()
 		ZEN_DEBUG("Cleaning server state '{}'", m_Config.ServerStateDir);
 		CleanDirectory(m_Config.ServerStateDir, ForceRemoveReadOnlyFiles);
 	}
-
-	// Note that we leave the storage state intact until next dehydration replaces the content
 }
 
 void
@@ -222,6 +230,12 @@ S3Hydrator::CreateS3Client() const
 	Options.BucketName = m_Bucket;
 	Options.Region	   = m_Region;
 
+	if (!m_Config.S3Endpoint.empty())
+	{
+		Options.Endpoint  = m_Config.S3Endpoint;
+		Options.PathStyle = m_Config.S3PathStyle;
+	}
+
 	if (m_CredentialProvider)
 	{
 		Options.CredentialProvider = m_CredentialProvider;
@@ -270,11 +284,20 @@ S3Hydrator::Dehydrate()
 		DirectoryContent DirContent;
 		GetDirectoryContent(m_Config.ServerStateDir, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive, DirContent);
 
-		for (const std::filesystem::path& RelPath : DirContent.Files)
+		for (const std::filesystem::path& AbsPath : DirContent.Files)
 		{
+			std::filesystem::path RelPath = AbsPath.lexically_relative(m_Config.ServerStateDir);
+			if (RelPath.empty() || *RelPath.begin() == "..")
+			{
+				throw zen::runtime_error(
+					"lexically_relative produced a '..'-escape path for '{}' relative to '{}' - "
+					"path form mismatch (e.g. \\\\?\\ prefix on one but not the other)",
+					AbsPath.string(),
+					m_Config.ServerStateDir.string());
+			}
 			std::string Key = MakeObjectKey(FolderName, RelPath);
 
-			BasicFile File(MakeSafeAbsolutePath(m_Config.ServerStateDir / RelPath), BasicFile::Mode::kRead);
+			BasicFile File(AbsPath, BasicFile::Mode::kRead);
 			uint64_t  FileSize = File.FileSize();
 
 			S3Result UploadResult =
@@ -505,4 +528,666 @@ CreateHydrator(const HydrationConfig& Config)
 	throw std::runtime_error(fmt::format("Unknown hydration strategy: {}", Config.TargetSpecification));
 }
 
+#if ZEN_WITH_TESTS
+
+namespace {
+
+	/// Scoped RAII helper to set/restore a single environment variable within a test.
+	/// Used to configure AWS credentials for each S3 test's MinIO instance
+	/// without polluting the global environment.
+	struct ScopedEnvVar
+	{
+		std::string				   m_Name;
+		std::optional<std::string> m_OldValue;	// nullopt = was not set; "" = was set to empty string
+
+		ScopedEnvVar(std::string_view Name, std::string_view Value) : m_Name(Name)
+		{
+#	if ZEN_PLATFORM_WINDOWS
+			// Use the raw API so we can distinguish "not set" (ERROR_ENVVAR_NOT_FOUND)
+			// from "set to empty string" (returns 0 with no error).
+			char  Buf[1];
+			DWORD Len = GetEnvironmentVariableA(m_Name.c_str(), Buf, sizeof(Buf));
+			if (Len == 0 && GetLastError() == ERROR_ENVVAR_NOT_FOUND)
+			{
+				m_OldValue = std::nullopt;
+			}
+			else
+			{
+				// Len == 0 with no error: variable exists but is empty.
+				// Len > sizeof(Buf): value is non-empty; Len is the required buffer size
+				//   (including null terminator) - allocate and re-read.
+				std::string Old(Len == 0 ? 0 : Len - 1, '\0');
+				if (Len > sizeof(Buf))
+				{
+					GetEnvironmentVariableA(m_Name.c_str(), Old.data(), Len);
+				}
+				m_OldValue = std::move(Old);
+			}
+			SetEnvironmentVariableA(m_Name.c_str(), std::string(Value).c_str());
+#	else
+			// getenv returns nullptr when not set, "" when set to empty string.
+			const char* Existing = getenv(m_Name.c_str());
+			m_OldValue			 = Existing ? std::optional<std::string>(Existing) : std::nullopt;
+			setenv(m_Name.c_str(), std::string(Value).c_str(), 1);
+#	endif
+		}
+		~ScopedEnvVar()
+		{
+#	if ZEN_PLATFORM_WINDOWS
+			SetEnvironmentVariableA(m_Name.c_str(), m_OldValue.has_value() ? m_OldValue->c_str() : nullptr);
+#	else
+			if (m_OldValue.has_value())
+			{
+				setenv(m_Name.c_str(), m_OldValue->c_str(), 1);
+			}
+			else
+			{
+				unsetenv(m_Name.c_str());
+			}
+#	endif
+		}
+	};
+
+	/// Create a small file hierarchy under BaseDir:
+	///   file_a.bin
+	///   subdir/file_b.bin
+	///   subdir/nested/file_c.bin
+	/// Returns a vector of (relative path, content) pairs for later verification.
+	std::vector<std::pair<std::filesystem::path, IoBuffer>> CreateTestTree(const std::filesystem::path& BaseDir)
+	{
+		std::vector<std::pair<std::filesystem::path, IoBuffer>> Files;
+
+		auto AddFile = [&](std::filesystem::path RelPath, IoBuffer Content) {
+			std::filesystem::path FullPath = BaseDir / RelPath;
+			CreateDirectories(FullPath.parent_path());
+			WriteFile(FullPath, Content);
+			Files.emplace_back(std::move(RelPath), std::move(Content));
+		};
+
+		AddFile("file_a.bin", CreateSemiRandomBlob(1024));
+		AddFile("subdir/file_b.bin", CreateSemiRandomBlob(2048));
+		AddFile("subdir/nested/file_c.bin", CreateSemiRandomBlob(512));
+
+		return Files;
+	}
+
+	void VerifyTree(const std::filesystem::path& Dir, const std::vector<std::pair<std::filesystem::path, IoBuffer>>& Expected)
+	{
+		for (const auto& [RelPath, Content] : Expected)
+		{
+			std::filesystem::path FullPath = Dir / RelPath;
+			REQUIRE_MESSAGE(std::filesystem::exists(FullPath), FullPath.string());
+			BasicFile ReadBack(FullPath, BasicFile::Mode::kRead);
+			IoBuffer  ReadContent = ReadBack.ReadRange(0, ReadBack.FileSize());
+			REQUIRE_EQ(ReadContent.GetSize(), Content.GetSize());
+			CHECK(std::memcmp(ReadContent.GetData(), Content.GetData(), Content.GetSize()) == 0);
+		}
+	}
+
+}  // namespace
+
+TEST_SUITE_BEGIN("server.hydration");
+
+// ---------------------------------------------------------------------------
+// FileHydrator tests
+// ---------------------------------------------------------------------------
+
+TEST_CASE("hydration.file.dehydrate_hydrate")
+{
+	ScopedTemporaryDirectory TempDir;
+
+	std::filesystem::path ServerStateDir = TempDir.Path() / "server_state";
+	std::filesystem::path HydrationStore = TempDir.Path() / "hydration_store";
+	std::filesystem::path HydrationTemp	 = TempDir.Path() / "hydration_temp";
+	CreateDirectories(ServerStateDir);
+	CreateDirectories(HydrationStore);
+	CreateDirectories(HydrationTemp);
+
+	const std::string ModuleId	= "testmodule";
+	auto			  TestFiles = CreateTestTree(ServerStateDir);
+
+	HydrationConfig Config;
+	Config.ServerStateDir	   = ServerStateDir;
+	Config.TempDir			   = HydrationTemp;
+	Config.ModuleId			   = ModuleId;
+	Config.TargetSpecification = "file://" + HydrationStore.string();
+
+	// Dehydrate: copy server state to file store
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Dehydrate();
+	}
+
+	// Verify the module folder exists in the store and ServerStateDir was wiped
+	CHECK(std::filesystem::exists(HydrationStore / ModuleId));
+	CHECK(std::filesystem::is_empty(ServerStateDir));
+
+	// Hydrate: restore server state from file store
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Hydrate();
+	}
+
+	// Verify restored contents match the original
+	VerifyTree(ServerStateDir, TestFiles);
+}
+
+TEST_CASE("hydration.file.dehydrate_cleans_server_state")
+{
+	ScopedTemporaryDirectory TempDir;
+
+	std::filesystem::path ServerStateDir = TempDir.Path() / "server_state";
+	std::filesystem::path HydrationStore = TempDir.Path() / "hydration_store";
+	std::filesystem::path HydrationTemp	 = TempDir.Path() / "hydration_temp";
+	CreateDirectories(ServerStateDir);
+	CreateDirectories(HydrationStore);
+	CreateDirectories(HydrationTemp);
+
+	CreateTestTree(ServerStateDir);
+
+	HydrationConfig Config;
+	Config.ServerStateDir	   = ServerStateDir;
+	Config.TempDir			   = HydrationTemp;
+	Config.ModuleId			   = "testmodule";
+	Config.TargetSpecification = "file://" + HydrationStore.string();
+
+	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+	Hydrator->Dehydrate();
+
+	// FileHydrator::Dehydrate() must wipe ServerStateDir when done
+	CHECK(std::filesystem::is_empty(ServerStateDir));
+}
+
+TEST_CASE("hydration.file.hydrate_overwrites_existing_state")
+{
+	ScopedTemporaryDirectory TempDir;
+
+	std::filesystem::path ServerStateDir = TempDir.Path() / "server_state";
+	std::filesystem::path HydrationStore = TempDir.Path() / "hydration_store";
+	std::filesystem::path HydrationTemp	 = TempDir.Path() / "hydration_temp";
+	CreateDirectories(ServerStateDir);
+	CreateDirectories(HydrationStore);
+	CreateDirectories(HydrationTemp);
+
+	auto TestFiles = CreateTestTree(ServerStateDir);
+
+	HydrationConfig Config;
+	Config.ServerStateDir	   = ServerStateDir;
+	Config.TempDir			   = HydrationTemp;
+	Config.ModuleId			   = "testmodule";
+	Config.TargetSpecification = "file://" + HydrationStore.string();
+
+	// Dehydrate the original state
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Dehydrate();
+	}
+
+	// Put a stale file in ServerStateDir to simulate leftover state
+	WriteFile(ServerStateDir / "stale.bin", CreateSemiRandomBlob(256));
+
+	// Hydrate - must wipe stale file and restore original
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Hydrate();
+	}
+
+	CHECK_FALSE(std::filesystem::exists(ServerStateDir / "stale.bin"));
+	VerifyTree(ServerStateDir, TestFiles);
+}
+
+// ---------------------------------------------------------------------------
+// FileHydrator concurrent test
+// ---------------------------------------------------------------------------
+
+TEST_CASE("hydration.file.concurrent")
+{
+	// N modules dehydrate and hydrate concurrently via ParallelWork.
+	// Each module operates in its own directory - tests for global/static state races.
+	constexpr int kModuleCount = 4;
+
+	ScopedTemporaryDirectory TempDir;
+	std::filesystem::path	 HydrationStore = TempDir.Path() / "hydration_store";
+	CreateDirectories(HydrationStore);
+
+	struct ModuleData
+	{
+		HydrationConfig											Config;
+		std::vector<std::pair<std::filesystem::path, IoBuffer>> Files;
+	};
+	std::vector<ModuleData> Modules(kModuleCount);
+
+	for (int I = 0; I < kModuleCount; ++I)
+	{
+		std::string			  ModuleId = fmt::format("file_concurrent_{}", I);
+		std::filesystem::path StateDir = TempDir.Path() / ModuleId / "state";
+		std::filesystem::path TempPath = TempDir.Path() / ModuleId / "temp";
+		CreateDirectories(StateDir);
+		CreateDirectories(TempPath);
+
+		Modules[I].Config.ServerStateDir	  = StateDir;
+		Modules[I].Config.TempDir			  = TempPath;
+		Modules[I].Config.ModuleId			  = ModuleId;
+		Modules[I].Config.TargetSpecification = "file://" + HydrationStore.string();
+		Modules[I].Files					  = CreateTestTree(StateDir);
+	}
+
+	// Concurrent dehydrate
+	{
+		WorkerThreadPool  Pool(kModuleCount, "hydration_file_dehy");
+		std::atomic<bool> AbortFlag{false};
+		std::atomic<bool> PauseFlag{false};
+		ParallelWork	  Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+		for (int I = 0; I < kModuleCount; ++I)
+		{
+			Work.ScheduleWork(Pool, [&Config = Modules[I].Config](std::atomic<bool>&) {
+				std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+				Hydrator->Dehydrate();
+			});
+		}
+		Work.Wait();
+		CHECK_FALSE(Work.IsAborted());
+	}
+
+	// Concurrent hydrate
+	{
+		WorkerThreadPool  Pool(kModuleCount, "hydration_file_hy");
+		std::atomic<bool> AbortFlag{false};
+		std::atomic<bool> PauseFlag{false};
+		ParallelWork	  Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+		for (int I = 0; I < kModuleCount; ++I)
+		{
+			Work.ScheduleWork(Pool, [&Config = Modules[I].Config](std::atomic<bool>&) {
+				std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+				Hydrator->Hydrate();
+			});
+		}
+		Work.Wait();
+		CHECK_FALSE(Work.IsAborted());
+	}
+
+	// Verify all modules restored correctly
+	for (int I = 0; I < kModuleCount; ++I)
+	{
+		VerifyTree(Modules[I].Config.ServerStateDir, Modules[I].Files);
+	}
+}
+
+// ---------------------------------------------------------------------------
+// S3Hydrator tests
+//
+// Each test case spawns its own local MinIO instance (self-contained, no external setup needed).
+// The MinIO binary must be present in the same directory as the test executable (copied by xmake).
+// ---------------------------------------------------------------------------
+
+TEST_CASE("hydration.s3.dehydrate_hydrate")
+{
+	MinioProcessOptions MinioOpts;
+	MinioOpts.Port = 19010;
+	MinioProcess Minio(MinioOpts);
+	Minio.SpawnMinioServer();
+	Minio.CreateBucket("zen-hydration-test");
+
+	ScopedEnvVar EnvAccessKey("AWS_ACCESS_KEY_ID", Minio.RootUser());
+	ScopedEnvVar EnvSecretKey("AWS_SECRET_ACCESS_KEY", Minio.RootPassword());
+
+	ScopedTemporaryDirectory TempDir;
+
+	std::filesystem::path ServerStateDir = TempDir.Path() / "server_state";
+	std::filesystem::path HydrationTemp	 = TempDir.Path() / "hydration_temp";
+	CreateDirectories(ServerStateDir);
+	CreateDirectories(HydrationTemp);
+
+	const std::string ModuleId	= "s3test_roundtrip";
+	auto			  TestFiles = CreateTestTree(ServerStateDir);
+
+	HydrationConfig Config;
+	Config.ServerStateDir	   = ServerStateDir;
+	Config.TempDir			   = HydrationTemp;
+	Config.ModuleId			   = ModuleId;
+	Config.TargetSpecification = "s3://zen-hydration-test";
+	Config.S3Endpoint		   = Minio.Endpoint();
+	Config.S3PathStyle		   = true;
+
+	// Dehydrate: upload server state to MinIO
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Dehydrate();
+	}
+
+	// Wipe server state
+	CleanDirectory(ServerStateDir, true);
+	CHECK(std::filesystem::is_empty(ServerStateDir));
+
+	// Hydrate: download from MinIO back to server state
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Hydrate();
+	}
+
+	// Verify restored contents match the original
+	VerifyTree(ServerStateDir, TestFiles);
+}
+
+TEST_CASE("hydration.s3.current_state_json_selects_latest_folder")
+{
+	// Each Dehydrate() uploads files to a new timestamp-named folder and then overwrites
+	// current-state.json to point at that folder. Old folders are NOT deleted.
+	// Hydrate() must read current-state.json to determine which folder to restore from.
+	//
+	// This test verifies that:
+	//   1. After two dehydrations, Hydrate() restores from the second snapshot, not the first,
+	//      confirming that current-state.json was updated between dehydrations.
+	//   2. current-state.json is updated to point at the second (latest) folder.
+	//   3. Hydrate() restores the v2 snapshot (identified by v2marker.bin), NOT the v1 snapshot.
+
+	MinioProcessOptions MinioOpts;
+	MinioOpts.Port = 19011;
+	MinioProcess Minio(MinioOpts);
+	Minio.SpawnMinioServer();
+	Minio.CreateBucket("zen-hydration-test");
+
+	ScopedEnvVar EnvAccessKey("AWS_ACCESS_KEY_ID", Minio.RootUser());
+	ScopedEnvVar EnvSecretKey("AWS_SECRET_ACCESS_KEY", Minio.RootPassword());
+
+	ScopedTemporaryDirectory TempDir;
+
+	std::filesystem::path ServerStateDir = TempDir.Path() / "server_state";
+	std::filesystem::path HydrationTemp	 = TempDir.Path() / "hydration_temp";
+	CreateDirectories(ServerStateDir);
+	CreateDirectories(HydrationTemp);
+
+	const std::string ModuleId = "s3test_folder_select";
+
+	HydrationConfig Config;
+	Config.ServerStateDir	   = ServerStateDir;
+	Config.TempDir			   = HydrationTemp;
+	Config.ModuleId			   = ModuleId;
+	Config.TargetSpecification = "s3://zen-hydration-test";
+	Config.S3Endpoint		   = Minio.Endpoint();
+	Config.S3PathStyle		   = true;
+
+	// v1: dehydrate without a marker file
+	CreateTestTree(ServerStateDir);
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Dehydrate();
+	}
+
+	// ServerStateDir is now empty. Wait briefly so the v2 timestamp folder name is strictly later
+	// (timestamp resolution is 1 ms).
+	Sleep(2);
+
+	// v2: dehydrate WITH a marker file that only v2 has
+	CreateTestTree(ServerStateDir);
+	WriteFile(ServerStateDir / "v2marker.bin", CreateSemiRandomBlob(64));
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Dehydrate();
+	}
+
+	// Hydrate must restore v2 (current-state.json points to the v2 folder)
+	CleanDirectory(ServerStateDir, true);
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Hydrate();
+	}
+
+	// v2 marker must be present - confirms current-state.json pointed to the v2 folder
+	CHECK(std::filesystem::exists(ServerStateDir / "v2marker.bin"));
+	// Subdirectory hierarchy must also be intact
+	CHECK(std::filesystem::exists(ServerStateDir / "subdir" / "file_b.bin"));
+	CHECK(std::filesystem::exists(ServerStateDir / "subdir" / "nested" / "file_c.bin"));
+}
+
+TEST_CASE("hydration.s3.module_isolation")
+{
+	// Two independent modules dehydrate/hydrate without interfering with each other.
+	// Uses VerifyTree with per-module byte content to detect cross-module data mixing.
+	MinioProcessOptions MinioOpts;
+	MinioOpts.Port = 19012;
+	MinioProcess Minio(MinioOpts);
+	Minio.SpawnMinioServer();
+	Minio.CreateBucket("zen-hydration-test");
+
+	ScopedEnvVar EnvAccessKey("AWS_ACCESS_KEY_ID", Minio.RootUser());
+	ScopedEnvVar EnvSecretKey("AWS_SECRET_ACCESS_KEY", Minio.RootPassword());
+
+	ScopedTemporaryDirectory TempDir;
+
+	struct ModuleData
+	{
+		HydrationConfig											Config;
+		std::vector<std::pair<std::filesystem::path, IoBuffer>> Files;
+	};
+
+	std::vector<ModuleData> Modules;
+	for (const char* ModuleId : {"s3test_iso_a", "s3test_iso_b"})
+	{
+		std::filesystem::path StateDir = TempDir.Path() / ModuleId / "state";
+		std::filesystem::path TempPath = TempDir.Path() / ModuleId / "temp";
+		CreateDirectories(StateDir);
+		CreateDirectories(TempPath);
+
+		ModuleData Data;
+		Data.Config.ServerStateDir		= StateDir;
+		Data.Config.TempDir				= TempPath;
+		Data.Config.ModuleId			= ModuleId;
+		Data.Config.TargetSpecification = "s3://zen-hydration-test";
+		Data.Config.S3Endpoint			= Minio.Endpoint();
+		Data.Config.S3PathStyle			= true;
+		Data.Files						= CreateTestTree(StateDir);
+
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Data.Config);
+		Hydrator->Dehydrate();
+
+		Modules.push_back(std::move(Data));
+	}
+
+	for (ModuleData& Module : Modules)
+	{
+		CleanDirectory(Module.Config.ServerStateDir, true);
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Module.Config);
+		Hydrator->Hydrate();
+
+		// Each module's files must be independently restorable with correct byte content.
+		// If S3 key prefixes were mixed up, CreateSemiRandomBlob content would differ.
+		VerifyTree(Module.Config.ServerStateDir, Module.Files);
+	}
+}
+
+// ---------------------------------------------------------------------------
+// S3Hydrator concurrent test
+// ---------------------------------------------------------------------------
+
+TEST_CASE("hydration.s3.concurrent")
+{
+	// N modules dehydrate and hydrate concurrently against MinIO.
+	// Each module has a distinct ModuleId, so S3 key prefixes don't overlap.
+	MinioProcessOptions MinioOpts;
+	MinioOpts.Port = 19013;
+	MinioProcess Minio(MinioOpts);
+	Minio.SpawnMinioServer();
+	Minio.CreateBucket("zen-hydration-test");
+
+	ScopedEnvVar EnvAccessKey("AWS_ACCESS_KEY_ID", Minio.RootUser());
+	ScopedEnvVar EnvSecretKey("AWS_SECRET_ACCESS_KEY", Minio.RootPassword());
+
+	constexpr int kModuleCount = 4;
+
+	ScopedTemporaryDirectory TempDir;
+
+	struct ModuleData
+	{
+		HydrationConfig											Config;
+		std::vector<std::pair<std::filesystem::path, IoBuffer>> Files;
+	};
+	std::vector<ModuleData> Modules(kModuleCount);
+
+	for (int I = 0; I < kModuleCount; ++I)
+	{
+		std::string			  ModuleId = fmt::format("s3_concurrent_{}", I);
+		std::filesystem::path StateDir = TempDir.Path() / ModuleId / "state";
+		std::filesystem::path TempPath = TempDir.Path() / ModuleId / "temp";
+		CreateDirectories(StateDir);
+		CreateDirectories(TempPath);
+
+		Modules[I].Config.ServerStateDir	  = StateDir;
+		Modules[I].Config.TempDir			  = TempPath;
+		Modules[I].Config.ModuleId			  = ModuleId;
+		Modules[I].Config.TargetSpecification = "s3://zen-hydration-test";
+		Modules[I].Config.S3Endpoint		  = Minio.Endpoint();
+		Modules[I].Config.S3PathStyle		  = true;
+		Modules[I].Files					  = CreateTestTree(StateDir);
+	}
+
+	// Concurrent dehydrate
+	{
+		WorkerThreadPool  Pool(kModuleCount, "hydration_s3_dehy");
+		std::atomic<bool> AbortFlag{false};
+		std::atomic<bool> PauseFlag{false};
+		ParallelWork	  Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+		for (int I = 0; I < kModuleCount; ++I)
+		{
+			Work.ScheduleWork(Pool, [&Config = Modules[I].Config](std::atomic<bool>&) {
+				std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+				Hydrator->Dehydrate();
+			});
+		}
+		Work.Wait();
+		CHECK_FALSE(Work.IsAborted());
+	}
+
+	// Concurrent hydrate
+	{
+		WorkerThreadPool  Pool(kModuleCount, "hydration_s3_hy");
+		std::atomic<bool> AbortFlag{false};
+		std::atomic<bool> PauseFlag{false};
+		ParallelWork	  Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+		for (int I = 0; I < kModuleCount; ++I)
+		{
+			Work.ScheduleWork(Pool, [&Config = Modules[I].Config](std::atomic<bool>&) {
+				CleanDirectory(Config.ServerStateDir, true);
+				std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+				Hydrator->Hydrate();
+			});
+		}
+		Work.Wait();
+		CHECK_FALSE(Work.IsAborted());
+	}
+
+	// Verify all modules restored correctly
+	for (int I = 0; I < kModuleCount; ++I)
+	{
+		VerifyTree(Modules[I].Config.ServerStateDir, Modules[I].Files);
+	}
+}
+
+// ---------------------------------------------------------------------------
+// S3Hydrator: no prior state (first-boot path)
+// ---------------------------------------------------------------------------
+
+TEST_CASE("hydration.s3.no_prior_state")
+{
+	// Hydrate() against an empty bucket (first-boot scenario) must leave ServerStateDir empty.
+	// The "No state found in S3" path goes through the error-cleanup branch, which wipes
+	// ServerStateDir to ensure no partial or stale content is left for the server to start on.
+	MinioProcessOptions MinioOpts;
+	MinioOpts.Port = 19014;
+	MinioProcess Minio(MinioOpts);
+	Minio.SpawnMinioServer();
+	Minio.CreateBucket("zen-hydration-test");
+
+	ScopedEnvVar EnvAccessKey("AWS_ACCESS_KEY_ID", Minio.RootUser());
+	ScopedEnvVar EnvSecretKey("AWS_SECRET_ACCESS_KEY", Minio.RootPassword());
+
+	ScopedTemporaryDirectory TempDir;
+
+	std::filesystem::path ServerStateDir = TempDir.Path() / "server_state";
+	std::filesystem::path HydrationTemp	 = TempDir.Path() / "hydration_temp";
+	CreateDirectories(ServerStateDir);
+	CreateDirectories(HydrationTemp);
+
+	// Pre-populate ServerStateDir to confirm the wipe actually runs.
+	WriteFile(ServerStateDir / "stale.bin", CreateSemiRandomBlob(256));
+
+	HydrationConfig Config;
+	Config.ServerStateDir	   = ServerStateDir;
+	Config.TempDir			   = HydrationTemp;
+	Config.ModuleId			   = "s3test_no_prior";
+	Config.TargetSpecification = "s3://zen-hydration-test";
+	Config.S3Endpoint		   = Minio.Endpoint();
+	Config.S3PathStyle		   = true;
+
+	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+	Hydrator->Hydrate();
+
+	// ServerStateDir must be empty: the error path wipes it to prevent a server start
+	// against stale or partially-installed content.
+	CHECK(std::filesystem::is_empty(ServerStateDir));
+}
+
+// ---------------------------------------------------------------------------
+// S3Hydrator: bucket path prefix in TargetSpecification
+// ---------------------------------------------------------------------------
+
+TEST_CASE("hydration.s3.path_prefix")
+{
+	// TargetSpecification of the form "s3://bucket/some/prefix" stores objects under
+	// "some/prefix/<ModuleId>/..." rather than directly under "<ModuleId>/...".
+	// Tests the second branch of the m_KeyPrefix calculation in S3Hydrator::Configure().
+	MinioProcessOptions MinioOpts;
+	MinioOpts.Port = 19015;
+	MinioProcess Minio(MinioOpts);
+	Minio.SpawnMinioServer();
+	Minio.CreateBucket("zen-hydration-test");
+
+	ScopedEnvVar EnvAccessKey("AWS_ACCESS_KEY_ID", Minio.RootUser());
+	ScopedEnvVar EnvSecretKey("AWS_SECRET_ACCESS_KEY", Minio.RootPassword());
+
+	ScopedTemporaryDirectory TempDir;
+
+	std::filesystem::path ServerStateDir = TempDir.Path() / "server_state";
+	std::filesystem::path HydrationTemp	 = TempDir.Path() / "hydration_temp";
+	CreateDirectories(ServerStateDir);
+	CreateDirectories(HydrationTemp);
+
+	std::vector<std::pair<std::filesystem::path, IoBuffer>> TestFiles = CreateTestTree(ServerStateDir);
+
+	HydrationConfig Config;
+	Config.ServerStateDir	   = ServerStateDir;
+	Config.TempDir			   = HydrationTemp;
+	Config.ModuleId			   = "s3test_prefix";
+	Config.TargetSpecification = "s3://zen-hydration-test/team/project";
+	Config.S3Endpoint		   = Minio.Endpoint();
+	Config.S3PathStyle		   = true;
+
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Dehydrate();
+	}
+
+	CleanDirectory(ServerStateDir, true);
+
+	{
+		std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
+		Hydrator->Hydrate();
+	}
+
+	VerifyTree(ServerStateDir, TestFiles);
+}
+
+TEST_SUITE_END();
+
+void
+hydration_forcelink()
+{
+}
+
+#endif	// ZEN_WITH_TESTS
+
 }  // namespace zen
diff --git a/src/zenserver/hub/hydration.h b/src/zenserver/hub/hydration.h
index fb12e7f7c..d29ffe5c0 100644
--- a/src/zenserver/hub/hydration.h
+++ b/src/zenserver/hub/hydration.h
@@ -16,6 +16,12 @@ struct HydrationConfig
 	std::string ModuleId;
 	// Back-end specific target specification (e.g. S3 bucket, file path, etc)
 	std::string TargetSpecification;
+
+	// Optional S3 endpoint override (e.g. "http://localhost:9000" for MinIO).
+	std::string S3Endpoint;
+	// Use path-style S3 URLs (endpoint/bucket/key) instead of virtual-hosted-style
+	// (bucket.endpoint/key). Required for MinIO and other non-AWS endpoints.
+	bool S3PathStyle = false;
 };
 
 /**
@@ -37,4 +43,8 @@ struct HydrationStrategyBase
 
 std::unique_ptr<HydrationStrategyBase> CreateHydrator(const HydrationConfig& Config);
 
+#if ZEN_WITH_TESTS
+void hydration_forcelink();
+#endif	// ZEN_WITH_TESTS
+
 }  // namespace zen