incremental dehydrate (#921)

- Feature: Incremental CAS-based hydration/dehydration replacing the previous full-copy approach
- Feature: S3 hydration backend with multipart upload/download support
- Feature: Configurable thread pools for hub instance provisioning and hydration
  `--hub-instance-provision-threads` defaults to `max(cpu_count / 4, 2)`. Set to 0 for synchronous operation.
  `--hub-hydration-threads` defaults to `max(cpu_count / 4, 2)`. Set to 0 for synchronous operation.
- Improvement: Hub triggers GC on instance before deprovisioning to compact storage before dehydration
- Improvement: GC status now reports pending triggers as running
- Improvement: S3 client debug logging gated behind verbose mode to reduce log noise at default verbosity
- Improvement: Hub dashboard Resources tile now shows total memory
- Improvement: `filesystemutils` moved from `zenremotestore` to `zenutil` for broader reuse
- Improvement: Hub uses separate provision and hydration worker pools to avoid deadlocks
- Improvement: Hibernate/wake/deprovision on non-existent or already-in-target-state modules are idempotent
- Improvement: `ScopedTemporaryDirectory` with empty path now creates a temporary directory instead of asserting

1 files changed, 18 insertions, 4 deletions

diff --git a/src/zenserver/hub/storageserverinstance.cpp b/src/zenserver/hub/storageserverinstance.cpp
index 0c9354990..6185a7f19 100644
--- a/src/zenserver/hub/storageserverinstance.cpp
+++ b/src/zenserver/hub/storageserverinstance.cpp
@@ -154,29 +154,43 @@ StorageServerInstance::WakeLocked()
 void
 StorageServerInstance::Hydrate()
 {
+	std::atomic<bool> AbortFlag{false};
+	std::atomic<bool> PauseFlag{false};
+
 	HydrationConfig Config{.ServerStateDir		= m_BaseDir,
 						   .TempDir				= m_TempDir,
 						   .ModuleId			= m_ModuleId,
 						   .TargetSpecification = m_Config.HydrationTargetSpecification,
 						   .Options				= m_Config.HydrationOptions};
+	if (m_Config.OptionalWorkerPool)
+	{
+		Config.Threading.emplace(
+			HydrationConfig::ThreadingOptions{.WorkerPool = m_Config.OptionalWorkerPool, .AbortFlag = &AbortFlag, .PauseFlag = &PauseFlag});
+	}
 
 	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
-
-	Hydrator->Hydrate();
+	m_HydrationState								= Hydrator->Hydrate();
 }
 
 void
 StorageServerInstance::Dehydrate()
 {
+	std::atomic<bool> AbortFlag{false};
+	std::atomic<bool> PauseFlag{false};
+
 	HydrationConfig Config{.ServerStateDir		= m_BaseDir,
 						   .TempDir				= m_TempDir,
 						   .ModuleId			= m_ModuleId,
 						   .TargetSpecification = m_Config.HydrationTargetSpecification,
 						   .Options				= m_Config.HydrationOptions};
+	if (m_Config.OptionalWorkerPool)
+	{
+		Config.Threading.emplace(
+			HydrationConfig::ThreadingOptions{.WorkerPool = m_Config.OptionalWorkerPool, .AbortFlag = &AbortFlag, .PauseFlag = &PauseFlag});
+	}
 
 	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateHydrator(Config);
-
-	Hydrator->Dehydrate();
+	Hydrator->Dehydrate(m_HydrationState);
 }
 
 StorageServerInstance::SharedLockedPtr::SharedLockedPtr() : m_Lock(nullptr), m_Instance(nullptr)