hydration with pack (#1016)

- Feature: Hub hydration packs small files into raw CAS pack blobs to reduce request count for modules dominated by tiny metadata files
  - `--hub-hydration-enable-pack` (Lua: `hub.hydration.enablepack`, default true)
  - `--hub-hydration-pack-threshold-bytes` (Lua: `hub.hydration.packthresholdbytes`, default 256 KiB)
  - `--hub-hydration-max-pack-bytes` (Lua: `hub.hydration.maxpackbytes`, default 4 MiB)
- Feature: Hub hydration and dehydration can be disabled per direction
  - `--hub-enable-hydration` (Lua: `hub.enablehydration`, default true)
  - `--hub-enable-dehydration` (Lua: `hub.enabledehydration`, default true)
- Feature: Hub hydration accepts a configurable file exclude list via `HydrationOptions` `excludes` (array of wildcards). Built-in defaults skip transient runtime files (`.lock`, `.sentry-native/*`, `state_marker`, `*.bak`, `gc/reserve.gc`, `auth/*`) so they no longer participate in dehydrate scans. Override semantics: a present field replaces the default outright; explicit `[]` opts out of all defaults.
- Improvement: Hub hydration completion logs now report per-request average and max latency, peak in-flight workers, queue wait, and hash-cache hit percentage; loose and pack-blob transfers are reported separately
- Improvement: Hub hydration pre-creates unique parent directories before scheduling parallel writes
- Improvement: S3 hydration retries transient HTTP failures (timeouts, 429 throttling, 5xx server errors, connection errors) up to 3 times via the HTTP client retry layer
- Improvement: S3 hydration multipart chunk size is persisted in `state.cbo` per module so hydrate replays the partitioning used at dehydrate; default raised to 64 MiB (was 32 MiB)
- Improvement: Hub hydration `Obliterate` retries backend delete once before falling back to local cleanup

1 files changed, 24 insertions, 0 deletions

diff --git a/src/zenserver/hub/storageserverinstance.cpp b/src/zenserver/hub/storageserverinstance.cpp
index 9d477fb10..8d36e6a46 100644
--- a/src/zenserver/hub/storageserverinstance.cpp
+++ b/src/zenserver/hub/storageserverinstance.cpp
@@ -10,6 +10,7 @@
 #include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zencore/timer.h>
+#include <zencore/trace.h>
 
 namespace zen {
 
@@ -31,6 +32,7 @@ StorageServerInstance::~StorageServerInstance()
 void
 StorageServerInstance::SpawnServerProcess()
 {
+	ZEN_TRACE_CPU("StorageServerInstance::SpawnServerProcess");
 	Stopwatch SpawnTimer;
 
 	ZEN_ASSERT_FORMAT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId);
@@ -103,6 +105,7 @@ StorageServerInstance::ShutdownServerProcess()
 	{
 		return;
 	}
+	ZEN_TRACE_CPU("StorageServerInstance::ShutdownServerProcess");
 	Stopwatch ShutdownTimer;
 	// m_ServerInstance.Shutdown() never throws.
 	m_ServerInstance.Shutdown();
@@ -131,6 +134,7 @@ StorageServerInstance::ProvisionLocked()
 		return;
 	}
 
+	ZEN_TRACE_CPU("StorageServerInstance::ProvisionLocked");
 	ZEN_INFO("Provisioning storage server instance for module '{}', at '{}'", m_ModuleId, m_Config.StateDir);
 	try
 	{
@@ -150,6 +154,7 @@ StorageServerInstance::ProvisionLocked()
 void
 StorageServerInstance::DeprovisionLocked()
 {
+	ZEN_TRACE_CPU("StorageServerInstance::DeprovisionLocked");
 	ShutdownServerProcess();
 
 	// Crashed or Hibernated: process already dead; skip Shutdown.
@@ -169,6 +174,7 @@ StorageServerInstance::DeprovisionLocked()
 void
 StorageServerInstance::ObliterateLocked()
 {
+	ZEN_TRACE_CPU("StorageServerInstance::ObliterateLocked");
 	ShutdownServerProcess();
 
 	std::atomic<bool>					   AbortFlag{false};
@@ -181,6 +187,7 @@ StorageServerInstance::ObliterateLocked()
 void
 StorageServerInstance::HibernateLocked()
 {
+	ZEN_TRACE_CPU("StorageServerInstance::HibernateLocked");
 	// Signal server to shut down, but keep data around for later wake
 	ShutdownServerProcess();
 }
@@ -195,6 +202,8 @@ StorageServerInstance::WakeLocked()
 		return;
 	}
 
+	ZEN_TRACE_CPU("StorageServerInstance::WakeLocked");
+
 	try
 	{
 		SpawnServerProcess();
@@ -212,6 +221,12 @@ StorageServerInstance::WakeLocked()
 void
 StorageServerInstance::Hydrate()
 {
+	if (!m_Config.EnableHydration)
+	{
+		ZEN_INFO("Hydration disabled; skipping hydrate for module '{}'", m_ModuleId);
+		return;
+	}
+	ZEN_TRACE_CPU("StorageServerInstance::Hydrate");
 	std::atomic<bool>					   AbortFlag{false};
 	std::atomic<bool>					   PauseFlag{false};
 	HydrationConfig						   Config	= MakeHydrationConfig(AbortFlag, PauseFlag);
@@ -222,6 +237,12 @@ StorageServerInstance::Hydrate()
 void
 StorageServerInstance::Dehydrate()
 {
+	if (!m_Config.EnableDehydration)
+	{
+		ZEN_INFO("Dehydration disabled; skipping dehydrate for module '{}'", m_ModuleId);
+		return;
+	}
+	ZEN_TRACE_CPU("StorageServerInstance::Dehydrate");
 	std::atomic<bool>					   AbortFlag{false};
 	std::atomic<bool>					   PauseFlag{false};
 	HydrationConfig						   Config	= MakeHydrationConfig(AbortFlag, PauseFlag);
@@ -238,6 +259,9 @@ StorageServerInstance::MakeHydrationConfig(std::atomic<bool>& AbortFlag, std::at
 		Config.Threading.emplace(
 			HydrationConfig::ThreadingOptions{.WorkerPool = m_Config.OptionalWorkerPool, .AbortFlag = &AbortFlag, .PauseFlag = &PauseFlag});
 	}
+	Config.PackEnabled		  = m_Config.HydrationPackEnabled;
+	Config.PackThresholdBytes = m_Config.HydrationPackThresholdBytes;
+	Config.MaxPackBytes		  = m_Config.HydrationMaxPackBytes;
 
 	return Config;
 }