hydration with pack (#1016)

- Feature: Hub hydration packs small files into raw CAS pack blobs to reduce request count for modules dominated by tiny metadata files
  - `--hub-hydration-enable-pack` (Lua: `hub.hydration.enablepack`, default true)
  - `--hub-hydration-pack-threshold-bytes` (Lua: `hub.hydration.packthresholdbytes`, default 256 KiB)
  - `--hub-hydration-max-pack-bytes` (Lua: `hub.hydration.maxpackbytes`, default 4 MiB)
- Feature: Hub hydration and dehydration can be disabled per direction
  - `--hub-enable-hydration` (Lua: `hub.enablehydration`, default true)
  - `--hub-enable-dehydration` (Lua: `hub.enabledehydration`, default true)
- Feature: Hub hydration accepts a configurable file exclude list via `HydrationOptions` `excludes` (array of wildcards). Built-in defaults skip transient runtime files (`.lock`, `.sentry-native/*`, `state_marker`, `*.bak`, `gc/reserve.gc`, `auth/*`) so they no longer participate in dehydrate scans. Override semantics: a present field replaces the default outright; explicit `[]` opts out of all defaults.
- Improvement: Hub hydration completion logs now report per-request average and max latency, peak in-flight workers, queue wait, and hash-cache hit percentage; loose and pack-blob transfers are reported separately
- Improvement: Hub hydration pre-creates unique parent directories before scheduling parallel writes
- Improvement: S3 hydration retries transient HTTP failures (timeouts, 429 throttling, 5xx server errors, connection errors) up to 3 times via the HTTP client retry layer
- Improvement: S3 hydration multipart chunk size is persisted in `state.cbo` per module so hydrate replays the partitioning used at dehydrate; default raised to 64 MiB (was 32 MiB)
- Improvement: Hub hydration `Obliterate` retries backend delete once before falling back to local cleanup

1 files changed, 51 insertions, 1 deletions

diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
index ebc2cf2f1..27c1c9fc4 100644
--- a/src/zenserver/hub/zenhubserver.cpp
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -235,6 +235,44 @@ ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options)
 
 	Options.add_option("hub",
 					   "",
+					   "hub-enable-hydration",
+					   "Load instance state from the hydration target on provision (default true)",
+					   cxxopts::value<bool>(m_ServerOptions.HubEnableHydration)->default_value("true"),
+					   "");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-enable-dehydration",
+					   "Save instance state to the hydration target on deprovision (default true)",
+					   cxxopts::value<bool>(m_ServerOptions.HubEnableDehydration)->default_value("true"),
+					   "");
+
+	Options.add_option(
+		"hub",
+		"",
+		"hub-hydration-enable-pack",
+		"Concatenate small files into raw CAS pack blobs during dehydrate (default true; see --hub-hydration-pack-threshold-bytes)",
+		cxxopts::value<bool>(m_ServerOptions.HubHydrationPackEnabled)->default_value("true"),
+		"");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-hydration-pack-threshold-bytes",
+					   fmt::format("Files strictly smaller than this are pack candidates (default {})", DefaultPackThresholdBytes),
+					   cxxopts::value<uint64_t>(m_ServerOptions.HubHydrationPackThresholdBytes)
+						   ->default_value(fmt::format("{}", DefaultPackThresholdBytes)),
+					   "<bytes>");
+
+	Options.add_option(
+		"hub",
+		"",
+		"hub-hydration-max-pack-bytes",
+		fmt::format("Upper bound on a pack's concatenation size (default {})", DefaultMaxPackBytes),
+		cxxopts::value<uint64_t>(m_ServerOptions.HubHydrationMaxPackBytes)->default_value(fmt::format("{}", DefaultMaxPackBytes)),
+		"<bytes>");
+
+	Options.add_option("hub",
+					   "",
 					   "hub-watchdog-cycle-interval-ms",
 					   "Interval between watchdog cycles in milliseconds",
 					   cxxopts::value<uint32_t>(m_ServerOptions.WatchdogConfig.CycleIntervalMs)->default_value("3000"),
@@ -367,6 +405,13 @@ ZenHubServerConfigurator::AddConfigOptions(LuaConfig::Options& Options)
 	Options.AddOption("hub.hydration.targetspec"sv, m_ServerOptions.HydrationTargetSpecification, "hub-hydration-target-spec"sv);
 	Options.AddOption("hub.hydration.targetconfig"sv, m_ServerOptions.HydrationTargetConfigPath, "hub-hydration-target-config"sv);
 	Options.AddOption("hub.hydration.threads"sv, m_ServerOptions.HubHydrationThreadCount, "hub-hydration-threads"sv);
+	Options.AddOption("hub.enablehydration"sv, m_ServerOptions.HubEnableHydration, "hub-enable-hydration"sv);
+	Options.AddOption("hub.enabledehydration"sv, m_ServerOptions.HubEnableDehydration, "hub-enable-dehydration"sv);
+	Options.AddOption("hub.hydration.enablepack"sv, m_ServerOptions.HubHydrationPackEnabled, "hub-hydration-enable-pack"sv);
+	Options.AddOption("hub.hydration.packthresholdbytes"sv,
+					  m_ServerOptions.HubHydrationPackThresholdBytes,
+					  "hub-hydration-pack-threshold-bytes"sv);
+	Options.AddOption("hub.hydration.maxpackbytes"sv, m_ServerOptions.HubHydrationMaxPackBytes, "hub-hydration-max-pack-bytes"sv);
 
 	Options.AddOption("hub.watchdog.cycleintervalms"sv, m_ServerOptions.WatchdogConfig.CycleIntervalMs, "hub-watchdog-cycle-interval-ms"sv);
 	Options.AddOption("hub.watchdog.cycleprocessingbudgetms"sv,
@@ -453,7 +498,7 @@ ZenHubServer::OnModuleStateChanged(std::string_view					 HubInstanceId,
 								   HubInstanceState					 PreviousState,
 								   HubInstanceState					 NewState)
 {
-	ZEN_UNUSED(PreviousState);
+	ZEN_INFO("Module '{}' changed state from {} to {}", ModuleId, zen::ToString(PreviousState), zen::ToString(NewState));
 
 	if (NewState == HubInstanceState::Deprovisioning || NewState == HubInstanceState::Hibernating)
 	{
@@ -661,6 +706,11 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig)
 		.InstanceTraceFile			  = ServerConfig.HubInstanceTraceFile,
 		.InstanceConfigPath			  = ServerConfig.HubInstanceConfigPath,
 		.HydrationTargetSpecification = ServerConfig.HydrationTargetSpecification,
+		.EnableHydration			  = ServerConfig.HubEnableHydration,
+		.EnableDehydration			  = ServerConfig.HubEnableDehydration,
+		.HydrationPackEnabled		  = ServerConfig.HubHydrationPackEnabled,
+		.HydrationPackThresholdBytes  = ServerConfig.HubHydrationPackThresholdBytes,
+		.HydrationMaxPackBytes		  = ServerConfig.HubHydrationMaxPackBytes,
 		.WatchDog =
 			{
 				.CycleInterval				  = std::chrono::milliseconds(ServerConfig.WatchdogConfig.CycleIntervalMs),