incremental dehydrate (#921)

- Feature: Incremental CAS-based hydration/dehydration replacing the previous full-copy approach
- Feature: S3 hydration backend with multipart upload/download support
- Feature: Configurable thread pools for hub instance provisioning and hydration
  `--hub-instance-provision-threads` defaults to `max(cpu_count / 4, 2)`. Set to 0 for synchronous operation.
  `--hub-hydration-threads` defaults to `max(cpu_count / 4, 2)`. Set to 0 for synchronous operation.
- Improvement: Hub triggers GC on instance before deprovisioning to compact storage before dehydration
- Improvement: GC status now reports pending triggers as running
- Improvement: S3 client debug logging gated behind verbose mode to reduce log noise at default verbosity
- Improvement: Hub dashboard Resources tile now shows total memory
- Improvement: `filesystemutils` moved from `zenremotestore` to `zenutil` for broader reuse
- Improvement: Hub uses separate provision and hydration worker pools to avoid deadlocks
- Improvement: Hibernate/wake/deprovision on non-existent or already-in-target-state modules are idempotent
- Improvement: `ScopedTemporaryDirectory` with empty path now creates a temporary directory instead of asserting

1 files changed, 57 insertions, 17 deletions

diff --git a/src/zenutil/cloud/s3client.cpp b/src/zenutil/cloud/s3client.cpp
index d9fde05d9..ef4b344ce 100644
--- a/src/zenutil/cloud/s3client.cpp
+++ b/src/zenutil/cloud/s3client.cpp
@@ -148,6 +148,7 @@ S3Client::S3Client(const S3ClientOptions& Options)
 , m_Credentials(Options.Credentials)
 , m_CredentialProvider(Options.CredentialProvider)
 , m_HttpClient(BuildEndpoint(), Options.HttpSettings)
+, m_Verbose(Options.HttpSettings.Verbose)
 {
 	m_Host = BuildHostHeader();
 	ZEN_INFO("S3 client configured for bucket '{}' in region '{}' (endpoint: {}, {})",
@@ -338,7 +339,10 @@ S3Client::PutObject(std::string_view Key, IoBuffer Content)
 		return S3Result{std::move(Err)};
 	}
 
-	ZEN_DEBUG("S3 PUT '{}' succeeded ({} bytes)", Key, Content.GetSize());
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 PUT '{}' succeeded ({} bytes)", Key, Content.GetSize());
+	}
 	return {};
 }
 
@@ -362,7 +366,10 @@ S3Client::GetObject(std::string_view Key, const std::filesystem::path& TempFileP
 		return S3GetObjectResult{S3Result{std::move(Err)}, {}};
 	}
 
-	ZEN_DEBUG("S3 GET '{}' succeeded ({} bytes)", Key, Response.ResponsePayload.GetSize());
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 GET '{}' succeeded ({} bytes)", Key, Response.ResponsePayload.GetSize());
+	}
 	return S3GetObjectResult{{}, std::move(Response.ResponsePayload)};
 }
 
@@ -403,11 +410,14 @@ S3Client::GetObjectRange(std::string_view Key, uint64_t RangeStart, uint64_t Ran
 		return S3GetObjectResult{S3Result{std::move(Err)}, {}};
 	}
 
-	ZEN_DEBUG("S3 GET range '{}' [{}-{}] succeeded ({} bytes)",
-			  Key,
-			  RangeStart,
-			  RangeStart + RangeSize - 1,
-			  Response.ResponsePayload.GetSize());
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 GET range '{}' [{}-{}] succeeded ({} bytes)",
+				 Key,
+				 RangeStart,
+				 RangeStart + RangeSize - 1,
+				 Response.ResponsePayload.GetSize());
+	}
 	return S3GetObjectResult{{}, std::move(Response.ResponsePayload)};
 }
 
@@ -426,7 +436,10 @@ S3Client::DeleteObject(std::string_view Key)
 		return S3Result{std::move(Err)};
 	}
 
-	ZEN_DEBUG("S3 DELETE '{}' succeeded", Key);
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 DELETE '{}' succeeded", Key);
+	}
 	return {};
 }
 
@@ -468,7 +481,10 @@ S3Client::HeadObject(std::string_view Key)
 		Info.LastModified = *V;
 	}
 
-	ZEN_DEBUG("S3 HEAD '{}' succeeded (size={})", Key, Info.Size);
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 HEAD '{}' succeeded (size={})", Key, Info.Size);
+	}
 	return S3HeadObjectResult{{}, std::move(Info), HeadObjectResult::Found};
 }
 
@@ -565,10 +581,16 @@ S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys)
 		}
 
 		ContinuationToken = std::string(NextToken);
-		ZEN_DEBUG("S3 ListObjectsV2 prefix='{}' fetching next page ({} objects so far)", Prefix, Result.Objects.size());
+		if (m_Verbose)
+		{
+			ZEN_INFO("S3 ListObjectsV2 prefix='{}' fetching next page ({} objects so far)", Prefix, Result.Objects.size());
+		}
 	}
 
-	ZEN_DEBUG("S3 ListObjectsV2 prefix='{}' returned {} objects", Prefix, Result.Objects.size());
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 ListObjectsV2 prefix='{}' returned {} objects", Prefix, Result.Objects.size());
+	}
 	return Result;
 }
 
@@ -607,7 +629,10 @@ S3Client::CreateMultipartUpload(std::string_view Key)
 		return S3CreateMultipartUploadResult{S3Result{std::move(Err)}, {}};
 	}
 
-	ZEN_DEBUG("S3 CreateMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 CreateMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+	}
 	return S3CreateMultipartUploadResult{{}, std::string(UploadId)};
 }
 
@@ -642,7 +667,10 @@ S3Client::UploadPart(std::string_view Key, std::string_view UploadId, uint32_t P
 		return S3UploadPartResult{S3Result{std::move(Err)}, {}};
 	}
 
-	ZEN_DEBUG("S3 UploadPart '{}' part {} succeeded ({} bytes, etag={})", Key, PartNumber, Content.GetSize(), *ETag);
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 UploadPart '{}' part {} succeeded ({} bytes, etag={})", Key, PartNumber, Content.GetSize(), *ETag);
+	}
 	return S3UploadPartResult{{}, *ETag};
 }
 
@@ -691,7 +719,10 @@ S3Client::CompleteMultipartUpload(std::string_view									   Key,
 		return S3Result{std::move(Err)};
 	}
 
-	ZEN_DEBUG("S3 CompleteMultipartUpload '{}' succeeded ({} parts)", Key, PartETags.size());
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 CompleteMultipartUpload '{}' succeeded ({} parts)", Key, PartETags.size());
+	}
 	return {};
 }
 
@@ -712,7 +743,10 @@ S3Client::AbortMultipartUpload(std::string_view Key, std::string_view UploadId)
 		return S3Result{std::move(Err)};
 	}
 
-	ZEN_DEBUG("S3 AbortMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 AbortMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+	}
 	return {};
 }
 
@@ -755,7 +789,10 @@ S3Client::PutObjectMultipart(std::string_view										 Key,
 		return PutObject(Key, TotalSize > 0 ? FetchRange(0, TotalSize) : IoBuffer{});
 	}
 
-	ZEN_DEBUG("S3 multipart upload '{}': {} bytes in ~{} parts", Key, TotalSize, (TotalSize + PartSize - 1) / PartSize);
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 multipart upload '{}': {} bytes in ~{} parts", Key, TotalSize, (TotalSize + PartSize - 1) / PartSize);
+	}
 
 	S3CreateMultipartUploadResult InitResult = CreateMultipartUpload(Key);
 	if (!InitResult)
@@ -803,7 +840,10 @@ S3Client::PutObjectMultipart(std::string_view										 Key,
 		throw;
 	}
 
-	ZEN_DEBUG("S3 multipart upload '{}' completed ({} parts, {} bytes)", Key, PartETags.size(), TotalSize);
+	if (m_Verbose)
+	{
+		ZEN_INFO("S3 multipart upload '{}' completed ({} parts, {} bytes)", Key, PartETags.size(), TotalSize);
+	}
 	return {};
 }