aboutsummaryrefslogtreecommitdiff
path: root/src/zenutil/cloud
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenutil/cloud')
-rw-r--r--src/zenutil/cloud/imdscredentials.cpp8
-rw-r--r--src/zenutil/cloud/minioprocess.cpp4
-rw-r--r--src/zenutil/cloud/mockimds.cpp6
-rw-r--r--src/zenutil/cloud/s3client.cpp168
4 files changed, 158 insertions, 28 deletions
diff --git a/src/zenutil/cloud/imdscredentials.cpp b/src/zenutil/cloud/imdscredentials.cpp
index dde1dc019..433afdc3c 100644
--- a/src/zenutil/cloud/imdscredentials.cpp
+++ b/src/zenutil/cloud/imdscredentials.cpp
@@ -115,7 +115,7 @@ ImdsCredentialProvider::FetchToken()
HttpClient::KeyValueMap Headers;
Headers->emplace("X-aws-ec2-metadata-token-ttl-seconds", "21600");
- HttpClient::Response Response = m_HttpClient.Put("/latest/api/token", Headers);
+ HttpClient::Response Response = m_HttpClient.Put("/latest/api/token", IoBuffer{}, Headers);
if (!Response.IsSuccess())
{
ZEN_WARN("IMDS token request failed: {}", Response.ErrorMessage("PUT /latest/api/token"));
@@ -213,7 +213,7 @@ ImdsCredentialProvider::FetchCredentials()
}
else
{
- // Expiration is in the past or unparseable — force refresh next time
+ // Expiration is in the past or unparseable - force refresh next time
NewExpiresAt = std::chrono::steady_clock::now();
}
@@ -226,7 +226,7 @@ ImdsCredentialProvider::FetchCredentials()
if (KeyChanged)
{
- ZEN_INFO("IMDS credentials refreshed (AccessKeyId: {}...)", m_CachedCredentials.AccessKeyId.substr(0, 8));
+ ZEN_INFO("IMDS credentials refreshed (AccessKeyId: {})", HideSensitiveString(m_CachedCredentials.AccessKeyId));
}
else
{
@@ -369,7 +369,7 @@ TEST_CASE("imdscredentials.fetch_from_mock")
TEST_CASE("imdscredentials.unreachable_endpoint")
{
- // Point at a non-existent server — should return empty credentials, not crash
+ // Point at a non-existent server - should return empty credentials, not crash
ImdsCredentialProviderOptions Opts;
Opts.Endpoint = "http://127.0.0.1:1"; // unlikely to have anything listening
Opts.ConnectTimeout = std::chrono::milliseconds(100);
diff --git a/src/zenutil/cloud/minioprocess.cpp b/src/zenutil/cloud/minioprocess.cpp
index 457453bd8..2db0010dc 100644
--- a/src/zenutil/cloud/minioprocess.cpp
+++ b/src/zenutil/cloud/minioprocess.cpp
@@ -45,7 +45,7 @@ struct MinioProcess::Impl
}
CreateProcOptions Options;
- Options.Flags |= CreateProcOptions::Flag_Windows_NewProcessGroup;
+ Options.Flags |= CreateProcOptions::Flag_NewProcessGroup;
Options.Environment.emplace_back("MINIO_ROOT_USER", m_Options.RootUser);
Options.Environment.emplace_back("MINIO_ROOT_PASSWORD", m_Options.RootPassword);
@@ -102,7 +102,7 @@ struct MinioProcess::Impl
{
if (m_DataDir.empty())
{
- ZEN_WARN("MinIO: Cannot create bucket before data directory is initialized — call SpawnMinioServer() first");
+ ZEN_WARN("MinIO: Cannot create bucket before data directory is initialized - call SpawnMinioServer() first");
return;
}
diff --git a/src/zenutil/cloud/mockimds.cpp b/src/zenutil/cloud/mockimds.cpp
index 6919fab4d..88b348ed6 100644
--- a/src/zenutil/cloud/mockimds.cpp
+++ b/src/zenutil/cloud/mockimds.cpp
@@ -93,7 +93,7 @@ MockImdsService::HandleAwsRequest(HttpServerRequest& Request)
return;
}
- // Autoscaling lifecycle state — 404 when not in an ASG
+ // Autoscaling lifecycle state - 404 when not in an ASG
if (Uri == "latest/meta-data/autoscaling/target-lifecycle-state")
{
if (Aws.AutoscalingState.empty())
@@ -105,7 +105,7 @@ MockImdsService::HandleAwsRequest(HttpServerRequest& Request)
return;
}
- // Spot interruption notice — 404 when no interruption pending
+ // Spot interruption notice - 404 when no interruption pending
if (Uri == "latest/meta-data/spot/instance-action")
{
if (Aws.SpotAction.empty())
@@ -117,7 +117,7 @@ MockImdsService::HandleAwsRequest(HttpServerRequest& Request)
return;
}
- // IAM role discovery — returns the role name
+ // IAM role discovery - returns the role name
if (Uri == "latest/meta-data/iam/security-credentials/")
{
if (Aws.IamRoleName.empty())
diff --git a/src/zenutil/cloud/s3client.cpp b/src/zenutil/cloud/s3client.cpp
index d9fde05d9..3d6dca562 100644
--- a/src/zenutil/cloud/s3client.cpp
+++ b/src/zenutil/cloud/s3client.cpp
@@ -148,6 +148,7 @@ S3Client::S3Client(const S3ClientOptions& Options)
, m_Credentials(Options.Credentials)
, m_CredentialProvider(Options.CredentialProvider)
, m_HttpClient(BuildEndpoint(), Options.HttpSettings)
+, m_Verbose(Options.HttpSettings.Verbose)
{
m_Host = BuildHostHeader();
ZEN_INFO("S3 client configured for bucket '{}' in region '{}' (endpoint: {}, {})",
@@ -285,7 +286,11 @@ S3Client::GetSigningKey(std::string_view DateStamp)
}
HttpClient::KeyValueMap
-S3Client::SignRequest(std::string_view Method, std::string_view Path, std::string_view CanonicalQueryString, std::string_view PayloadHash)
+S3Client::SignRequest(std::string_view Method,
+ std::string_view Path,
+ std::string_view CanonicalQueryString,
+ std::string_view PayloadHash,
+ std::span<const std::pair<std::string, std::string>> ExtraSignedHeaders)
{
SigV4Credentials Credentials = GetCurrentCredentials();
@@ -293,6 +298,7 @@ S3Client::SignRequest(std::string_view Method, std::string_view Path, std::strin
// Build sorted headers to sign (must be sorted by lowercase name)
std::vector<std::pair<std::string, std::string>> HeadersToSign;
+ HeadersToSign.reserve(4 + ExtraSignedHeaders.size());
HeadersToSign.emplace_back("host", m_Host);
HeadersToSign.emplace_back("x-amz-content-sha256", std::string(PayloadHash));
HeadersToSign.emplace_back("x-amz-date", AmzDate);
@@ -300,6 +306,10 @@ S3Client::SignRequest(std::string_view Method, std::string_view Path, std::strin
{
HeadersToSign.emplace_back("x-amz-security-token", Credentials.SessionToken);
}
+ for (const auto& [K, V] : ExtraSignedHeaders)
+ {
+ HeadersToSign.emplace_back(K, V);
+ }
std::sort(HeadersToSign.begin(), HeadersToSign.end());
std::string_view DateStamp(AmzDate.data(), 8);
@@ -316,6 +326,10 @@ S3Client::SignRequest(std::string_view Method, std::string_view Path, std::strin
{
Result->emplace("x-amz-security-token", Credentials.SessionToken);
}
+ for (const auto& [K, V] : ExtraSignedHeaders)
+ {
+ Result->emplace(K, V);
+ }
return Result;
}
@@ -338,7 +352,10 @@ S3Client::PutObject(std::string_view Key, IoBuffer Content)
return S3Result{std::move(Err)};
}
- ZEN_DEBUG("S3 PUT '{}' succeeded ({} bytes)", Key, Content.GetSize());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 PUT '{}' succeeded ({} bytes)", Key, Content.GetSize());
+ }
return {};
}
@@ -362,7 +379,10 @@ S3Client::GetObject(std::string_view Key, const std::filesystem::path& TempFileP
return S3GetObjectResult{S3Result{std::move(Err)}, {}};
}
- ZEN_DEBUG("S3 GET '{}' succeeded ({} bytes)", Key, Response.ResponsePayload.GetSize());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 GET '{}' succeeded ({} bytes)", Key, Response.ResponsePayload.GetSize());
+ }
return S3GetObjectResult{{}, std::move(Response.ResponsePayload)};
}
@@ -403,11 +423,14 @@ S3Client::GetObjectRange(std::string_view Key, uint64_t RangeStart, uint64_t Ran
return S3GetObjectResult{S3Result{std::move(Err)}, {}};
}
- ZEN_DEBUG("S3 GET range '{}' [{}-{}] succeeded ({} bytes)",
- Key,
- RangeStart,
- RangeStart + RangeSize - 1,
- Response.ResponsePayload.GetSize());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 GET range '{}' [{}-{}] succeeded ({} bytes)",
+ Key,
+ RangeStart,
+ RangeStart + RangeSize - 1,
+ Response.ResponsePayload.GetSize());
+ }
return S3GetObjectResult{{}, std::move(Response.ResponsePayload)};
}
@@ -426,7 +449,51 @@ S3Client::DeleteObject(std::string_view Key)
return S3Result{std::move(Err)};
}
- ZEN_DEBUG("S3 DELETE '{}' succeeded", Key);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 DELETE '{}' succeeded", Key);
+ }
+ return {};
+}
+
+S3Result
+S3Client::Touch(std::string_view Key)
+{
+ std::string Path = KeyToPath(Key);
+
+ // x-amz-copy-source is always "/bucket/key" regardless of addressing style.
+ // Key must be URI-encoded except for '/' separators. When source and destination
+ // are identical, REPLACE is required; COPY is rejected with InvalidRequest.
+ const std::array<std::pair<std::string, std::string>, 2> ExtraSigned{{
+ {"x-amz-copy-source", fmt::format("/{}/{}", m_BucketName, AwsUriEncode(Key, false))},
+ {"x-amz-metadata-directive", "REPLACE"},
+ }};
+
+ HttpClient::KeyValueMap Headers = SignRequest("PUT", Path, "", EmptyPayloadHash, ExtraSigned);
+
+ HttpClient::Response Response = m_HttpClient.Put(Path, IoBuffer{}, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = Response.ErrorMessage("S3 Touch failed");
+ ZEN_WARN("S3 Touch '{}' failed: {}", Key, Err);
+ return S3Result{std::move(Err)};
+ }
+
+ // Copy operations can return HTTP 200 with an error in the XML body.
+ std::string_view ResponseBody = Response.AsText();
+ if (ResponseBody.find("<Error>") != std::string_view::npos)
+ {
+ std::string_view ErrorCode = ExtractXmlValue(ResponseBody, "Code");
+ std::string_view ErrorMessage = ExtractXmlValue(ResponseBody, "Message");
+ std::string Err = fmt::format("S3 Touch '{}' returned error: {} - {}", Key, ErrorCode, ErrorMessage);
+ ZEN_WARN("{}", Err);
+ return S3Result{std::move(Err)};
+ }
+
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 Touch '{}' succeeded", Key);
+ }
return {};
}
@@ -468,7 +535,10 @@ S3Client::HeadObject(std::string_view Key)
Info.LastModified = *V;
}
- ZEN_DEBUG("S3 HEAD '{}' succeeded (size={})", Key, Info.Size);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 HEAD '{}' succeeded (size={})", Key, Info.Size);
+ }
return S3HeadObjectResult{{}, std::move(Info), HeadObjectResult::Found};
}
@@ -565,10 +635,16 @@ S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys)
}
ContinuationToken = std::string(NextToken);
- ZEN_DEBUG("S3 ListObjectsV2 prefix='{}' fetching next page ({} objects so far)", Prefix, Result.Objects.size());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 ListObjectsV2 prefix='{}' fetching next page ({} objects so far)", Prefix, Result.Objects.size());
+ }
}
- ZEN_DEBUG("S3 ListObjectsV2 prefix='{}' returned {} objects", Prefix, Result.Objects.size());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 ListObjectsV2 prefix='{}' returned {} objects", Prefix, Result.Objects.size());
+ }
return Result;
}
@@ -607,7 +683,10 @@ S3Client::CreateMultipartUpload(std::string_view Key)
return S3CreateMultipartUploadResult{S3Result{std::move(Err)}, {}};
}
- ZEN_DEBUG("S3 CreateMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 CreateMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ }
return S3CreateMultipartUploadResult{{}, std::string(UploadId)};
}
@@ -642,7 +721,10 @@ S3Client::UploadPart(std::string_view Key, std::string_view UploadId, uint32_t P
return S3UploadPartResult{S3Result{std::move(Err)}, {}};
}
- ZEN_DEBUG("S3 UploadPart '{}' part {} succeeded ({} bytes, etag={})", Key, PartNumber, Content.GetSize(), *ETag);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 UploadPart '{}' part {} succeeded ({} bytes, etag={})", Key, PartNumber, Content.GetSize(), *ETag);
+ }
return S3UploadPartResult{{}, *ETag};
}
@@ -691,7 +773,10 @@ S3Client::CompleteMultipartUpload(std::string_view Key,
return S3Result{std::move(Err)};
}
- ZEN_DEBUG("S3 CompleteMultipartUpload '{}' succeeded ({} parts)", Key, PartETags.size());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 CompleteMultipartUpload '{}' succeeded ({} parts)", Key, PartETags.size());
+ }
return {};
}
@@ -712,7 +797,10 @@ S3Client::AbortMultipartUpload(std::string_view Key, std::string_view UploadId)
return S3Result{std::move(Err)};
}
- ZEN_DEBUG("S3 AbortMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 AbortMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ }
return {};
}
@@ -755,7 +843,10 @@ S3Client::PutObjectMultipart(std::string_view Key,
return PutObject(Key, TotalSize > 0 ? FetchRange(0, TotalSize) : IoBuffer{});
}
- ZEN_DEBUG("S3 multipart upload '{}': {} bytes in ~{} parts", Key, TotalSize, (TotalSize + PartSize - 1) / PartSize);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 multipart upload '{}': {} bytes in ~{} parts", Key, TotalSize, (TotalSize + PartSize - 1) / PartSize);
+ }
S3CreateMultipartUploadResult InitResult = CreateMultipartUpload(Key);
if (!InitResult)
@@ -803,7 +894,10 @@ S3Client::PutObjectMultipart(std::string_view Key,
throw;
}
- ZEN_DEBUG("S3 multipart upload '{}' completed ({} parts, {} bytes)", Key, PartETags.size(), TotalSize);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 multipart upload '{}' completed ({} parts, {} bytes)", Key, PartETags.size(), TotalSize);
+ }
return {};
}
@@ -892,7 +986,7 @@ TEST_CASE("s3client.minio_integration")
using namespace std::literals;
// Spawn a single MinIO server for the entire test case. Previously each SUBCASE re-entered
- // the TEST_CASE from the top, spawning and killing MinIO per subcase — slow and flaky on
+ // the TEST_CASE from the top, spawning and killing MinIO per subcase - slow and flaky on
// macOS CI. Sequential sections avoid the re-entry while still sharing one MinIO instance
// that is torn down via RAII at scope exit.
MinioProcessOptions MinioOpts;
@@ -943,6 +1037,42 @@ TEST_CASE("s3client.minio_integration")
CHECK(HeadRes2.Status == HeadObjectResult::NotFound);
}
+ // -- touch ----------------------------------------------------------------
+ {
+ std::string_view TestData = "touch-me"sv;
+ IoBuffer Content = IoBufferBuilder::MakeFromMemory(MakeMemoryView(TestData));
+ S3Result PutRes = Client.PutObject("touch/obj.txt", std::move(Content));
+ REQUIRE(PutRes.IsSuccess());
+
+ S3HeadObjectResult Before = Client.HeadObject("touch/obj.txt");
+ REQUIRE(Before.IsSuccess());
+ REQUIRE(Before.Status == HeadObjectResult::Found);
+
+ // S3 LastModified has second precision; sleep past the second boundary so
+ // the touched timestamp is strictly greater.
+ Sleep(1100);
+
+ S3Result TouchRes = Client.Touch("touch/obj.txt");
+ REQUIRE(TouchRes.IsSuccess());
+
+ S3HeadObjectResult After = Client.HeadObject("touch/obj.txt");
+ REQUIRE(After.IsSuccess());
+ REQUIRE(After.Status == HeadObjectResult::Found);
+ CHECK(After.Info.Size == Before.Info.Size);
+ CHECK(After.Info.LastModified != Before.Info.LastModified);
+
+ // Content must be unchanged by a self-copy.
+ S3GetObjectResult GetRes = Client.GetObject("touch/obj.txt");
+ REQUIRE(GetRes.IsSuccess());
+ CHECK(GetRes.AsText() == TestData);
+
+ // Touching a missing key must fail.
+ S3Result MissRes = Client.Touch("touch/does-not-exist.txt");
+ CHECK_FALSE(MissRes.IsSuccess());
+
+ Client.DeleteObject("touch/obj.txt");
+ }
+
// -- head_not_found -------------------------------------------------------
{
S3HeadObjectResult Res = Client.HeadObject("nonexistent/key.dat");
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-21 15:41:56 +0000