aboutsummaryrefslogtreecommitdiff
path: root/src/zenutil/cloud
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenutil/cloud')
-rw-r--r--src/zenutil/cloud/imdscredentials.cpp9
-rw-r--r--src/zenutil/cloud/minioprocess.cpp4
-rw-r--r--src/zenutil/cloud/mockimds.cpp6
-rw-r--r--src/zenutil/cloud/s3client.cpp338
4 files changed, 302 insertions, 55 deletions
diff --git a/src/zenutil/cloud/imdscredentials.cpp b/src/zenutil/cloud/imdscredentials.cpp
index dde1dc019..a23cb9c28 100644
--- a/src/zenutil/cloud/imdscredentials.cpp
+++ b/src/zenutil/cloud/imdscredentials.cpp
@@ -64,6 +64,7 @@ ImdsCredentialProvider::ImdsCredentialProvider(const ImdsCredentialProviderOptio
.LogCategory = "imds",
.ConnectTimeout = Options.ConnectTimeout,
.Timeout = Options.RequestTimeout,
+ .RetryCount = 3,
})
{
ZEN_INFO("IMDS credential provider configured (endpoint: {})", m_HttpClient.GetBaseUri());
@@ -115,7 +116,7 @@ ImdsCredentialProvider::FetchToken()
HttpClient::KeyValueMap Headers;
Headers->emplace("X-aws-ec2-metadata-token-ttl-seconds", "21600");
- HttpClient::Response Response = m_HttpClient.Put("/latest/api/token", Headers);
+ HttpClient::Response Response = m_HttpClient.Put("/latest/api/token", IoBuffer{}, Headers);
if (!Response.IsSuccess())
{
ZEN_WARN("IMDS token request failed: {}", Response.ErrorMessage("PUT /latest/api/token"));
@@ -213,7 +214,7 @@ ImdsCredentialProvider::FetchCredentials()
}
else
{
- // Expiration is in the past or unparseable — force refresh next time
+ // Expiration is in the past or unparseable - force refresh next time
NewExpiresAt = std::chrono::steady_clock::now();
}
@@ -226,7 +227,7 @@ ImdsCredentialProvider::FetchCredentials()
if (KeyChanged)
{
- ZEN_INFO("IMDS credentials refreshed (AccessKeyId: {}...)", m_CachedCredentials.AccessKeyId.substr(0, 8));
+ ZEN_INFO("IMDS credentials refreshed (AccessKeyId: {})", HideSensitiveString(m_CachedCredentials.AccessKeyId));
}
else
{
@@ -369,7 +370,7 @@ TEST_CASE("imdscredentials.fetch_from_mock")
TEST_CASE("imdscredentials.unreachable_endpoint")
{
- // Point at a non-existent server — should return empty credentials, not crash
+ // Point at a non-existent server - should return empty credentials, not crash
ImdsCredentialProviderOptions Opts;
Opts.Endpoint = "http://127.0.0.1:1"; // unlikely to have anything listening
Opts.ConnectTimeout = std::chrono::milliseconds(100);
diff --git a/src/zenutil/cloud/minioprocess.cpp b/src/zenutil/cloud/minioprocess.cpp
index 457453bd8..2db0010dc 100644
--- a/src/zenutil/cloud/minioprocess.cpp
+++ b/src/zenutil/cloud/minioprocess.cpp
@@ -45,7 +45,7 @@ struct MinioProcess::Impl
}
CreateProcOptions Options;
- Options.Flags |= CreateProcOptions::Flag_Windows_NewProcessGroup;
+ Options.Flags |= CreateProcOptions::Flag_NewProcessGroup;
Options.Environment.emplace_back("MINIO_ROOT_USER", m_Options.RootUser);
Options.Environment.emplace_back("MINIO_ROOT_PASSWORD", m_Options.RootPassword);
@@ -102,7 +102,7 @@ struct MinioProcess::Impl
{
if (m_DataDir.empty())
{
- ZEN_WARN("MinIO: Cannot create bucket before data directory is initialized — call SpawnMinioServer() first");
+ ZEN_WARN("MinIO: Cannot create bucket before data directory is initialized - call SpawnMinioServer() first");
return;
}
diff --git a/src/zenutil/cloud/mockimds.cpp b/src/zenutil/cloud/mockimds.cpp
index 6919fab4d..88b348ed6 100644
--- a/src/zenutil/cloud/mockimds.cpp
+++ b/src/zenutil/cloud/mockimds.cpp
@@ -93,7 +93,7 @@ MockImdsService::HandleAwsRequest(HttpServerRequest& Request)
return;
}
- // Autoscaling lifecycle state — 404 when not in an ASG
+ // Autoscaling lifecycle state - 404 when not in an ASG
if (Uri == "latest/meta-data/autoscaling/target-lifecycle-state")
{
if (Aws.AutoscalingState.empty())
@@ -105,7 +105,7 @@ MockImdsService::HandleAwsRequest(HttpServerRequest& Request)
return;
}
- // Spot interruption notice — 404 when no interruption pending
+ // Spot interruption notice - 404 when no interruption pending
if (Uri == "latest/meta-data/spot/instance-action")
{
if (Aws.SpotAction.empty())
@@ -117,7 +117,7 @@ MockImdsService::HandleAwsRequest(HttpServerRequest& Request)
return;
}
- // IAM role discovery — returns the role name
+ // IAM role discovery - returns the role name
if (Uri == "latest/meta-data/iam/security-credentials/")
{
if (Aws.IamRoleName.empty())
diff --git a/src/zenutil/cloud/s3client.cpp b/src/zenutil/cloud/s3client.cpp
index d9fde05d9..f8bed92da 100644
--- a/src/zenutil/cloud/s3client.cpp
+++ b/src/zenutil/cloud/s3client.cpp
@@ -135,6 +135,41 @@ namespace {
return nullptr;
}
+ /// Extract Code/Message from an S3 XML error body. Returns true if an <Error> element was
+ /// found, even if Code/Message are empty.
+ bool ExtractS3Error(std::string_view Body, std::string_view& OutCode, std::string_view& OutMessage)
+ {
+ if (Body.find("<Error>") == std::string_view::npos)
+ {
+ return false;
+ }
+ OutCode = ExtractXmlValue(Body, "Code");
+ OutMessage = ExtractXmlValue(Body, "Message");
+ return true;
+ }
+
+ /// Build a human-readable error message for a failed S3 response. When the response body
+ /// contains an S3 `<Error>` element, the Code and Message fields are included in the string
+ /// so transient 4xx/5xx failures (SignatureDoesNotMatch, AuthorizationHeaderMalformed, etc.)
+ /// show up in logs instead of being swallowed. Falls back to the generic HTTP/transport
+ /// message when no XML body is available (HEAD responses, transport errors).
+ std::string S3ErrorMessage(std::string_view Prefix, const HttpClient::Response& Response)
+ {
+ if (!Response.Error.has_value() && Response.ResponsePayload)
+ {
+ std::string_view Body(reinterpret_cast<const char*>(Response.ResponsePayload.GetData()), Response.ResponsePayload.GetSize());
+ std::string_view Code;
+ std::string_view Message;
+ if (ExtractS3Error(Body, Code, Message) && (!Code.empty() || !Message.empty()))
+ {
+ ExtendableStringBuilder<256> Decoded;
+ DecodeXmlEntities(Message, Decoded);
+ return fmt::format("{}: HTTP status ({}) {} - {}", Prefix, static_cast<int>(Response.StatusCode), Code, Decoded.ToView());
+ }
+ }
+ return Response.ErrorMessage(Prefix);
+ }
+
} // namespace
std::string_view S3GetObjectResult::NotFoundErrorText = "Not found";
@@ -148,6 +183,7 @@ S3Client::S3Client(const S3ClientOptions& Options)
, m_Credentials(Options.Credentials)
, m_CredentialProvider(Options.CredentialProvider)
, m_HttpClient(BuildEndpoint(), Options.HttpSettings)
+, m_Verbose(Options.HttpSettings.Verbose)
{
m_Host = BuildHostHeader();
ZEN_INFO("S3 client configured for bucket '{}' in region '{}' (endpoint: {}, {})",
@@ -186,6 +222,14 @@ S3Client::GetCurrentCredentials()
}
std::string
+S3Client::BuildNoCredentialsError(std::string Context)
+{
+ std::string Err = fmt::format("{}: no credentials available", Context);
+ ZEN_WARN("{}", Err);
+ return Err;
+}
+
+std::string
S3Client::BuildEndpoint() const
{
if (!m_Endpoint.empty())
@@ -285,14 +329,18 @@ S3Client::GetSigningKey(std::string_view DateStamp)
}
HttpClient::KeyValueMap
-S3Client::SignRequest(std::string_view Method, std::string_view Path, std::string_view CanonicalQueryString, std::string_view PayloadHash)
+S3Client::SignRequest(const SigV4Credentials& Credentials,
+ std::string_view Method,
+ std::string_view Path,
+ std::string_view CanonicalQueryString,
+ std::string_view PayloadHash,
+ std::span<const std::pair<std::string, std::string>> ExtraSignedHeaders)
{
- SigV4Credentials Credentials = GetCurrentCredentials();
-
std::string AmzDate = GetAmzTimestamp();
// Build sorted headers to sign (must be sorted by lowercase name)
std::vector<std::pair<std::string, std::string>> HeadersToSign;
+ HeadersToSign.reserve(4 + ExtraSignedHeaders.size());
HeadersToSign.emplace_back("host", m_Host);
HeadersToSign.emplace_back("x-amz-content-sha256", std::string(PayloadHash));
HeadersToSign.emplace_back("x-amz-date", AmzDate);
@@ -300,6 +348,10 @@ S3Client::SignRequest(std::string_view Method, std::string_view Path, std::strin
{
HeadersToSign.emplace_back("x-amz-security-token", Credentials.SessionToken);
}
+ for (const auto& [K, V] : ExtraSignedHeaders)
+ {
+ HeadersToSign.emplace_back(K, V);
+ }
std::sort(HeadersToSign.begin(), HeadersToSign.end());
std::string_view DateStamp(AmzDate.data(), 8);
@@ -316,6 +368,10 @@ S3Client::SignRequest(std::string_view Method, std::string_view Path, std::strin
{
Result->emplace("x-amz-security-token", Credentials.SessionToken);
}
+ for (const auto& [K, V] : ExtraSignedHeaders)
+ {
+ Result->emplace(K, V);
+ }
return Result;
}
@@ -323,31 +379,46 @@ S3Client::SignRequest(std::string_view Method, std::string_view Path, std::strin
S3Result
S3Client::PutObject(std::string_view Key, IoBuffer Content)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 PUT '{}' failed", Key); !Err.empty())
+ {
+ return S3Result{std::move(Err)};
+ }
+
std::string Path = KeyToPath(Key);
// Hash the payload
std::string PayloadHash = Sha256ToHex(ComputeSha256(Content.GetData(), Content.GetSize()));
- HttpClient::KeyValueMap Headers = SignRequest("PUT", Path, "", PayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "PUT", Path, "", PayloadHash);
HttpClient::Response Response = m_HttpClient.Put(Path, Content, Headers);
if (!Response.IsSuccess())
{
- std::string Err = Response.ErrorMessage("S3 PUT failed");
+ std::string Err = S3ErrorMessage("S3 PUT failed", Response);
ZEN_WARN("S3 PUT '{}' failed: {}", Key, Err);
return S3Result{std::move(Err)};
}
- ZEN_DEBUG("S3 PUT '{}' succeeded ({} bytes)", Key, Content.GetSize());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 PUT '{}' succeeded ({} bytes)", Key, Content.GetSize());
+ }
return {};
}
S3GetObjectResult
S3Client::GetObject(std::string_view Key, const std::filesystem::path& TempFilePath)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 GET '{}' failed", Key); !Err.empty())
+ {
+ return S3GetObjectResult{S3Result{std::move(Err)}, {}};
+ }
+
std::string Path = KeyToPath(Key);
- HttpClient::KeyValueMap Headers = SignRequest("GET", Path, "", EmptyPayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "GET", Path, "", EmptyPayloadHash);
HttpClient::Response Response = m_HttpClient.Download(Path, TempFilePath, Headers);
if (!Response.IsSuccess())
@@ -357,12 +428,15 @@ S3Client::GetObject(std::string_view Key, const std::filesystem::path& TempFileP
return S3GetObjectResult{S3Result{.Error = std::string(S3GetObjectResult::NotFoundErrorText)}, {}};
}
- std::string Err = Response.ErrorMessage("S3 GET failed");
+ std::string Err = S3ErrorMessage("S3 GET failed", Response);
ZEN_WARN("S3 GET '{}' failed: {}", Key, Err);
return S3GetObjectResult{S3Result{std::move(Err)}, {}};
}
- ZEN_DEBUG("S3 GET '{}' succeeded ({} bytes)", Key, Response.ResponsePayload.GetSize());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 GET '{}' succeeded ({} bytes)", Key, Response.ResponsePayload.GetSize());
+ }
return S3GetObjectResult{{}, std::move(Response.ResponsePayload)};
}
@@ -370,9 +444,17 @@ S3GetObjectResult
S3Client::GetObjectRange(std::string_view Key, uint64_t RangeStart, uint64_t RangeSize)
{
ZEN_ASSERT(RangeSize > 0);
+
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 GET range '{}' [{}-{}] failed", Key, RangeStart, RangeStart + RangeSize - 1);
+ !Err.empty())
+ {
+ return S3GetObjectResult{S3Result{std::move(Err)}, {}};
+ }
+
std::string Path = KeyToPath(Key);
- HttpClient::KeyValueMap Headers = SignRequest("GET", Path, "", EmptyPayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "GET", Path, "", EmptyPayloadHash);
Headers->emplace("Range", fmt::format("bytes={}-{}", RangeStart, RangeStart + RangeSize - 1));
HttpClient::Response Response = m_HttpClient.Get(Path, Headers);
@@ -383,7 +465,7 @@ S3Client::GetObjectRange(std::string_view Key, uint64_t RangeStart, uint64_t Ran
return S3GetObjectResult{S3Result{.Error = std::string(S3GetObjectResult::NotFoundErrorText)}, {}};
}
- std::string Err = Response.ErrorMessage("S3 GET range failed");
+ std::string Err = S3ErrorMessage("S3 GET range failed", Response);
ZEN_WARN("S3 GET range '{}' [{}-{}] failed: {}", Key, RangeStart, RangeStart + RangeSize - 1, Err);
return S3GetObjectResult{S3Result{std::move(Err)}, {}};
}
@@ -403,39 +485,104 @@ S3Client::GetObjectRange(std::string_view Key, uint64_t RangeStart, uint64_t Ran
return S3GetObjectResult{S3Result{std::move(Err)}, {}};
}
- ZEN_DEBUG("S3 GET range '{}' [{}-{}] succeeded ({} bytes)",
- Key,
- RangeStart,
- RangeStart + RangeSize - 1,
- Response.ResponsePayload.GetSize());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 GET range '{}' [{}-{}] succeeded ({} bytes)",
+ Key,
+ RangeStart,
+ RangeStart + RangeSize - 1,
+ Response.ResponsePayload.GetSize());
+ }
return S3GetObjectResult{{}, std::move(Response.ResponsePayload)};
}
S3Result
S3Client::DeleteObject(std::string_view Key)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 DELETE '{}' failed", Key); !Err.empty())
+ {
+ return S3Result{std::move(Err)};
+ }
+
std::string Path = KeyToPath(Key);
- HttpClient::KeyValueMap Headers = SignRequest("DELETE", Path, "", EmptyPayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "DELETE", Path, "", EmptyPayloadHash);
HttpClient::Response Response = m_HttpClient.Delete(Path, Headers);
if (!Response.IsSuccess())
{
- std::string Err = Response.ErrorMessage("S3 DELETE failed");
+ std::string Err = S3ErrorMessage("S3 DELETE failed", Response);
ZEN_WARN("S3 DELETE '{}' failed: {}", Key, Err);
return S3Result{std::move(Err)};
}
- ZEN_DEBUG("S3 DELETE '{}' succeeded", Key);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 DELETE '{}' succeeded", Key);
+ }
+ return {};
+}
+
+S3Result
+S3Client::Touch(std::string_view Key)
+{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 Touch '{}' failed", Key); !Err.empty())
+ {
+ return S3Result{std::move(Err)};
+ }
+
+ std::string Path = KeyToPath(Key);
+
+ // x-amz-copy-source is always "/bucket/key" regardless of addressing style.
+ // Key must be URI-encoded except for '/' separators. When source and destination
+ // are identical, REPLACE is required; COPY is rejected with InvalidRequest.
+ const std::array<std::pair<std::string, std::string>, 2> ExtraSigned{{
+ {"x-amz-copy-source", fmt::format("/{}/{}", m_BucketName, AwsUriEncode(Key, false))},
+ {"x-amz-metadata-directive", "REPLACE"},
+ }};
+
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "PUT", Path, "", EmptyPayloadHash, ExtraSigned);
+
+ HttpClient::Response Response = m_HttpClient.Put(Path, IoBuffer{}, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = S3ErrorMessage("S3 Touch failed", Response);
+ ZEN_WARN("S3 Touch '{}' failed: {}", Key, Err);
+ return S3Result{std::move(Err)};
+ }
+
+ // Copy operations can return HTTP 200 with an error in the XML body.
+ std::string_view ResponseBody = Response.AsText();
+ std::string_view ErrorCode;
+ std::string_view ErrorMessage;
+ if (ExtractS3Error(ResponseBody, ErrorCode, ErrorMessage))
+ {
+ std::string Err = fmt::format("S3 Touch '{}' returned error: {} - {}", Key, ErrorCode, ErrorMessage);
+ ZEN_WARN("{}", Err);
+ return S3Result{std::move(Err)};
+ }
+
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 Touch '{}' succeeded", Key);
+ }
return {};
}
S3HeadObjectResult
S3Client::HeadObject(std::string_view Key)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 HEAD '{}' failed", Key); !Err.empty())
+ {
+ return S3HeadObjectResult{S3Result{std::move(Err)}, {}, HeadObjectResult::Error};
+ }
+
std::string Path = KeyToPath(Key);
- HttpClient::KeyValueMap Headers = SignRequest("HEAD", Path, "", EmptyPayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "HEAD", Path, "", EmptyPayloadHash);
HttpClient::Response Response = m_HttpClient.Head(Path, Headers);
if (!Response.IsSuccess())
@@ -445,7 +592,7 @@ S3Client::HeadObject(std::string_view Key)
return S3HeadObjectResult{{}, {}, HeadObjectResult::NotFound};
}
- std::string Err = Response.ErrorMessage("S3 HEAD failed");
+ std::string Err = S3ErrorMessage("S3 HEAD failed", Response);
ZEN_WARN("S3 HEAD '{}' failed: {}", Key, Err);
return S3HeadObjectResult{S3Result{std::move(Err)}, {}, HeadObjectResult::Error};
}
@@ -468,7 +615,10 @@ S3Client::HeadObject(std::string_view Key)
Info.LastModified = *V;
}
- ZEN_DEBUG("S3 HEAD '{}' succeeded (size={})", Key, Info.Size);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 HEAD '{}' succeeded (size={})", Key, Info.Size);
+ }
return S3HeadObjectResult{{}, std::move(Info), HeadObjectResult::Found};
}
@@ -481,6 +631,13 @@ S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys)
for (;;)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 ListObjectsV2 prefix='{}' failed", Prefix); !Err.empty())
+ {
+ Result.Error = std::move(Err);
+ return Result;
+ }
+
// Build query parameters for ListObjectsV2
std::vector<std::pair<std::string, std::string>> QueryParams;
QueryParams.emplace_back("list-type", "2");
@@ -499,13 +656,13 @@ S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys)
std::string CanonicalQS = BuildCanonicalQueryString(std::move(QueryParams));
std::string RootPath = BucketRootPath();
- HttpClient::KeyValueMap Headers = SignRequest("GET", RootPath, CanonicalQS, EmptyPayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "GET", RootPath, CanonicalQS, EmptyPayloadHash);
std::string FullPath = BuildRequestPath(RootPath, CanonicalQS);
HttpClient::Response Response = m_HttpClient.Get(FullPath, Headers);
if (!Response.IsSuccess())
{
- std::string Err = Response.ErrorMessage("S3 ListObjectsV2 failed");
+ std::string Err = S3ErrorMessage("S3 ListObjectsV2 failed", Response);
ZEN_WARN("S3 ListObjectsV2 prefix='{}' failed: {}", Prefix, Err);
Result.Error = std::move(Err);
return Result;
@@ -565,10 +722,16 @@ S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys)
}
ContinuationToken = std::string(NextToken);
- ZEN_DEBUG("S3 ListObjectsV2 prefix='{}' fetching next page ({} objects so far)", Prefix, Result.Objects.size());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 ListObjectsV2 prefix='{}' fetching next page ({} objects so far)", Prefix, Result.Objects.size());
+ }
}
- ZEN_DEBUG("S3 ListObjectsV2 prefix='{}' returned {} objects", Prefix, Result.Objects.size());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 ListObjectsV2 prefix='{}' returned {} objects", Prefix, Result.Objects.size());
+ }
return Result;
}
@@ -578,16 +741,22 @@ S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys)
S3CreateMultipartUploadResult
S3Client::CreateMultipartUpload(std::string_view Key)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 CreateMultipartUpload '{}' failed", Key); !Err.empty())
+ {
+ return S3CreateMultipartUploadResult{S3Result{std::move(Err)}, {}};
+ }
+
std::string Path = KeyToPath(Key);
std::string CanonicalQS = BuildCanonicalQueryString({{"uploads", ""}});
- HttpClient::KeyValueMap Headers = SignRequest("POST", Path, CanonicalQS, EmptyPayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "POST", Path, CanonicalQS, EmptyPayloadHash);
std::string FullPath = BuildRequestPath(Path, CanonicalQS);
HttpClient::Response Response = m_HttpClient.Post(FullPath, Headers);
if (!Response.IsSuccess())
{
- std::string Err = Response.ErrorMessage("S3 CreateMultipartUpload failed");
+ std::string Err = S3ErrorMessage("S3 CreateMultipartUpload failed", Response);
ZEN_WARN("S3 CreateMultipartUpload '{}' failed: {}", Key, Err);
return S3CreateMultipartUploadResult{S3Result{std::move(Err)}, {}};
}
@@ -607,13 +776,22 @@ S3Client::CreateMultipartUpload(std::string_view Key)
return S3CreateMultipartUploadResult{S3Result{std::move(Err)}, {}};
}
- ZEN_DEBUG("S3 CreateMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 CreateMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ }
return S3CreateMultipartUploadResult{{}, std::string(UploadId)};
}
S3UploadPartResult
S3Client::UploadPart(std::string_view Key, std::string_view UploadId, uint32_t PartNumber, IoBuffer Content)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 UploadPart '{}' part {} failed", Key, PartNumber); !Err.empty())
+ {
+ return S3UploadPartResult{S3Result{std::move(Err)}, {}};
+ }
+
std::string Path = KeyToPath(Key);
std::string CanonicalQS = BuildCanonicalQueryString({
{"partNumber", fmt::format("{}", PartNumber)},
@@ -622,13 +800,13 @@ S3Client::UploadPart(std::string_view Key, std::string_view UploadId, uint32_t P
std::string PayloadHash = Sha256ToHex(ComputeSha256(Content.GetData(), Content.GetSize()));
- HttpClient::KeyValueMap Headers = SignRequest("PUT", Path, CanonicalQS, PayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "PUT", Path, CanonicalQS, PayloadHash);
std::string FullPath = BuildRequestPath(Path, CanonicalQS);
HttpClient::Response Response = m_HttpClient.Put(FullPath, Content, Headers);
if (!Response.IsSuccess())
{
- std::string Err = Response.ErrorMessage(fmt::format("S3 UploadPart {} failed", PartNumber));
+ std::string Err = S3ErrorMessage(fmt::format("S3 UploadPart {} failed", PartNumber), Response);
ZEN_WARN("S3 UploadPart '{}' part {} failed: {}", Key, PartNumber, Err);
return S3UploadPartResult{S3Result{std::move(Err)}, {}};
}
@@ -642,7 +820,10 @@ S3Client::UploadPart(std::string_view Key, std::string_view UploadId, uint32_t P
return S3UploadPartResult{S3Result{std::move(Err)}, {}};
}
- ZEN_DEBUG("S3 UploadPart '{}' part {} succeeded ({} bytes, etag={})", Key, PartNumber, Content.GetSize(), *ETag);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 UploadPart '{}' part {} succeeded ({} bytes, etag={})", Key, PartNumber, Content.GetSize(), *ETag);
+ }
return S3UploadPartResult{{}, *ETag};
}
@@ -651,6 +832,12 @@ S3Client::CompleteMultipartUpload(std::string_view Key,
std::string_view UploadId,
const std::vector<std::pair<uint32_t, std::string>>& PartETags)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 CompleteMultipartUpload '{}' failed", Key); !Err.empty())
+ {
+ return S3Result{std::move(Err)};
+ }
+
std::string Path = KeyToPath(Key);
std::string CanonicalQS = BuildCanonicalQueryString({{"uploadId", std::string(UploadId)}});
@@ -666,7 +853,7 @@ S3Client::CompleteMultipartUpload(std::string_view Key,
std::string_view XmlView = XmlBody.ToView();
std::string PayloadHash = Sha256ToHex(ComputeSha256(XmlView));
- HttpClient::KeyValueMap Headers = SignRequest("POST", Path, CanonicalQS, PayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "POST", Path, CanonicalQS, PayloadHash);
Headers->emplace("Content-Type", "application/xml");
IoBuffer Payload(IoBuffer::Clone, XmlView.data(), XmlView.size());
@@ -675,44 +862,56 @@ S3Client::CompleteMultipartUpload(std::string_view Key,
HttpClient::Response Response = m_HttpClient.Post(FullPath, Payload, Headers);
if (!Response.IsSuccess())
{
- std::string Err = Response.ErrorMessage("S3 CompleteMultipartUpload failed");
+ std::string Err = S3ErrorMessage("S3 CompleteMultipartUpload failed", Response);
ZEN_WARN("S3 CompleteMultipartUpload '{}' failed: {}", Key, Err);
return S3Result{std::move(Err)};
}
// Check for error in response body - S3 can return 200 with an error in the XML body
std::string_view ResponseBody = Response.AsText();
- if (ResponseBody.find("<Error>") != std::string_view::npos)
+ std::string_view ErrorCode;
+ std::string_view ErrorMessage;
+ if (ExtractS3Error(ResponseBody, ErrorCode, ErrorMessage))
{
- std::string_view ErrorCode = ExtractXmlValue(ResponseBody, "Code");
- std::string_view ErrorMessage = ExtractXmlValue(ResponseBody, "Message");
- std::string Err = fmt::format("S3 CompleteMultipartUpload '{}' returned error: {} - {}", Key, ErrorCode, ErrorMessage);
+ std::string Err = fmt::format("S3 CompleteMultipartUpload '{}' returned error: {} - {}", Key, ErrorCode, ErrorMessage);
ZEN_WARN("{}", Err);
return S3Result{std::move(Err)};
}
- ZEN_DEBUG("S3 CompleteMultipartUpload '{}' succeeded ({} parts)", Key, PartETags.size());
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 CompleteMultipartUpload '{}' succeeded ({} parts)", Key, PartETags.size());
+ }
return {};
}
S3Result
S3Client::AbortMultipartUpload(std::string_view Key, std::string_view UploadId)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 AbortMultipartUpload '{}' failed", Key); !Err.empty())
+ {
+ return S3Result{std::move(Err)};
+ }
+
std::string Path = KeyToPath(Key);
std::string CanonicalQS = BuildCanonicalQueryString({{"uploadId", std::string(UploadId)}});
- HttpClient::KeyValueMap Headers = SignRequest("DELETE", Path, CanonicalQS, EmptyPayloadHash);
+ HttpClient::KeyValueMap Headers = SignRequest(Credentials, "DELETE", Path, CanonicalQS, EmptyPayloadHash);
std::string FullPath = BuildRequestPath(Path, CanonicalQS);
HttpClient::Response Response = m_HttpClient.Delete(FullPath, Headers);
if (!Response.IsSuccess())
{
- std::string Err = Response.ErrorMessage("S3 AbortMultipartUpload failed");
+ std::string Err = S3ErrorMessage("S3 AbortMultipartUpload failed", Response);
ZEN_WARN("S3 AbortMultipartUpload '{}' failed: {}", Key, Err);
return S3Result{std::move(Err)};
}
- ZEN_DEBUG("S3 AbortMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 AbortMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ }
return {};
}
@@ -731,6 +930,12 @@ S3Client::GeneratePresignedPutUrl(std::string_view Key, std::chrono::seconds Exp
std::string
S3Client::GeneratePresignedUrlForMethod(std::string_view Key, std::string_view Method, std::chrono::seconds ExpiresIn)
{
+ SigV4Credentials Credentials;
+ if (std::string Err = RequireCredentials(Credentials, "S3 GeneratePresignedUrl '{}' {} failed", Key, Method); !Err.empty())
+ {
+ return {};
+ }
+
std::string Path = KeyToPath(Key);
std::string Scheme = "https";
@@ -739,7 +944,6 @@ S3Client::GeneratePresignedUrlForMethod(std::string_view Key, std::string_view M
Scheme = "http";
}
- SigV4Credentials Credentials = GetCurrentCredentials();
return GeneratePresignedUrl(Credentials, Method, Scheme, m_Host, Path, m_Region, "s3", ExpiresIn);
}
@@ -755,7 +959,10 @@ S3Client::PutObjectMultipart(std::string_view Key,
return PutObject(Key, TotalSize > 0 ? FetchRange(0, TotalSize) : IoBuffer{});
}
- ZEN_DEBUG("S3 multipart upload '{}': {} bytes in ~{} parts", Key, TotalSize, (TotalSize + PartSize - 1) / PartSize);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 multipart upload '{}': {} bytes in ~{} parts", Key, TotalSize, (TotalSize + PartSize - 1) / PartSize);
+ }
S3CreateMultipartUploadResult InitResult = CreateMultipartUpload(Key);
if (!InitResult)
@@ -803,7 +1010,10 @@ S3Client::PutObjectMultipart(std::string_view Key,
throw;
}
- ZEN_DEBUG("S3 multipart upload '{}' completed ({} parts, {} bytes)", Key, PartETags.size(), TotalSize);
+ if (m_Verbose)
+ {
+ ZEN_INFO("S3 multipart upload '{}' completed ({} parts, {} bytes)", Key, PartETags.size(), TotalSize);
+ }
return {};
}
@@ -892,7 +1102,7 @@ TEST_CASE("s3client.minio_integration")
using namespace std::literals;
// Spawn a single MinIO server for the entire test case. Previously each SUBCASE re-entered
- // the TEST_CASE from the top, spawning and killing MinIO per subcase — slow and flaky on
+ // the TEST_CASE from the top, spawning and killing MinIO per subcase - slow and flaky on
// macOS CI. Sequential sections avoid the re-entry while still sharing one MinIO instance
// that is torn down via RAII at scope exit.
MinioProcessOptions MinioOpts;
@@ -943,6 +1153,42 @@ TEST_CASE("s3client.minio_integration")
CHECK(HeadRes2.Status == HeadObjectResult::NotFound);
}
+ // -- touch ----------------------------------------------------------------
+ {
+ std::string_view TestData = "touch-me"sv;
+ IoBuffer Content = IoBufferBuilder::MakeFromMemory(MakeMemoryView(TestData));
+ S3Result PutRes = Client.PutObject("touch/obj.txt", std::move(Content));
+ REQUIRE(PutRes.IsSuccess());
+
+ S3HeadObjectResult Before = Client.HeadObject("touch/obj.txt");
+ REQUIRE(Before.IsSuccess());
+ REQUIRE(Before.Status == HeadObjectResult::Found);
+
+ // S3 LastModified has second precision; sleep past the second boundary so
+ // the touched timestamp is strictly greater.
+ Sleep(1100);
+
+ S3Result TouchRes = Client.Touch("touch/obj.txt");
+ REQUIRE(TouchRes.IsSuccess());
+
+ S3HeadObjectResult After = Client.HeadObject("touch/obj.txt");
+ REQUIRE(After.IsSuccess());
+ REQUIRE(After.Status == HeadObjectResult::Found);
+ CHECK(After.Info.Size == Before.Info.Size);
+ CHECK(After.Info.LastModified != Before.Info.LastModified);
+
+ // Content must be unchanged by a self-copy.
+ S3GetObjectResult GetRes = Client.GetObject("touch/obj.txt");
+ REQUIRE(GetRes.IsSuccess());
+ CHECK(GetRes.AsText() == TestData);
+
+ // Touching a missing key must fail.
+ S3Result MissRes = Client.Touch("touch/does-not-exist.txt");
+ CHECK_FALSE(MissRes.IsSuccess());
+
+ Client.DeleteObject("touch/obj.txt");
+ }
+
// -- head_not_found -------------------------------------------------------
{
S3HeadObjectResult Res = Client.HeadObject("nonexistent/key.dat");
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-26 07:15:05 +0000