diff options
Diffstat (limited to 'src/zenutil/cloud/s3client.cpp')
| -rw-r--r-- | src/zenutil/cloud/s3client.cpp | 184 |
1 files changed, 150 insertions, 34 deletions
diff --git a/src/zenutil/cloud/s3client.cpp b/src/zenutil/cloud/s3client.cpp index 3d6dca562..f8bed92da 100644 --- a/src/zenutil/cloud/s3client.cpp +++ b/src/zenutil/cloud/s3client.cpp @@ -135,6 +135,41 @@ namespace { return nullptr; } + /// Extract Code/Message from an S3 XML error body. Returns true if an <Error> element was + /// found, even if Code/Message are empty. + bool ExtractS3Error(std::string_view Body, std::string_view& OutCode, std::string_view& OutMessage) + { + if (Body.find("<Error>") == std::string_view::npos) + { + return false; + } + OutCode = ExtractXmlValue(Body, "Code"); + OutMessage = ExtractXmlValue(Body, "Message"); + return true; + } + + /// Build a human-readable error message for a failed S3 response. When the response body + /// contains an S3 `<Error>` element, the Code and Message fields are included in the string + /// so transient 4xx/5xx failures (SignatureDoesNotMatch, AuthorizationHeaderMalformed, etc.) + /// show up in logs instead of being swallowed. Falls back to the generic HTTP/transport + /// message when no XML body is available (HEAD responses, transport errors). + std::string S3ErrorMessage(std::string_view Prefix, const HttpClient::Response& Response) + { + if (!Response.Error.has_value() && Response.ResponsePayload) + { + std::string_view Body(reinterpret_cast<const char*>(Response.ResponsePayload.GetData()), Response.ResponsePayload.GetSize()); + std::string_view Code; + std::string_view Message; + if (ExtractS3Error(Body, Code, Message) && (!Code.empty() || !Message.empty())) + { + ExtendableStringBuilder<256> Decoded; + DecodeXmlEntities(Message, Decoded); + return fmt::format("{}: HTTP status ({}) {} - {}", Prefix, static_cast<int>(Response.StatusCode), Code, Decoded.ToView()); + } + } + return Response.ErrorMessage(Prefix); + } + } // namespace std::string_view S3GetObjectResult::NotFoundErrorText = "Not found"; @@ -187,6 +222,14 @@ S3Client::GetCurrentCredentials() } std::string +S3Client::BuildNoCredentialsError(std::string Context) +{ + std::string Err = fmt::format("{}: no credentials available", Context); + ZEN_WARN("{}", Err); + return Err; +} + +std::string S3Client::BuildEndpoint() const { if (!m_Endpoint.empty()) @@ -286,14 +329,13 @@ S3Client::GetSigningKey(std::string_view DateStamp) } HttpClient::KeyValueMap -S3Client::SignRequest(std::string_view Method, +S3Client::SignRequest(const SigV4Credentials& Credentials, + std::string_view Method, std::string_view Path, std::string_view CanonicalQueryString, std::string_view PayloadHash, std::span<const std::pair<std::string, std::string>> ExtraSignedHeaders) { - SigV4Credentials Credentials = GetCurrentCredentials(); - std::string AmzDate = GetAmzTimestamp(); // Build sorted headers to sign (must be sorted by lowercase name) @@ -337,17 +379,23 @@ S3Client::SignRequest(std::string_view Method, S3Result S3Client::PutObject(std::string_view Key, IoBuffer Content) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 PUT '{}' failed", Key); !Err.empty()) + { + return S3Result{std::move(Err)}; + } + std::string Path = KeyToPath(Key); // Hash the payload std::string PayloadHash = Sha256ToHex(ComputeSha256(Content.GetData(), Content.GetSize())); - HttpClient::KeyValueMap Headers = SignRequest("PUT", Path, "", PayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "PUT", Path, "", PayloadHash); HttpClient::Response Response = m_HttpClient.Put(Path, Content, Headers); if (!Response.IsSuccess()) { - std::string Err = Response.ErrorMessage("S3 PUT failed"); + std::string Err = S3ErrorMessage("S3 PUT failed", Response); ZEN_WARN("S3 PUT '{}' failed: {}", Key, Err); return S3Result{std::move(Err)}; } @@ -362,9 +410,15 @@ S3Client::PutObject(std::string_view Key, IoBuffer Content) S3GetObjectResult S3Client::GetObject(std::string_view Key, const std::filesystem::path& TempFilePath) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 GET '{}' failed", Key); !Err.empty()) + { + return S3GetObjectResult{S3Result{std::move(Err)}, {}}; + } + std::string Path = KeyToPath(Key); - HttpClient::KeyValueMap Headers = SignRequest("GET", Path, "", EmptyPayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "GET", Path, "", EmptyPayloadHash); HttpClient::Response Response = m_HttpClient.Download(Path, TempFilePath, Headers); if (!Response.IsSuccess()) @@ -374,7 +428,7 @@ S3Client::GetObject(std::string_view Key, const std::filesystem::path& TempFileP return S3GetObjectResult{S3Result{.Error = std::string(S3GetObjectResult::NotFoundErrorText)}, {}}; } - std::string Err = Response.ErrorMessage("S3 GET failed"); + std::string Err = S3ErrorMessage("S3 GET failed", Response); ZEN_WARN("S3 GET '{}' failed: {}", Key, Err); return S3GetObjectResult{S3Result{std::move(Err)}, {}}; } @@ -390,9 +444,17 @@ S3GetObjectResult S3Client::GetObjectRange(std::string_view Key, uint64_t RangeStart, uint64_t RangeSize) { ZEN_ASSERT(RangeSize > 0); + + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 GET range '{}' [{}-{}] failed", Key, RangeStart, RangeStart + RangeSize - 1); + !Err.empty()) + { + return S3GetObjectResult{S3Result{std::move(Err)}, {}}; + } + std::string Path = KeyToPath(Key); - HttpClient::KeyValueMap Headers = SignRequest("GET", Path, "", EmptyPayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "GET", Path, "", EmptyPayloadHash); Headers->emplace("Range", fmt::format("bytes={}-{}", RangeStart, RangeStart + RangeSize - 1)); HttpClient::Response Response = m_HttpClient.Get(Path, Headers); @@ -403,7 +465,7 @@ S3Client::GetObjectRange(std::string_view Key, uint64_t RangeStart, uint64_t Ran return S3GetObjectResult{S3Result{.Error = std::string(S3GetObjectResult::NotFoundErrorText)}, {}}; } - std::string Err = Response.ErrorMessage("S3 GET range failed"); + std::string Err = S3ErrorMessage("S3 GET range failed", Response); ZEN_WARN("S3 GET range '{}' [{}-{}] failed: {}", Key, RangeStart, RangeStart + RangeSize - 1, Err); return S3GetObjectResult{S3Result{std::move(Err)}, {}}; } @@ -437,14 +499,20 @@ S3Client::GetObjectRange(std::string_view Key, uint64_t RangeStart, uint64_t Ran S3Result S3Client::DeleteObject(std::string_view Key) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 DELETE '{}' failed", Key); !Err.empty()) + { + return S3Result{std::move(Err)}; + } + std::string Path = KeyToPath(Key); - HttpClient::KeyValueMap Headers = SignRequest("DELETE", Path, "", EmptyPayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "DELETE", Path, "", EmptyPayloadHash); HttpClient::Response Response = m_HttpClient.Delete(Path, Headers); if (!Response.IsSuccess()) { - std::string Err = Response.ErrorMessage("S3 DELETE failed"); + std::string Err = S3ErrorMessage("S3 DELETE failed", Response); ZEN_WARN("S3 DELETE '{}' failed: {}", Key, Err); return S3Result{std::move(Err)}; } @@ -459,6 +527,12 @@ S3Client::DeleteObject(std::string_view Key) S3Result S3Client::Touch(std::string_view Key) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 Touch '{}' failed", Key); !Err.empty()) + { + return S3Result{std::move(Err)}; + } + std::string Path = KeyToPath(Key); // x-amz-copy-source is always "/bucket/key" regardless of addressing style. @@ -469,23 +543,23 @@ S3Client::Touch(std::string_view Key) {"x-amz-metadata-directive", "REPLACE"}, }}; - HttpClient::KeyValueMap Headers = SignRequest("PUT", Path, "", EmptyPayloadHash, ExtraSigned); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "PUT", Path, "", EmptyPayloadHash, ExtraSigned); HttpClient::Response Response = m_HttpClient.Put(Path, IoBuffer{}, Headers); if (!Response.IsSuccess()) { - std::string Err = Response.ErrorMessage("S3 Touch failed"); + std::string Err = S3ErrorMessage("S3 Touch failed", Response); ZEN_WARN("S3 Touch '{}' failed: {}", Key, Err); return S3Result{std::move(Err)}; } // Copy operations can return HTTP 200 with an error in the XML body. std::string_view ResponseBody = Response.AsText(); - if (ResponseBody.find("<Error>") != std::string_view::npos) + std::string_view ErrorCode; + std::string_view ErrorMessage; + if (ExtractS3Error(ResponseBody, ErrorCode, ErrorMessage)) { - std::string_view ErrorCode = ExtractXmlValue(ResponseBody, "Code"); - std::string_view ErrorMessage = ExtractXmlValue(ResponseBody, "Message"); - std::string Err = fmt::format("S3 Touch '{}' returned error: {} - {}", Key, ErrorCode, ErrorMessage); + std::string Err = fmt::format("S3 Touch '{}' returned error: {} - {}", Key, ErrorCode, ErrorMessage); ZEN_WARN("{}", Err); return S3Result{std::move(Err)}; } @@ -500,9 +574,15 @@ S3Client::Touch(std::string_view Key) S3HeadObjectResult S3Client::HeadObject(std::string_view Key) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 HEAD '{}' failed", Key); !Err.empty()) + { + return S3HeadObjectResult{S3Result{std::move(Err)}, {}, HeadObjectResult::Error}; + } + std::string Path = KeyToPath(Key); - HttpClient::KeyValueMap Headers = SignRequest("HEAD", Path, "", EmptyPayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "HEAD", Path, "", EmptyPayloadHash); HttpClient::Response Response = m_HttpClient.Head(Path, Headers); if (!Response.IsSuccess()) @@ -512,7 +592,7 @@ S3Client::HeadObject(std::string_view Key) return S3HeadObjectResult{{}, {}, HeadObjectResult::NotFound}; } - std::string Err = Response.ErrorMessage("S3 HEAD failed"); + std::string Err = S3ErrorMessage("S3 HEAD failed", Response); ZEN_WARN("S3 HEAD '{}' failed: {}", Key, Err); return S3HeadObjectResult{S3Result{std::move(Err)}, {}, HeadObjectResult::Error}; } @@ -551,6 +631,13 @@ S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys) for (;;) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 ListObjectsV2 prefix='{}' failed", Prefix); !Err.empty()) + { + Result.Error = std::move(Err); + return Result; + } + // Build query parameters for ListObjectsV2 std::vector<std::pair<std::string, std::string>> QueryParams; QueryParams.emplace_back("list-type", "2"); @@ -569,13 +656,13 @@ S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys) std::string CanonicalQS = BuildCanonicalQueryString(std::move(QueryParams)); std::string RootPath = BucketRootPath(); - HttpClient::KeyValueMap Headers = SignRequest("GET", RootPath, CanonicalQS, EmptyPayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "GET", RootPath, CanonicalQS, EmptyPayloadHash); std::string FullPath = BuildRequestPath(RootPath, CanonicalQS); HttpClient::Response Response = m_HttpClient.Get(FullPath, Headers); if (!Response.IsSuccess()) { - std::string Err = Response.ErrorMessage("S3 ListObjectsV2 failed"); + std::string Err = S3ErrorMessage("S3 ListObjectsV2 failed", Response); ZEN_WARN("S3 ListObjectsV2 prefix='{}' failed: {}", Prefix, Err); Result.Error = std::move(Err); return Result; @@ -654,16 +741,22 @@ S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys) S3CreateMultipartUploadResult S3Client::CreateMultipartUpload(std::string_view Key) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 CreateMultipartUpload '{}' failed", Key); !Err.empty()) + { + return S3CreateMultipartUploadResult{S3Result{std::move(Err)}, {}}; + } + std::string Path = KeyToPath(Key); std::string CanonicalQS = BuildCanonicalQueryString({{"uploads", ""}}); - HttpClient::KeyValueMap Headers = SignRequest("POST", Path, CanonicalQS, EmptyPayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "POST", Path, CanonicalQS, EmptyPayloadHash); std::string FullPath = BuildRequestPath(Path, CanonicalQS); HttpClient::Response Response = m_HttpClient.Post(FullPath, Headers); if (!Response.IsSuccess()) { - std::string Err = Response.ErrorMessage("S3 CreateMultipartUpload failed"); + std::string Err = S3ErrorMessage("S3 CreateMultipartUpload failed", Response); ZEN_WARN("S3 CreateMultipartUpload '{}' failed: {}", Key, Err); return S3CreateMultipartUploadResult{S3Result{std::move(Err)}, {}}; } @@ -693,6 +786,12 @@ S3Client::CreateMultipartUpload(std::string_view Key) S3UploadPartResult S3Client::UploadPart(std::string_view Key, std::string_view UploadId, uint32_t PartNumber, IoBuffer Content) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 UploadPart '{}' part {} failed", Key, PartNumber); !Err.empty()) + { + return S3UploadPartResult{S3Result{std::move(Err)}, {}}; + } + std::string Path = KeyToPath(Key); std::string CanonicalQS = BuildCanonicalQueryString({ {"partNumber", fmt::format("{}", PartNumber)}, @@ -701,13 +800,13 @@ S3Client::UploadPart(std::string_view Key, std::string_view UploadId, uint32_t P std::string PayloadHash = Sha256ToHex(ComputeSha256(Content.GetData(), Content.GetSize())); - HttpClient::KeyValueMap Headers = SignRequest("PUT", Path, CanonicalQS, PayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "PUT", Path, CanonicalQS, PayloadHash); std::string FullPath = BuildRequestPath(Path, CanonicalQS); HttpClient::Response Response = m_HttpClient.Put(FullPath, Content, Headers); if (!Response.IsSuccess()) { - std::string Err = Response.ErrorMessage(fmt::format("S3 UploadPart {} failed", PartNumber)); + std::string Err = S3ErrorMessage(fmt::format("S3 UploadPart {} failed", PartNumber), Response); ZEN_WARN("S3 UploadPart '{}' part {} failed: {}", Key, PartNumber, Err); return S3UploadPartResult{S3Result{std::move(Err)}, {}}; } @@ -733,6 +832,12 @@ S3Client::CompleteMultipartUpload(std::string_view Key, std::string_view UploadId, const std::vector<std::pair<uint32_t, std::string>>& PartETags) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 CompleteMultipartUpload '{}' failed", Key); !Err.empty()) + { + return S3Result{std::move(Err)}; + } + std::string Path = KeyToPath(Key); std::string CanonicalQS = BuildCanonicalQueryString({{"uploadId", std::string(UploadId)}}); @@ -748,7 +853,7 @@ S3Client::CompleteMultipartUpload(std::string_view Key, std::string_view XmlView = XmlBody.ToView(); std::string PayloadHash = Sha256ToHex(ComputeSha256(XmlView)); - HttpClient::KeyValueMap Headers = SignRequest("POST", Path, CanonicalQS, PayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "POST", Path, CanonicalQS, PayloadHash); Headers->emplace("Content-Type", "application/xml"); IoBuffer Payload(IoBuffer::Clone, XmlView.data(), XmlView.size()); @@ -757,18 +862,18 @@ S3Client::CompleteMultipartUpload(std::string_view Key, HttpClient::Response Response = m_HttpClient.Post(FullPath, Payload, Headers); if (!Response.IsSuccess()) { - std::string Err = Response.ErrorMessage("S3 CompleteMultipartUpload failed"); + std::string Err = S3ErrorMessage("S3 CompleteMultipartUpload failed", Response); ZEN_WARN("S3 CompleteMultipartUpload '{}' failed: {}", Key, Err); return S3Result{std::move(Err)}; } // Check for error in response body - S3 can return 200 with an error in the XML body std::string_view ResponseBody = Response.AsText(); - if (ResponseBody.find("<Error>") != std::string_view::npos) + std::string_view ErrorCode; + std::string_view ErrorMessage; + if (ExtractS3Error(ResponseBody, ErrorCode, ErrorMessage)) { - std::string_view ErrorCode = ExtractXmlValue(ResponseBody, "Code"); - std::string_view ErrorMessage = ExtractXmlValue(ResponseBody, "Message"); - std::string Err = fmt::format("S3 CompleteMultipartUpload '{}' returned error: {} - {}", Key, ErrorCode, ErrorMessage); + std::string Err = fmt::format("S3 CompleteMultipartUpload '{}' returned error: {} - {}", Key, ErrorCode, ErrorMessage); ZEN_WARN("{}", Err); return S3Result{std::move(Err)}; } @@ -783,16 +888,22 @@ S3Client::CompleteMultipartUpload(std::string_view Key, S3Result S3Client::AbortMultipartUpload(std::string_view Key, std::string_view UploadId) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 AbortMultipartUpload '{}' failed", Key); !Err.empty()) + { + return S3Result{std::move(Err)}; + } + std::string Path = KeyToPath(Key); std::string CanonicalQS = BuildCanonicalQueryString({{"uploadId", std::string(UploadId)}}); - HttpClient::KeyValueMap Headers = SignRequest("DELETE", Path, CanonicalQS, EmptyPayloadHash); + HttpClient::KeyValueMap Headers = SignRequest(Credentials, "DELETE", Path, CanonicalQS, EmptyPayloadHash); std::string FullPath = BuildRequestPath(Path, CanonicalQS); HttpClient::Response Response = m_HttpClient.Delete(FullPath, Headers); if (!Response.IsSuccess()) { - std::string Err = Response.ErrorMessage("S3 AbortMultipartUpload failed"); + std::string Err = S3ErrorMessage("S3 AbortMultipartUpload failed", Response); ZEN_WARN("S3 AbortMultipartUpload '{}' failed: {}", Key, Err); return S3Result{std::move(Err)}; } @@ -819,6 +930,12 @@ S3Client::GeneratePresignedPutUrl(std::string_view Key, std::chrono::seconds Exp std::string S3Client::GeneratePresignedUrlForMethod(std::string_view Key, std::string_view Method, std::chrono::seconds ExpiresIn) { + SigV4Credentials Credentials; + if (std::string Err = RequireCredentials(Credentials, "S3 GeneratePresignedUrl '{}' {} failed", Key, Method); !Err.empty()) + { + return {}; + } + std::string Path = KeyToPath(Key); std::string Scheme = "https"; @@ -827,7 +944,6 @@ S3Client::GeneratePresignedUrlForMethod(std::string_view Key, std::string_view M Scheme = "http"; } - SigV4Credentials Credentials = GetCurrentCredentials(); return GeneratePresignedUrl(Credentials, Method, Scheme, m_Host, Path, m_Region, "s3", ExpiresIn); } |