aboutsummaryrefslogtreecommitdiff
path: root/src/zenutil/cloud
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2026-03-18 11:27:07 +0100
committerGitHub Enterprise <[email protected]>2026-03-18 11:27:07 +0100
commite64d76ae1b6993582bf161a61049f0771414a779 (patch)
tree083f3df42cc9e2c7ddbee225708b4848eb217d11 /src/zenutil/cloud
parentCompute batching (#849) (diff)
downloadzen-e64d76ae1b6993582bf161a61049f0771414a779.tar.xz
zen-e64d76ae1b6993582bf161a61049f0771414a779.zip
Simple S3 client (#836)
This functionality is intended to be used to manage datasets for test cases, but may be useful elsewhere in the future. - **Add S3 client with AWS Signature V4 (SigV4) signing** — new `S3Client` in `zenutil/cloud/` supporting `GetObject`, `PutObject`, `DeleteObject`, `HeadObject`, and `ListObjects` operations - **Add EC2 IMDS credential provider** — automatically fetches and refreshes temporary AWS credentials from the EC2 Instance Metadata Service (IMDSv2) for use by the S3 client - **Add SigV4 signing library** — standalone implementation of AWS Signature Version 4 request signing (headers and query-string presigning) - **Add path-style addressing support** — enables compatibility with S3-compatible stores like MinIO (in addition to virtual-hosted style) - **Add S3 integration tests** — includes a `MinioProcess` test helper that spins up a local MinIO server, plus integration tests exercising the S3 client end-to-end - **Add S3-backed `HttpObjectStoreService` tests** — integration tests verifying the zenserver object store works against an S3 backend - **Refactor mock IMDS into `zenutil/cloud/`** — moved and generalized the mock IMDS server from `zencompute` so it can be reused by both compute and S3 credential tests
Diffstat (limited to 'src/zenutil/cloud')
-rw-r--r--src/zenutil/cloud/cloudprovider.cpp23
-rw-r--r--src/zenutil/cloud/imdscredentials.cpp387
-rw-r--r--src/zenutil/cloud/minioprocess.cpp174
-rw-r--r--src/zenutil/cloud/mockimds.cpp237
-rw-r--r--src/zenutil/cloud/s3client.cpp986
-rw-r--r--src/zenutil/cloud/sigv4.cpp531
6 files changed, 2338 insertions, 0 deletions
diff --git a/src/zenutil/cloud/cloudprovider.cpp b/src/zenutil/cloud/cloudprovider.cpp
new file mode 100644
index 000000000..e32a50c64
--- /dev/null
+++ b/src/zenutil/cloud/cloudprovider.cpp
@@ -0,0 +1,23 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/cloud/cloudprovider.h>
+
+namespace zen::compute {
+
+std::string_view
+ToString(CloudProvider Provider)
+{
+ switch (Provider)
+ {
+ case CloudProvider::AWS:
+ return "AWS";
+ case CloudProvider::Azure:
+ return "Azure";
+ case CloudProvider::GCP:
+ return "GCP";
+ default:
+ return "None";
+ }
+}
+
+} // namespace zen::compute
diff --git a/src/zenutil/cloud/imdscredentials.cpp b/src/zenutil/cloud/imdscredentials.cpp
new file mode 100644
index 000000000..dde1dc019
--- /dev/null
+++ b/src/zenutil/cloud/imdscredentials.cpp
@@ -0,0 +1,387 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/cloud/imdscredentials.h>
+
+#include <zenutil/cloud/mockimds.h>
+
+#include <zencore/string.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
+#include <zenhttp/httpserver.h>
+
+#include <thread>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+#include <json11.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+namespace {
+
+ /// Margin before expiration at which we proactively refresh credentials.
+ constexpr auto kRefreshMargin = std::chrono::minutes(5);
+
+ /// Parse an ISO 8601 UTC timestamp (e.g. "2026-03-14T20:00:00Z") into a system_clock time_point.
+ /// Returns epoch on failure.
+ std::chrono::system_clock::time_point ParseIso8601(std::string_view Timestamp)
+ {
+ // Expected format: YYYY-MM-DDTHH:MM:SSZ
+ if (Timestamp.size() < 19)
+ {
+ return {};
+ }
+
+ std::tm Tm = {};
+ // Manual parse since std::get_time is locale-dependent
+ Tm.tm_year = ParseInt<int>(Timestamp.substr(0, 4)).value_or(1970) - 1900;
+ Tm.tm_mon = ParseInt<int>(Timestamp.substr(5, 2)).value_or(1) - 1;
+ Tm.tm_mday = ParseInt<int>(Timestamp.substr(8, 2)).value_or(1);
+ Tm.tm_hour = ParseInt<int>(Timestamp.substr(11, 2)).value_or(0);
+ Tm.tm_min = ParseInt<int>(Timestamp.substr(14, 2)).value_or(0);
+ Tm.tm_sec = ParseInt<int>(Timestamp.substr(17, 2)).value_or(0);
+
+#if ZEN_PLATFORM_WINDOWS
+ time_t EpochSeconds = _mkgmtime(&Tm);
+#else
+ time_t EpochSeconds = timegm(&Tm);
+#endif
+ if (EpochSeconds == -1)
+ {
+ return {};
+ }
+
+ return std::chrono::system_clock::from_time_t(EpochSeconds);
+ }
+
+} // namespace
+
+ImdsCredentialProvider::ImdsCredentialProvider(const ImdsCredentialProviderOptions& Options)
+: m_Log(logging::Get("imds"))
+, m_HttpClient(Options.Endpoint,
+ HttpClientSettings{
+ .LogCategory = "imds",
+ .ConnectTimeout = Options.ConnectTimeout,
+ .Timeout = Options.RequestTimeout,
+ })
+{
+ ZEN_INFO("IMDS credential provider configured (endpoint: {})", m_HttpClient.GetBaseUri());
+}
+
+ImdsCredentialProvider::~ImdsCredentialProvider() = default;
+
+SigV4Credentials
+ImdsCredentialProvider::GetCredentials()
+{
+ // Fast path: shared lock for cache hit
+ {
+ RwLock::SharedLockScope SharedLock(m_Lock);
+ if (!m_CachedCredentials.AccessKeyId.empty() && std::chrono::steady_clock::now() < m_ExpiresAt)
+ {
+ return m_CachedCredentials;
+ }
+ }
+
+ // Slow path: exclusive lock to refresh
+ RwLock::ExclusiveLockScope ExclusiveLock(m_Lock);
+
+ // Double-check after acquiring exclusive lock
+ if (!m_CachedCredentials.AccessKeyId.empty() && std::chrono::steady_clock::now() < m_ExpiresAt)
+ {
+ return m_CachedCredentials;
+ }
+
+ if (!FetchCredentials())
+ {
+ ZEN_WARN("failed to fetch credentials from IMDS");
+ return {};
+ }
+
+ return m_CachedCredentials;
+}
+
+void
+ImdsCredentialProvider::InvalidateCache()
+{
+ RwLock::ExclusiveLockScope ExclusiveLock(m_Lock);
+ m_CachedCredentials = {};
+ m_ExpiresAt = {};
+}
+
+bool
+ImdsCredentialProvider::FetchToken()
+{
+ HttpClient::KeyValueMap Headers;
+ Headers->emplace("X-aws-ec2-metadata-token-ttl-seconds", "21600");
+
+ HttpClient::Response Response = m_HttpClient.Put("/latest/api/token", Headers);
+ if (!Response.IsSuccess())
+ {
+ ZEN_WARN("IMDS token request failed: {}", Response.ErrorMessage("PUT /latest/api/token"));
+ return false;
+ }
+
+ m_ImdsToken = std::string(Response.AsText());
+ if (m_ImdsToken.empty())
+ {
+ ZEN_WARN("IMDS returned empty token");
+ return false;
+ }
+
+ return true;
+}
+
+bool
+ImdsCredentialProvider::FetchCredentials()
+{
+ // Step 1: Get IMDSv2 session token
+ if (!FetchToken())
+ {
+ return false;
+ }
+
+ HttpClient::KeyValueMap TokenHeader;
+ TokenHeader->emplace("X-aws-ec2-metadata-token", m_ImdsToken);
+
+ // Step 2: Discover IAM role name (if not already known)
+ if (m_RoleName.empty())
+ {
+ HttpClient::Response RoleResponse = m_HttpClient.Get("/latest/meta-data/iam/security-credentials/", TokenHeader);
+ if (!RoleResponse.IsSuccess())
+ {
+ ZEN_WARN("IMDS role discovery failed: {}", RoleResponse.ErrorMessage("GET iam/security-credentials/"));
+ return false;
+ }
+
+ m_RoleName = std::string(RoleResponse.AsText());
+ // Trim any trailing whitespace/newlines
+ while (!m_RoleName.empty() && (m_RoleName.back() == '\n' || m_RoleName.back() == '\r' || m_RoleName.back() == ' '))
+ {
+ m_RoleName.pop_back();
+ }
+
+ if (m_RoleName.empty())
+ {
+ ZEN_WARN("IMDS returned empty IAM role name");
+ return false;
+ }
+
+ ZEN_INFO("IMDS discovered IAM role: {}", m_RoleName);
+ }
+
+ // Step 3: Fetch credentials for the role
+ std::string CredentialPath = fmt::format("/latest/meta-data/iam/security-credentials/{}", m_RoleName);
+
+ HttpClient::Response CredResponse = m_HttpClient.Get(CredentialPath, TokenHeader);
+ if (!CredResponse.IsSuccess())
+ {
+ ZEN_WARN("IMDS credential fetch failed: {}", CredResponse.ErrorMessage("GET iam/security-credentials/" + m_RoleName));
+ return false;
+ }
+
+ // Step 4: Parse JSON response
+ std::string JsonError;
+ const json11::Json Json = json11::Json::parse(std::string(CredResponse.AsText()), JsonError);
+
+ if (!JsonError.empty())
+ {
+ ZEN_WARN("IMDS credential response JSON parse error: {}", JsonError);
+ return false;
+ }
+
+ std::string AccessKeyId = Json["AccessKeyId"].string_value();
+ std::string SecretAccessKey = Json["SecretAccessKey"].string_value();
+ std::string SessionToken = Json["Token"].string_value();
+ std::string Expiration = Json["Expiration"].string_value();
+
+ if (AccessKeyId.empty() || SecretAccessKey.empty())
+ {
+ ZEN_WARN("IMDS credential response missing AccessKeyId or SecretAccessKey");
+ return false;
+ }
+
+ // Compute local expiration time based on the Expiration field
+ auto ExpirationTime = ParseIso8601(Expiration);
+ auto Now = std::chrono::system_clock::now();
+
+ std::chrono::steady_clock::time_point NewExpiresAt;
+ if (ExpirationTime > Now)
+ {
+ auto TimeUntilExpiry = ExpirationTime - Now;
+ NewExpiresAt = std::chrono::steady_clock::now() + TimeUntilExpiry - kRefreshMargin;
+ }
+ else
+ {
+ // Expiration is in the past or unparseable — force refresh next time
+ NewExpiresAt = std::chrono::steady_clock::now();
+ }
+
+ bool KeyChanged = (m_CachedCredentials.AccessKeyId != AccessKeyId);
+
+ m_CachedCredentials.AccessKeyId = std::move(AccessKeyId);
+ m_CachedCredentials.SecretAccessKey = std::move(SecretAccessKey);
+ m_CachedCredentials.SessionToken = std::move(SessionToken);
+ m_ExpiresAt = NewExpiresAt;
+
+ if (KeyChanged)
+ {
+ ZEN_INFO("IMDS credentials refreshed (AccessKeyId: {}...)", m_CachedCredentials.AccessKeyId.substr(0, 8));
+ }
+ else
+ {
+ ZEN_DEBUG("IMDS credentials refreshed (unchanged key)");
+ }
+
+ return true;
+}
+
+//////////////////////////////////////////////////////////////////////////
+// Tests
+
+#if ZEN_WITH_TESTS
+
+void
+imdscredentials_forcelink()
+{
+}
+
+TEST_SUITE_BEGIN("util.cloud.imdscredentials");
+
+TEST_CASE("imdscredentials.parse_iso8601")
+{
+ // Verify basic ISO 8601 parsing
+ auto Tp = ParseIso8601("2026-03-14T20:00:00Z");
+ CHECK(Tp != std::chrono::system_clock::time_point{});
+
+ auto Epoch = std::chrono::system_clock::to_time_t(Tp);
+ std::tm Tm;
+# if ZEN_PLATFORM_WINDOWS
+ gmtime_s(&Tm, &Epoch);
+# else
+ gmtime_r(&Epoch, &Tm);
+# endif
+ CHECK(Tm.tm_year + 1900 == 2026);
+ CHECK(Tm.tm_mon + 1 == 3);
+ CHECK(Tm.tm_mday == 14);
+ CHECK(Tm.tm_hour == 20);
+ CHECK(Tm.tm_min == 0);
+ CHECK(Tm.tm_sec == 0);
+
+ // Invalid input
+ auto Bad = ParseIso8601("bad");
+ CHECK(Bad == std::chrono::system_clock::time_point{});
+}
+
+// ---------------------------------------------------------------------------
+// Integration test with mock IMDS server
+// ---------------------------------------------------------------------------
+
+struct TestImdsServer
+{
+ compute::MockImdsService Mock;
+
+ void Start()
+ {
+ m_TmpDir.emplace();
+ m_Server = CreateHttpServer(HttpServerConfig{.ServerClass = "asio"});
+ m_Port = m_Server->Initialize(7576, m_TmpDir->Path() / "http");
+ REQUIRE(m_Port != -1);
+ m_Server->RegisterService(Mock);
+ m_ServerThread = std::thread([this]() { m_Server->Run(false); });
+ }
+
+ std::string Endpoint() const { return fmt::format("http://127.0.0.1:{}", m_Port); }
+
+ ~TestImdsServer()
+ {
+ if (m_Server)
+ {
+ m_Server->RequestExit();
+ }
+ if (m_ServerThread.joinable())
+ {
+ m_ServerThread.join();
+ }
+ if (m_Server)
+ {
+ m_Server->Close();
+ }
+ }
+
+private:
+ std::optional<ScopedTemporaryDirectory> m_TmpDir;
+ Ref<HttpServer> m_Server;
+ std::thread m_ServerThread;
+ int m_Port = -1;
+};
+
+TEST_CASE("imdscredentials.fetch_from_mock")
+{
+ TestImdsServer Imds;
+ Imds.Mock.ActiveProvider = compute::CloudProvider::AWS;
+ Imds.Start();
+
+ ImdsCredentialProviderOptions Opts;
+ Opts.Endpoint = Imds.Endpoint();
+
+ Ref<ImdsCredentialProvider> Provider(new ImdsCredentialProvider(Opts));
+
+ SUBCASE("basic_credential_fetch")
+ {
+ SigV4Credentials Creds = Provider->GetCredentials();
+ CHECK(!Creds.AccessKeyId.empty());
+ CHECK(Creds.AccessKeyId == "ASIAIOSFODNN7EXAMPLE");
+ CHECK(Creds.SecretAccessKey == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
+ CHECK(Creds.SessionToken == "FwoGZXIvYXdzEBYaDEXAMPLETOKEN");
+ }
+
+ SUBCASE("credentials_are_cached")
+ {
+ SigV4Credentials First = Provider->GetCredentials();
+ SigV4Credentials Second = Provider->GetCredentials();
+ CHECK(First.AccessKeyId == Second.AccessKeyId);
+ CHECK(First.SecretAccessKey == Second.SecretAccessKey);
+ }
+
+ SUBCASE("invalidate_forces_refresh")
+ {
+ SigV4Credentials First = Provider->GetCredentials();
+ CHECK(!First.AccessKeyId.empty());
+
+ // Change the credentials on the mock
+ Imds.Mock.Aws.IamAccessKeyId = "ASIANEWKEYEXAMPLE12";
+
+ Provider->InvalidateCache();
+ SigV4Credentials Second = Provider->GetCredentials();
+ CHECK(Second.AccessKeyId == "ASIANEWKEYEXAMPLE12");
+ }
+
+ SUBCASE("custom_role_name")
+ {
+ Imds.Mock.Aws.IamRoleName = "my-custom-role";
+
+ Ref<ImdsCredentialProvider> Provider2(new ImdsCredentialProvider(Opts));
+ SigV4Credentials Creds = Provider2->GetCredentials();
+ CHECK(!Creds.AccessKeyId.empty());
+ }
+}
+
+TEST_CASE("imdscredentials.unreachable_endpoint")
+{
+ // Point at a non-existent server — should return empty credentials, not crash
+ ImdsCredentialProviderOptions Opts;
+ Opts.Endpoint = "http://127.0.0.1:1"; // unlikely to have anything listening
+ Opts.ConnectTimeout = std::chrono::milliseconds(100);
+ Opts.RequestTimeout = std::chrono::milliseconds(200);
+
+ Ref<ImdsCredentialProvider> Provider(new ImdsCredentialProvider(Opts));
+ SigV4Credentials Creds = Provider->GetCredentials();
+ CHECK(Creds.AccessKeyId.empty());
+}
+
+TEST_SUITE_END();
+
+#endif
+
+} // namespace zen
diff --git a/src/zenutil/cloud/minioprocess.cpp b/src/zenutil/cloud/minioprocess.cpp
new file mode 100644
index 000000000..565705731
--- /dev/null
+++ b/src/zenutil/cloud/minioprocess.cpp
@@ -0,0 +1,174 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/cloud/minioprocess.h>
+
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/process.h>
+#include <zencore/timer.h>
+#include <zenhttp/httpclient.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+struct MinioProcess::Impl
+{
+ Impl(const MinioProcessOptions& Options) : m_Options(Options), m_HttpClient(fmt::format("http://localhost:{}/", Options.Port)) {}
+ ~Impl() = default;
+
+ void SpawnMinioServer()
+ {
+ if (m_ProcessHandle.IsValid())
+ {
+ return;
+ }
+
+ // Create a clean temp data directory, removing any stale data from a previous run
+ std::error_code Ec;
+ m_DataDir = std::filesystem::temp_directory_path(Ec) / fmt::format("zen-minio-{}", GetCurrentProcessId());
+ if (Ec)
+ {
+ ZEN_WARN("MinIO: Failed to get temp directory: {}", Ec.message());
+ return;
+ }
+ std::filesystem::remove_all(m_DataDir, Ec);
+ Ec.clear();
+ std::filesystem::create_directories(m_DataDir, Ec);
+ if (Ec)
+ {
+ ZEN_WARN("MinIO: Failed to create data directory '{}': {}", m_DataDir.string(), Ec.message());
+ return;
+ }
+
+ CreateProcOptions Options;
+ Options.Flags |= CreateProcOptions::Flag_Windows_NewProcessGroup;
+ Options.Environment.emplace_back("MINIO_ROOT_USER", m_Options.RootUser);
+ Options.Environment.emplace_back("MINIO_ROOT_PASSWORD", m_Options.RootPassword);
+
+ const std::filesystem::path MinioExe = GetRunningExecutablePath().parent_path() / ("minio" ZEN_EXE_SUFFIX_LITERAL);
+
+ std::string CommandLine =
+ fmt::format("minio" ZEN_EXE_SUFFIX_LITERAL " server {} --address :{} --quiet", m_DataDir.string(), m_Options.Port);
+
+ CreateProcResult Result = CreateProc(MinioExe, CommandLine, Options);
+
+ if (Result)
+ {
+ m_ProcessHandle.Initialize(Result);
+
+ Stopwatch Timer;
+
+ // Poll to check when the server is ready
+ do
+ {
+ Sleep(100);
+ HttpClient::Response Resp = m_HttpClient.Get("minio/health/live");
+ if (Resp)
+ {
+ ZEN_INFO("MinIO server started successfully (waited {})", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ return;
+ }
+ } while (Timer.GetElapsedTimeMs() < 10000);
+ }
+
+ // Report failure
+ ZEN_WARN("MinIO server failed to start within timeout period");
+ }
+
+ void StopMinioServer()
+ {
+ if (!m_ProcessHandle.IsValid())
+ {
+ return;
+ }
+
+ m_ProcessHandle.Kill();
+
+ // Clean up temp data directory
+ std::error_code Ec;
+ std::filesystem::remove_all(m_DataDir, Ec);
+ if (Ec)
+ {
+ ZEN_WARN("MinIO: Failed to clean up data directory '{}': {}", m_DataDir.string(), Ec.message());
+ }
+ }
+
+ void CreateBucket(std::string_view BucketName)
+ {
+ if (m_DataDir.empty())
+ {
+ ZEN_WARN("MinIO: Cannot create bucket before data directory is initialized — call SpawnMinioServer() first");
+ return;
+ }
+
+ std::filesystem::path BucketDir = m_DataDir / std::string(BucketName);
+ std::error_code Ec;
+ std::filesystem::create_directories(BucketDir, Ec);
+ if (Ec)
+ {
+ ZEN_WARN("MinIO: Failed to create bucket directory '{}': {}", BucketDir.string(), Ec.message());
+ }
+ }
+
+ MinioProcessOptions m_Options;
+ ProcessHandle m_ProcessHandle;
+ HttpClient m_HttpClient;
+ std::filesystem::path m_DataDir;
+};
+
+MinioProcess::MinioProcess(const MinioProcessOptions& Options) : m_Impl(std::make_unique<Impl>(Options))
+{
+}
+
+MinioProcess::~MinioProcess()
+{
+ m_Impl->StopMinioServer();
+}
+
+void
+MinioProcess::SpawnMinioServer()
+{
+ m_Impl->SpawnMinioServer();
+}
+
+void
+MinioProcess::StopMinioServer()
+{
+ m_Impl->StopMinioServer();
+}
+
+void
+MinioProcess::CreateBucket(std::string_view BucketName)
+{
+ m_Impl->CreateBucket(BucketName);
+}
+
+uint16_t
+MinioProcess::Port() const
+{
+ return m_Impl->m_Options.Port;
+}
+
+std::string_view
+MinioProcess::RootUser() const
+{
+ return m_Impl->m_Options.RootUser;
+}
+
+std::string_view
+MinioProcess::RootPassword() const
+{
+ return m_Impl->m_Options.RootPassword;
+}
+
+std::string
+MinioProcess::Endpoint() const
+{
+ return fmt::format("http://localhost:{}", m_Impl->m_Options.Port);
+}
+
+} // namespace zen
diff --git a/src/zenutil/cloud/mockimds.cpp b/src/zenutil/cloud/mockimds.cpp
new file mode 100644
index 000000000..6919fab4d
--- /dev/null
+++ b/src/zenutil/cloud/mockimds.cpp
@@ -0,0 +1,237 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/cloud/mockimds.h>
+
+#include <zencore/fmtutils.h>
+
+#if ZEN_WITH_TESTS
+
+namespace zen::compute {
+
+const char*
+MockImdsService::BaseUri() const
+{
+ return "/";
+}
+
+void
+MockImdsService::HandleRequest(HttpServerRequest& Request)
+{
+ std::string_view Uri = Request.RelativeUri();
+
+ // AWS endpoints live under /latest/
+ if (Uri.starts_with("latest/"))
+ {
+ if (ActiveProvider == CloudProvider::AWS)
+ {
+ HandleAwsRequest(Request);
+ return;
+ }
+ Request.WriteResponse(HttpResponseCode::NotFound);
+ return;
+ }
+
+ // Azure endpoints live under /metadata/
+ if (Uri.starts_with("metadata/"))
+ {
+ if (ActiveProvider == CloudProvider::Azure)
+ {
+ HandleAzureRequest(Request);
+ return;
+ }
+ Request.WriteResponse(HttpResponseCode::NotFound);
+ return;
+ }
+
+ // GCP endpoints live under /computeMetadata/
+ if (Uri.starts_with("computeMetadata/"))
+ {
+ if (ActiveProvider == CloudProvider::GCP)
+ {
+ HandleGcpRequest(Request);
+ return;
+ }
+ Request.WriteResponse(HttpResponseCode::NotFound);
+ return;
+ }
+
+ Request.WriteResponse(HttpResponseCode::NotFound);
+}
+
+// ---------------------------------------------------------------------------
+// AWS
+// ---------------------------------------------------------------------------
+
+void
+MockImdsService::HandleAwsRequest(HttpServerRequest& Request)
+{
+ std::string_view Uri = Request.RelativeUri();
+
+ // IMDSv2 token acquisition (PUT only)
+ if (Uri == "latest/api/token" && Request.RequestVerb() == HttpVerb::kPut)
+ {
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.Token);
+ return;
+ }
+
+ // Instance identity
+ if (Uri == "latest/meta-data/instance-id")
+ {
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.InstanceId);
+ return;
+ }
+
+ if (Uri == "latest/meta-data/placement/availability-zone")
+ {
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.AvailabilityZone);
+ return;
+ }
+
+ if (Uri == "latest/meta-data/instance-life-cycle")
+ {
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.LifeCycle);
+ return;
+ }
+
+ // Autoscaling lifecycle state — 404 when not in an ASG
+ if (Uri == "latest/meta-data/autoscaling/target-lifecycle-state")
+ {
+ if (Aws.AutoscalingState.empty())
+ {
+ Request.WriteResponse(HttpResponseCode::NotFound);
+ return;
+ }
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.AutoscalingState);
+ return;
+ }
+
+ // Spot interruption notice — 404 when no interruption pending
+ if (Uri == "latest/meta-data/spot/instance-action")
+ {
+ if (Aws.SpotAction.empty())
+ {
+ Request.WriteResponse(HttpResponseCode::NotFound);
+ return;
+ }
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.SpotAction);
+ return;
+ }
+
+ // IAM role discovery — returns the role name
+ if (Uri == "latest/meta-data/iam/security-credentials/")
+ {
+ if (Aws.IamRoleName.empty())
+ {
+ Request.WriteResponse(HttpResponseCode::NotFound);
+ return;
+ }
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.IamRoleName);
+ return;
+ }
+
+ // IAM credentials for a specific role
+ constexpr std::string_view kIamCredPrefix = "latest/meta-data/iam/security-credentials/";
+ if (Uri.starts_with(kIamCredPrefix) && Uri.size() > kIamCredPrefix.size())
+ {
+ std::string_view RequestedRole = Uri.substr(kIamCredPrefix.size());
+ if (RequestedRole == Aws.IamRoleName)
+ {
+ std::string Json =
+ fmt::format(R"({{"Code":"Success","AccessKeyId":"{}","SecretAccessKey":"{}","Token":"{}","Expiration":"{}"}})",
+ Aws.IamAccessKeyId,
+ Aws.IamSecretAccessKey,
+ Aws.IamSessionToken,
+ Aws.IamExpiration);
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Json);
+ return;
+ }
+ Request.WriteResponse(HttpResponseCode::NotFound);
+ return;
+ }
+
+ Request.WriteResponse(HttpResponseCode::NotFound);
+}
+
+// ---------------------------------------------------------------------------
+// Azure
+// ---------------------------------------------------------------------------
+
+void
+MockImdsService::HandleAzureRequest(HttpServerRequest& Request)
+{
+ std::string_view Uri = Request.RelativeUri();
+
+ // Instance metadata (single JSON document)
+ if (Uri == "metadata/instance")
+ {
+ std::string Json = fmt::format(R"({{"compute":{{"vmId":"{}","location":"{}","priority":"{}","vmScaleSetName":"{}"}}}})",
+ Azure.VmId,
+ Azure.Location,
+ Azure.Priority,
+ Azure.VmScaleSetName);
+
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Json);
+ return;
+ }
+
+ // Scheduled events for termination monitoring
+ if (Uri == "metadata/scheduledevents")
+ {
+ std::string Json;
+ if (Azure.ScheduledEventType.empty())
+ {
+ Json = R"({"Events":[]})";
+ }
+ else
+ {
+ Json = fmt::format(R"({{"Events":[{{"EventType":"{}","EventStatus":"{}"}}]}})",
+ Azure.ScheduledEventType,
+ Azure.ScheduledEventStatus);
+ }
+
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Json);
+ return;
+ }
+
+ Request.WriteResponse(HttpResponseCode::NotFound);
+}
+
+// ---------------------------------------------------------------------------
+// GCP
+// ---------------------------------------------------------------------------
+
+void
+MockImdsService::HandleGcpRequest(HttpServerRequest& Request)
+{
+ std::string_view Uri = Request.RelativeUri();
+
+ if (Uri == "computeMetadata/v1/instance/id")
+ {
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.InstanceId);
+ return;
+ }
+
+ if (Uri == "computeMetadata/v1/instance/zone")
+ {
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.Zone);
+ return;
+ }
+
+ if (Uri == "computeMetadata/v1/instance/scheduling/preemptible")
+ {
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.Preemptible);
+ return;
+ }
+
+ if (Uri == "computeMetadata/v1/instance/maintenance-event")
+ {
+ Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.MaintenanceEvent);
+ return;
+ }
+
+ Request.WriteResponse(HttpResponseCode::NotFound);
+}
+
+} // namespace zen::compute
+
+#endif // ZEN_WITH_TESTS
diff --git a/src/zenutil/cloud/s3client.cpp b/src/zenutil/cloud/s3client.cpp
new file mode 100644
index 000000000..88d844b61
--- /dev/null
+++ b/src/zenutil/cloud/s3client.cpp
@@ -0,0 +1,986 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/cloud/s3client.h>
+
+#include <zenutil/cloud/imdscredentials.h>
+#include <zenutil/cloud/minioprocess.h>
+
+#include <zencore/except_fmt.h>
+#include <zencore/iobuffer.h>
+#include <zencore/memoryview.h>
+#include <zencore/string.h>
+#include <zencore/testing.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#include <algorithm>
+
+namespace zen {
+
+namespace {
+
+ /// The SHA-256 hash of an empty payload, precomputed
+ constexpr std::string_view EmptyPayloadHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+
+ /// Simple XML value extractor. Finds the text content between <Tag> and </Tag>.
+ /// This is intentionally minimal - we only need to parse ListBucketResult responses.
+ /// Returns a string_view into the original XML when no entity decoding is needed.
+ std::string_view ExtractXmlValue(std::string_view Xml, std::string_view Tag)
+ {
+ std::string OpenTag = fmt::format("<{}>", Tag);
+ std::string CloseTag = fmt::format("</{}>", Tag);
+
+ size_t Start = Xml.find(OpenTag);
+ if (Start == std::string_view::npos)
+ {
+ return {};
+ }
+ Start += OpenTag.size();
+
+ size_t End = Xml.find(CloseTag, Start);
+ if (End == std::string_view::npos)
+ {
+ return {};
+ }
+
+ return Xml.substr(Start, End - Start);
+ }
+
+ /// Decode the five standard XML entities (&amp; &lt; &gt; &quot; &apos;) into a StringBuilderBase.
+ void DecodeXmlEntities(std::string_view Input, StringBuilderBase& Out)
+ {
+ if (Input.find('&') == std::string_view::npos)
+ {
+ Out.Append(Input);
+ return;
+ }
+
+ for (size_t i = 0; i < Input.size(); ++i)
+ {
+ if (Input[i] == '&')
+ {
+ std::string_view Remaining = Input.substr(i);
+ if (Remaining.starts_with("&amp;"))
+ {
+ Out.Append('&');
+ i += 4;
+ }
+ else if (Remaining.starts_with("&lt;"))
+ {
+ Out.Append('<');
+ i += 3;
+ }
+ else if (Remaining.starts_with("&gt;"))
+ {
+ Out.Append('>');
+ i += 3;
+ }
+ else if (Remaining.starts_with("&quot;"))
+ {
+ Out.Append('"');
+ i += 5;
+ }
+ else if (Remaining.starts_with("&apos;"))
+ {
+ Out.Append('\'');
+ i += 5;
+ }
+ else
+ {
+ Out.Append(Input[i]);
+ }
+ }
+ else
+ {
+ Out.Append(Input[i]);
+ }
+ }
+ }
+
+ /// Convenience: decode XML entities and return as std::string.
+ std::string DecodeXmlEntities(std::string_view Input)
+ {
+ if (Input.find('&') == std::string_view::npos)
+ {
+ return std::string(Input);
+ }
+
+ ExtendableStringBuilder<256> Sb;
+ DecodeXmlEntities(Input, Sb);
+ return Sb.ToString();
+ }
+
+ /// Join a path and canonical query string into a full request path for the HTTP client.
+ std::string BuildRequestPath(std::string_view Path, std::string_view CanonicalQS)
+ {
+ if (CanonicalQS.empty())
+ {
+ return std::string(Path);
+ }
+ return fmt::format("{}?{}", Path, CanonicalQS);
+ }
+
+ /// Case-insensitive header lookup in an HttpClient response header map.
+ const std::string* FindResponseHeader(const HttpClient::KeyValueMap& Headers, std::string_view Name)
+ {
+ for (const auto& [K, V] : *Headers)
+ {
+ if (StrCaseCompare(K, Name) == 0)
+ {
+ return &V;
+ }
+ }
+ return nullptr;
+ }
+
+} // namespace
+
+S3Client::S3Client(const S3ClientOptions& Options)
+: m_Log(logging::Get("s3"))
+, m_BucketName(Options.BucketName)
+, m_Region(Options.Region)
+, m_Endpoint(Options.Endpoint)
+, m_PathStyle(Options.PathStyle)
+, m_Credentials(Options.Credentials)
+, m_CredentialProvider(Options.CredentialProvider)
+, m_HttpClient(BuildEndpoint(),
+ HttpClientSettings{
+ .LogCategory = "s3",
+ .ConnectTimeout = Options.ConnectTimeout,
+ .Timeout = Options.Timeout,
+ .RetryCount = Options.RetryCount,
+ })
+{
+ m_Host = BuildHostHeader();
+ ZEN_INFO("S3 client configured for bucket '{}' in region '{}' (endpoint: {}, {})",
+ m_BucketName,
+ m_Region,
+ m_HttpClient.GetBaseUri(),
+ m_PathStyle ? "path-style" : "virtual-hosted");
+}
+
+S3Client::~S3Client() = default;
+
+SigV4Credentials
+S3Client::GetCurrentCredentials()
+{
+ if (m_CredentialProvider)
+ {
+ SigV4Credentials Creds = m_CredentialProvider->GetCredentials();
+ if (!Creds.AccessKeyId.empty())
+ {
+ // Invalidate the signing key cache when the access key changes
+ if (Creds.AccessKeyId != m_Credentials.AccessKeyId)
+ {
+ RwLock::ExclusiveLockScope ExclusiveLock(m_SigningKeyLock);
+ m_CachedDateStamp.clear();
+ }
+ m_Credentials = Creds;
+ }
+ return m_Credentials;
+ }
+ return m_Credentials;
+}
+
+std::string
+S3Client::BuildEndpoint() const
+{
+ if (!m_Endpoint.empty())
+ {
+ return m_Endpoint;
+ }
+
+ if (m_PathStyle)
+ {
+ // Path-style: https://s3.region.amazonaws.com
+ return fmt::format("https://s3.{}.amazonaws.com", m_Region);
+ }
+
+ // Virtual-hosted style: https://bucket.s3.region.amazonaws.com
+ return fmt::format("https://{}.s3.{}.amazonaws.com", m_BucketName, m_Region);
+}
+
+std::string
+S3Client::BuildHostHeader() const
+{
+ if (!m_Endpoint.empty())
+ {
+ // Extract host from custom endpoint URL (strip scheme)
+ std::string_view Ep = m_Endpoint;
+ if (size_t Pos = Ep.find("://"); Pos != std::string_view::npos)
+ {
+ Ep = Ep.substr(Pos + 3);
+ }
+ // Strip trailing slash
+ if (!Ep.empty() && Ep.back() == '/')
+ {
+ Ep = Ep.substr(0, Ep.size() - 1);
+ }
+ return std::string(Ep);
+ }
+
+ if (m_PathStyle)
+ {
+ return fmt::format("s3.{}.amazonaws.com", m_Region);
+ }
+
+ return fmt::format("{}.s3.{}.amazonaws.com", m_BucketName, m_Region);
+}
+
+std::string
+S3Client::KeyToPath(std::string_view Key) const
+{
+ if (m_PathStyle)
+ {
+ return fmt::format("/{}/{}", m_BucketName, Key);
+ }
+ return fmt::format("/{}", Key);
+}
+
+std::string
+S3Client::BucketRootPath() const
+{
+ if (m_PathStyle)
+ {
+ return fmt::format("/{}/", m_BucketName);
+ }
+ return "/";
+}
+
+Sha256Digest
+S3Client::GetSigningKey(std::string_view DateStamp)
+{
+ // Fast path: shared lock for cache hit (common case — key only changes once per day)
+ {
+ RwLock::SharedLockScope SharedLock(m_SigningKeyLock);
+ if (m_CachedDateStamp == DateStamp)
+ {
+ return m_CachedSigningKey;
+ }
+ }
+
+ // Slow path: exclusive lock to recompute the signing key
+ RwLock::ExclusiveLockScope ExclusiveLock(m_SigningKeyLock);
+
+ // Double-check after acquiring exclusive lock (another thread may have updated it)
+ if (m_CachedDateStamp == DateStamp)
+ {
+ return m_CachedSigningKey;
+ }
+
+ std::string SecretPrefix = fmt::format("AWS4{}", m_Credentials.SecretAccessKey);
+
+ Sha256Digest DateKey = ComputeHmacSha256(SecretPrefix.data(), SecretPrefix.size(), DateStamp.data(), DateStamp.size());
+ SecureZeroSecret(SecretPrefix.data(), SecretPrefix.size());
+
+ Sha256Digest RegionKey = ComputeHmacSha256(DateKey, m_Region);
+ Sha256Digest ServiceKey = ComputeHmacSha256(RegionKey, "s3");
+ m_CachedSigningKey = ComputeHmacSha256(ServiceKey, "aws4_request");
+ m_CachedDateStamp = std::string(DateStamp);
+
+ return m_CachedSigningKey;
+}
+
+HttpClient::KeyValueMap
+S3Client::SignRequest(std::string_view Method, std::string_view Path, std::string_view CanonicalQueryString, std::string_view PayloadHash)
+{
+ SigV4Credentials Credentials = GetCurrentCredentials();
+
+ std::string AmzDate = GetAmzTimestamp();
+
+ // Build sorted headers to sign (must be sorted by lowercase name)
+ std::vector<std::pair<std::string, std::string>> HeadersToSign;
+ HeadersToSign.emplace_back("host", m_Host);
+ HeadersToSign.emplace_back("x-amz-content-sha256", std::string(PayloadHash));
+ HeadersToSign.emplace_back("x-amz-date", AmzDate);
+ if (!Credentials.SessionToken.empty())
+ {
+ HeadersToSign.emplace_back("x-amz-security-token", Credentials.SessionToken);
+ }
+ std::sort(HeadersToSign.begin(), HeadersToSign.end());
+
+ std::string_view DateStamp(AmzDate.data(), 8);
+ Sha256Digest SigningKey = GetSigningKey(DateStamp);
+
+ SigV4SignedHeaders Signed =
+ SignRequestV4(Credentials, Method, Path, CanonicalQueryString, m_Region, "s3", AmzDate, HeadersToSign, PayloadHash, &SigningKey);
+
+ HttpClient::KeyValueMap Result;
+ Result->emplace("Authorization", std::move(Signed.Authorization));
+ Result->emplace("x-amz-date", std::move(Signed.AmzDate));
+ Result->emplace("x-amz-content-sha256", std::move(Signed.PayloadHash));
+ if (!Credentials.SessionToken.empty())
+ {
+ Result->emplace("x-amz-security-token", Credentials.SessionToken);
+ }
+
+ return Result;
+}
+
+S3Result
+S3Client::PutObject(std::string_view Key, IoBuffer Content)
+{
+ std::string Path = KeyToPath(Key);
+
+ // Hash the payload
+ std::string PayloadHash = Sha256ToHex(ComputeSha256(Content.GetData(), Content.GetSize()));
+
+ HttpClient::KeyValueMap Headers = SignRequest("PUT", Path, "", PayloadHash);
+
+ HttpClient::Response Response = m_HttpClient.Put(Path, Content, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = Response.ErrorMessage("S3 PUT failed");
+ ZEN_WARN("S3 PUT '{}' failed: {}", Key, Err);
+ return S3Result{std::move(Err)};
+ }
+
+ ZEN_DEBUG("S3 PUT '{}' succeeded ({} bytes)", Key, Content.GetSize());
+ return {};
+}
+
+S3GetObjectResult
+S3Client::GetObject(std::string_view Key)
+{
+ std::string Path = KeyToPath(Key);
+
+ HttpClient::KeyValueMap Headers = SignRequest("GET", Path, "", EmptyPayloadHash);
+
+ HttpClient::Response Response = m_HttpClient.Get(Path, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = Response.ErrorMessage("S3 GET failed");
+ ZEN_WARN("S3 GET '{}' failed: {}", Key, Err);
+ return S3GetObjectResult{S3Result{std::move(Err)}, {}};
+ }
+
+ ZEN_DEBUG("S3 GET '{}' succeeded ({} bytes)", Key, Response.ResponsePayload.GetSize());
+ return S3GetObjectResult{{}, std::move(Response.ResponsePayload)};
+}
+
+S3Result
+S3Client::DeleteObject(std::string_view Key)
+{
+ std::string Path = KeyToPath(Key);
+
+ HttpClient::KeyValueMap Headers = SignRequest("DELETE", Path, "", EmptyPayloadHash);
+
+ HttpClient::Response Response = m_HttpClient.Delete(Path, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = Response.ErrorMessage("S3 DELETE failed");
+ ZEN_WARN("S3 DELETE '{}' failed: {}", Key, Err);
+ return S3Result{std::move(Err)};
+ }
+
+ ZEN_DEBUG("S3 DELETE '{}' succeeded", Key);
+ return {};
+}
+
+S3HeadObjectResult
+S3Client::HeadObject(std::string_view Key)
+{
+ std::string Path = KeyToPath(Key);
+
+ HttpClient::KeyValueMap Headers = SignRequest("HEAD", Path, "", EmptyPayloadHash);
+
+ HttpClient::Response Response = m_HttpClient.Head(Path, Headers);
+ if (!Response.IsSuccess())
+ {
+ if (Response.StatusCode == HttpResponseCode::NotFound)
+ {
+ return S3HeadObjectResult{{}, {}, HeadObjectResult::NotFound};
+ }
+
+ std::string Err = Response.ErrorMessage("S3 HEAD failed");
+ ZEN_WARN("S3 HEAD '{}' failed: {}", Key, Err);
+ return S3HeadObjectResult{S3Result{std::move(Err)}, {}, HeadObjectResult::Error};
+ }
+
+ S3ObjectInfo Info;
+ Info.Key = std::string(Key);
+
+ if (const std::string* V = FindResponseHeader(Response.Header, "content-length"))
+ {
+ Info.Size = ParseInt<uint64_t>(*V).value_or(0);
+ }
+
+ if (const std::string* V = FindResponseHeader(Response.Header, "etag"))
+ {
+ Info.ETag = *V;
+ }
+
+ if (const std::string* V = FindResponseHeader(Response.Header, "last-modified"))
+ {
+ Info.LastModified = *V;
+ }
+
+ ZEN_DEBUG("S3 HEAD '{}' succeeded (size={})", Key, Info.Size);
+ return S3HeadObjectResult{{}, std::move(Info), HeadObjectResult::Found};
+}
+
+S3ListObjectsResult
+S3Client::ListObjects(std::string_view Prefix, uint32_t MaxKeys)
+{
+ S3ListObjectsResult Result;
+
+ std::string ContinuationToken;
+
+ for (;;)
+ {
+ // Build query parameters for ListObjectsV2
+ std::vector<std::pair<std::string, std::string>> QueryParams;
+ QueryParams.emplace_back("list-type", "2");
+ if (!Prefix.empty())
+ {
+ QueryParams.emplace_back("prefix", std::string(Prefix));
+ }
+ if (MaxKeys > 0)
+ {
+ QueryParams.emplace_back("max-keys", fmt::format("{}", MaxKeys));
+ }
+ if (!ContinuationToken.empty())
+ {
+ QueryParams.emplace_back("continuation-token", ContinuationToken);
+ }
+
+ std::string CanonicalQS = BuildCanonicalQueryString(std::move(QueryParams));
+ std::string RootPath = BucketRootPath();
+ HttpClient::KeyValueMap Headers = SignRequest("GET", RootPath, CanonicalQS, EmptyPayloadHash);
+
+ std::string FullPath = BuildRequestPath(RootPath, CanonicalQS);
+ HttpClient::Response Response = m_HttpClient.Get(FullPath, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = Response.ErrorMessage("S3 ListObjectsV2 failed");
+ ZEN_WARN("S3 ListObjectsV2 prefix='{}' failed: {}", Prefix, Err);
+ Result.Error = std::move(Err);
+ return Result;
+ }
+
+ // Parse the XML response to extract object keys
+ std::string_view ResponseBody = Response.AsText();
+
+ // Find all <Contents> elements
+ std::string_view Remaining = ResponseBody;
+ while (true)
+ {
+ size_t ContentsStart = Remaining.find("<Contents>");
+ if (ContentsStart == std::string_view::npos)
+ {
+ break;
+ }
+
+ size_t ContentsEnd = Remaining.find("</Contents>", ContentsStart);
+ if (ContentsEnd == std::string_view::npos)
+ {
+ break;
+ }
+
+ std::string_view ContentsXml = Remaining.substr(ContentsStart, ContentsEnd - ContentsStart + 11);
+
+ S3ObjectInfo Info;
+ Info.Key = DecodeXmlEntities(ExtractXmlValue(ContentsXml, "Key"));
+ Info.ETag = DecodeXmlEntities(ExtractXmlValue(ContentsXml, "ETag"));
+ Info.LastModified = std::string(ExtractXmlValue(ContentsXml, "LastModified"));
+
+ std::string_view SizeStr = ExtractXmlValue(ContentsXml, "Size");
+ if (!SizeStr.empty())
+ {
+ Info.Size = ParseInt<uint64_t>(SizeStr).value_or(0);
+ }
+
+ if (!Info.Key.empty())
+ {
+ Result.Objects.push_back(std::move(Info));
+ }
+
+ Remaining = Remaining.substr(ContentsEnd + 11);
+ }
+
+ // Check if there are more pages
+ std::string_view IsTruncated = ExtractXmlValue(ResponseBody, "IsTruncated");
+ if (IsTruncated != "true")
+ {
+ break;
+ }
+
+ std::string_view NextToken = ExtractXmlValue(ResponseBody, "NextContinuationToken");
+ if (NextToken.empty())
+ {
+ break;
+ }
+
+ ContinuationToken = std::string(NextToken);
+ ZEN_DEBUG("S3 ListObjectsV2 prefix='{}' fetching next page ({} objects so far)", Prefix, Result.Objects.size());
+ }
+
+ ZEN_DEBUG("S3 ListObjectsV2 prefix='{}' returned {} objects", Prefix, Result.Objects.size());
+ return Result;
+}
+
+//////////////////////////////////////////////////////////////////////////
+// Multipart Upload
+
+S3CreateMultipartUploadResult
+S3Client::CreateMultipartUpload(std::string_view Key)
+{
+ std::string Path = KeyToPath(Key);
+ std::string CanonicalQS = BuildCanonicalQueryString({{"uploads", ""}});
+
+ HttpClient::KeyValueMap Headers = SignRequest("POST", Path, CanonicalQS, EmptyPayloadHash);
+
+ std::string FullPath = BuildRequestPath(Path, CanonicalQS);
+ HttpClient::Response Response = m_HttpClient.Post(FullPath, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = Response.ErrorMessage("S3 CreateMultipartUpload failed");
+ ZEN_WARN("S3 CreateMultipartUpload '{}' failed: {}", Key, Err);
+ return S3CreateMultipartUploadResult{S3Result{std::move(Err)}, {}};
+ }
+
+ // Parse UploadId from XML response:
+ // <InitiateMultipartUploadResult>
+ // <Bucket>...</Bucket>
+ // <Key>...</Key>
+ // <UploadId>...</UploadId>
+ // </InitiateMultipartUploadResult>
+ std::string_view ResponseBody = Response.AsText();
+ std::string_view UploadId = ExtractXmlValue(ResponseBody, "UploadId");
+ if (UploadId.empty())
+ {
+ std::string Err = "failed to parse UploadId from CreateMultipartUpload response";
+ ZEN_WARN("S3 CreateMultipartUpload '{}': {}", Key, Err);
+ return S3CreateMultipartUploadResult{S3Result{std::move(Err)}, {}};
+ }
+
+ ZEN_DEBUG("S3 CreateMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ return S3CreateMultipartUploadResult{{}, std::string(UploadId)};
+}
+
+S3UploadPartResult
+S3Client::UploadPart(std::string_view Key, std::string_view UploadId, uint32_t PartNumber, IoBuffer Content)
+{
+ std::string Path = KeyToPath(Key);
+ std::string CanonicalQS = BuildCanonicalQueryString({
+ {"partNumber", fmt::format("{}", PartNumber)},
+ {"uploadId", std::string(UploadId)},
+ });
+
+ std::string PayloadHash = Sha256ToHex(ComputeSha256(Content.GetData(), Content.GetSize()));
+
+ HttpClient::KeyValueMap Headers = SignRequest("PUT", Path, CanonicalQS, PayloadHash);
+
+ std::string FullPath = BuildRequestPath(Path, CanonicalQS);
+ HttpClient::Response Response = m_HttpClient.Put(FullPath, Content, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = Response.ErrorMessage(fmt::format("S3 UploadPart {} failed", PartNumber));
+ ZEN_WARN("S3 UploadPart '{}' part {} failed: {}", Key, PartNumber, Err);
+ return S3UploadPartResult{S3Result{std::move(Err)}, {}};
+ }
+
+ // Extract ETag from response headers
+ const std::string* ETag = FindResponseHeader(Response.Header, "etag");
+ if (!ETag)
+ {
+ std::string Err = "S3 UploadPart response missing ETag header";
+ ZEN_WARN("S3 UploadPart '{}' part {}: {}", Key, PartNumber, Err);
+ return S3UploadPartResult{S3Result{std::move(Err)}, {}};
+ }
+
+ ZEN_DEBUG("S3 UploadPart '{}' part {} succeeded ({} bytes, etag={})", Key, PartNumber, Content.GetSize(), *ETag);
+ return S3UploadPartResult{{}, *ETag};
+}
+
+S3Result
+S3Client::CompleteMultipartUpload(std::string_view Key,
+ std::string_view UploadId,
+ const std::vector<std::pair<uint32_t, std::string>>& PartETags)
+{
+ std::string Path = KeyToPath(Key);
+ std::string CanonicalQS = BuildCanonicalQueryString({{"uploadId", std::string(UploadId)}});
+
+ // Build the CompleteMultipartUpload XML payload
+ ExtendableStringBuilder<1024> XmlBody;
+ XmlBody.Append("<CompleteMultipartUpload>");
+ for (const auto& [PartNumber, ETag] : PartETags)
+ {
+ XmlBody.Append(fmt::format("<Part><PartNumber>{}</PartNumber><ETag>{}</ETag></Part>", PartNumber, ETag));
+ }
+ XmlBody.Append("</CompleteMultipartUpload>");
+
+ std::string_view XmlView = XmlBody.ToView();
+ std::string PayloadHash = Sha256ToHex(ComputeSha256(XmlView));
+
+ HttpClient::KeyValueMap Headers = SignRequest("POST", Path, CanonicalQS, PayloadHash);
+ Headers->emplace("Content-Type", "application/xml");
+
+ IoBuffer Payload(IoBuffer::Clone, XmlView.data(), XmlView.size());
+
+ std::string FullPath = BuildRequestPath(Path, CanonicalQS);
+ HttpClient::Response Response = m_HttpClient.Post(FullPath, Payload, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = Response.ErrorMessage("S3 CompleteMultipartUpload failed");
+ ZEN_WARN("S3 CompleteMultipartUpload '{}' failed: {}", Key, Err);
+ return S3Result{std::move(Err)};
+ }
+
+ // Check for error in response body - S3 can return 200 with an error in the XML body
+ std::string_view ResponseBody = Response.AsText();
+ if (ResponseBody.find("<Error>") != std::string_view::npos)
+ {
+ std::string_view ErrorCode = ExtractXmlValue(ResponseBody, "Code");
+ std::string_view ErrorMessage = ExtractXmlValue(ResponseBody, "Message");
+ std::string Err = fmt::format("S3 CompleteMultipartUpload '{}' returned error: {} - {}", Key, ErrorCode, ErrorMessage);
+ ZEN_WARN("{}", Err);
+ return S3Result{std::move(Err)};
+ }
+
+ ZEN_DEBUG("S3 CompleteMultipartUpload '{}' succeeded ({} parts)", Key, PartETags.size());
+ return {};
+}
+
+S3Result
+S3Client::AbortMultipartUpload(std::string_view Key, std::string_view UploadId)
+{
+ std::string Path = KeyToPath(Key);
+ std::string CanonicalQS = BuildCanonicalQueryString({{"uploadId", std::string(UploadId)}});
+
+ HttpClient::KeyValueMap Headers = SignRequest("DELETE", Path, CanonicalQS, EmptyPayloadHash);
+
+ std::string FullPath = BuildRequestPath(Path, CanonicalQS);
+ HttpClient::Response Response = m_HttpClient.Delete(FullPath, Headers);
+ if (!Response.IsSuccess())
+ {
+ std::string Err = Response.ErrorMessage("S3 AbortMultipartUpload failed");
+ ZEN_WARN("S3 AbortMultipartUpload '{}' failed: {}", Key, Err);
+ return S3Result{std::move(Err)};
+ }
+
+ ZEN_DEBUG("S3 AbortMultipartUpload '{}' succeeded (uploadId={})", Key, UploadId);
+ return {};
+}
+
+std::string
+S3Client::GeneratePresignedGetUrl(std::string_view Key, std::chrono::seconds ExpiresIn)
+{
+ return GeneratePresignedUrlForMethod(Key, "GET", ExpiresIn);
+}
+
+std::string
+S3Client::GeneratePresignedPutUrl(std::string_view Key, std::chrono::seconds ExpiresIn)
+{
+ return GeneratePresignedUrlForMethod(Key, "PUT", ExpiresIn);
+}
+
+std::string
+S3Client::GeneratePresignedUrlForMethod(std::string_view Key, std::string_view Method, std::chrono::seconds ExpiresIn)
+{
+ std::string Path = KeyToPath(Key);
+ std::string Scheme = "https";
+
+ if (!m_Endpoint.empty() && m_Endpoint.starts_with("http://"))
+ {
+ Scheme = "http";
+ }
+
+ SigV4Credentials Credentials = GetCurrentCredentials();
+ return GeneratePresignedUrl(Credentials, Method, Scheme, m_Host, Path, m_Region, "s3", ExpiresIn);
+}
+
+S3Result
+S3Client::PutObjectMultipart(std::string_view Key, IoBuffer Content, uint64_t PartSize)
+{
+ const uint64_t ContentSize = Content.GetSize();
+
+ // If the content fits in a single part, just use PutObject
+ if (ContentSize <= PartSize)
+ {
+ return PutObject(Key, Content);
+ }
+
+ ZEN_INFO("S3 multipart upload '{}': {} bytes in ~{} parts", Key, ContentSize, (ContentSize + PartSize - 1) / PartSize);
+
+ // Initiate multipart upload
+
+ S3CreateMultipartUploadResult InitResult = CreateMultipartUpload(Key);
+ if (!InitResult)
+ {
+ return S3Result{std::move(InitResult.Error)};
+ }
+
+ const std::string& UploadId = InitResult.UploadId;
+
+ // Upload parts sequentially
+ // TODO: upload parts in parallel for improved throughput on large uploads
+
+ std::vector<std::pair<uint32_t, std::string>> PartETags;
+ uint64_t Offset = 0;
+ uint32_t PartNumber = 1;
+
+ while (Offset < ContentSize)
+ {
+ uint64_t ThisPartSize = std::min(PartSize, ContentSize - Offset);
+
+ // Create a sub-buffer referencing the part data within the original content
+ IoBuffer PartContent(Content, Offset, ThisPartSize);
+
+ S3UploadPartResult PartResult = UploadPart(Key, UploadId, PartNumber, PartContent);
+ if (!PartResult)
+ {
+ // Attempt to abort the multipart upload on failure
+ AbortMultipartUpload(Key, UploadId);
+ return S3Result{std::move(PartResult.Error)};
+ }
+
+ PartETags.emplace_back(PartNumber, std::move(PartResult.ETag));
+ Offset += ThisPartSize;
+ PartNumber++;
+ }
+
+ // Complete multipart upload
+ S3Result CompleteResult = CompleteMultipartUpload(Key, UploadId, PartETags);
+ if (!CompleteResult)
+ {
+ AbortMultipartUpload(Key, UploadId);
+ return CompleteResult;
+ }
+
+ ZEN_INFO("S3 multipart upload '{}' completed ({} parts, {} bytes)", Key, PartETags.size(), ContentSize);
+ return {};
+}
+
+//////////////////////////////////////////////////////////////////////////
+// Tests
+
+#if ZEN_WITH_TESTS
+
+void
+s3client_forcelink()
+{
+}
+
+TEST_SUITE_BEGIN("util.cloud.s3client");
+
+TEST_CASE("s3client.xml_extract")
+{
+ std::string_view Xml =
+ "<Contents><Key>test/file.txt</Key><Size>1234</Size>"
+ "<ETag>\"abc123\"</ETag><LastModified>2024-01-01T00:00:00Z</LastModified></Contents>";
+
+ CHECK(ExtractXmlValue(Xml, "Key") == "test/file.txt");
+ CHECK(ExtractXmlValue(Xml, "Size") == "1234");
+ CHECK(ExtractXmlValue(Xml, "ETag") == "\"abc123\"");
+ CHECK(ExtractXmlValue(Xml, "LastModified") == "2024-01-01T00:00:00Z");
+ CHECK(ExtractXmlValue(Xml, "NonExistent") == "");
+}
+
+TEST_CASE("s3client.xml_entity_decode")
+{
+ CHECK(DecodeXmlEntities("no entities") == "no entities");
+ CHECK(DecodeXmlEntities("a&amp;b") == "a&b");
+ CHECK(DecodeXmlEntities("&lt;tag&gt;") == "<tag>");
+ CHECK(DecodeXmlEntities("&quot;hello&apos;") == "\"hello'");
+ CHECK(DecodeXmlEntities("&amp;&amp;") == "&&");
+ CHECK(DecodeXmlEntities("") == "");
+
+ // Key with entities as S3 would return it
+ std::string_view Xml = "<Key>path/file&amp;name&lt;1&gt;.txt</Key>";
+ CHECK(DecodeXmlEntities(ExtractXmlValue(Xml, "Key")) == "path/file&name<1>.txt");
+}
+
+TEST_CASE("s3client.path_style_addressing")
+{
+ // Verify path-style builds /{bucket}/{key} paths
+ S3ClientOptions Opts;
+ Opts.BucketName = "test-bucket";
+ Opts.Region = "us-east-1";
+ Opts.Endpoint = "http://localhost:9000";
+ Opts.PathStyle = true;
+ Opts.Credentials.AccessKeyId = "minioadmin";
+ Opts.Credentials.SecretAccessKey = "minioadmin";
+
+ S3Client Client(Opts);
+ CHECK(Client.BucketName() == "test-bucket");
+ CHECK(Client.Region() == "us-east-1");
+}
+
+TEST_CASE("s3client.virtual_hosted_addressing")
+{
+ // Verify virtual-hosted style derives endpoint from region + bucket
+ S3ClientOptions Opts;
+ Opts.BucketName = "my-bucket";
+ Opts.Region = "eu-west-1";
+ Opts.PathStyle = false;
+ Opts.Credentials.AccessKeyId = "key";
+ Opts.Credentials.SecretAccessKey = "secret";
+
+ S3Client Client(Opts);
+ CHECK(Client.BucketName() == "my-bucket");
+ CHECK(Client.Region() == "eu-west-1");
+}
+
+TEST_CASE("s3client.minio_integration")
+{
+ using namespace std::literals;
+
+ // Spawn a local MinIO server
+ MinioProcessOptions MinioOpts;
+ MinioOpts.Port = 19000;
+ MinioOpts.RootUser = "testuser";
+ MinioOpts.RootPassword = "testpassword";
+
+ MinioProcess Minio(MinioOpts);
+ Minio.SpawnMinioServer();
+
+ // Pre-create the test bucket (creates a subdirectory in MinIO's data dir)
+ Minio.CreateBucket("integration-test");
+
+ // Configure S3Client for the test bucket
+ S3ClientOptions Opts;
+ Opts.BucketName = "integration-test";
+ Opts.Region = "us-east-1";
+ Opts.Endpoint = Minio.Endpoint();
+ Opts.PathStyle = true;
+ Opts.Credentials.AccessKeyId = std::string(Minio.RootUser());
+ Opts.Credentials.SecretAccessKey = std::string(Minio.RootPassword());
+
+ S3Client Client(Opts);
+
+ SUBCASE("put_get_delete")
+ {
+ // PUT
+ std::string_view TestData = "hello, minio integration test!"sv;
+ IoBuffer Content = IoBufferBuilder::MakeFromMemory(MakeMemoryView(TestData));
+ S3Result PutRes = Client.PutObject("test/hello.txt", std::move(Content));
+ REQUIRE(PutRes.IsSuccess());
+
+ // GET
+ S3GetObjectResult GetRes = Client.GetObject("test/hello.txt");
+ REQUIRE(GetRes.IsSuccess());
+ CHECK(GetRes.AsText() == TestData);
+
+ // HEAD
+ S3HeadObjectResult HeadRes = Client.HeadObject("test/hello.txt");
+ REQUIRE(HeadRes.IsSuccess());
+ CHECK(HeadRes.Status == HeadObjectResult::Found);
+ CHECK(HeadRes.Info.Size == TestData.size());
+
+ // DELETE
+ S3Result DelRes = Client.DeleteObject("test/hello.txt");
+ REQUIRE(DelRes.IsSuccess());
+
+ // HEAD after delete
+ S3HeadObjectResult HeadRes2 = Client.HeadObject("test/hello.txt");
+ REQUIRE(HeadRes2.IsSuccess());
+ CHECK(HeadRes2.Status == HeadObjectResult::NotFound);
+ }
+
+ SUBCASE("head_not_found")
+ {
+ S3HeadObjectResult Res = Client.HeadObject("nonexistent/key.dat");
+ CHECK(Res.IsSuccess());
+ CHECK(Res.Status == HeadObjectResult::NotFound);
+ }
+
+ SUBCASE("list_objects")
+ {
+ // Upload several objects with a common prefix
+ for (int i = 0; i < 3; ++i)
+ {
+ std::string Key = fmt::format("list-test/item-{}.txt", i);
+ std::string Payload = fmt::format("payload-{}", i);
+ IoBuffer Buf = IoBufferBuilder::MakeFromMemory(MakeMemoryView(Payload));
+ S3Result Res = Client.PutObject(Key, std::move(Buf));
+ REQUIRE(Res.IsSuccess());
+ }
+
+ // List with prefix
+ S3ListObjectsResult ListRes = Client.ListObjects("list-test/");
+ REQUIRE(ListRes.IsSuccess());
+ CHECK(ListRes.Objects.size() == 3);
+
+ // Verify keys are present
+ std::vector<std::string> Keys;
+ for (const S3ObjectInfo& Obj : ListRes.Objects)
+ {
+ Keys.push_back(Obj.Key);
+ }
+ std::sort(Keys.begin(), Keys.end());
+ CHECK(Keys[0] == "list-test/item-0.txt");
+ CHECK(Keys[1] == "list-test/item-1.txt");
+ CHECK(Keys[2] == "list-test/item-2.txt");
+
+ // Cleanup
+ for (int i = 0; i < 3; ++i)
+ {
+ Client.DeleteObject(fmt::format("list-test/item-{}.txt", i));
+ }
+ }
+
+ SUBCASE("multipart_upload")
+ {
+ // Create a payload large enough to exercise multipart (use minimum part size)
+ constexpr uint64_t PartSize = 5 * 1024 * 1024; // 5 MB minimum
+ constexpr uint64_t PayloadSize = PartSize + 1024; // slightly over one part
+
+ std::string LargePayload(PayloadSize, 'X');
+ // Add some variation
+ for (uint64_t i = 0; i < PayloadSize; i += 1024)
+ {
+ LargePayload[i] = char('A' + (i / 1024) % 26);
+ }
+
+ IoBuffer Content = IoBufferBuilder::MakeFromMemory(MakeMemoryView(LargePayload));
+ S3Result Res = Client.PutObjectMultipart("multipart/large.bin", std::move(Content), PartSize);
+ REQUIRE(Res.IsSuccess());
+
+ // Verify via GET
+ S3GetObjectResult GetRes = Client.GetObject("multipart/large.bin");
+ REQUIRE(GetRes.IsSuccess());
+ CHECK(GetRes.Content.GetSize() == PayloadSize);
+ CHECK(GetRes.AsText() == std::string_view(LargePayload));
+
+ // Cleanup
+ Client.DeleteObject("multipart/large.bin");
+ }
+
+ SUBCASE("presigned_urls")
+ {
+ // Upload an object
+ std::string_view TestData = "presigned-url-test-data"sv;
+ IoBuffer Content = IoBufferBuilder::MakeFromMemory(MakeMemoryView(TestData));
+ S3Result PutRes = Client.PutObject("presigned/test.txt", std::move(Content));
+ REQUIRE(PutRes.IsSuccess());
+
+ // Generate a pre-signed GET URL
+ std::string Url = Client.GeneratePresignedGetUrl("presigned/test.txt", std::chrono::seconds(60));
+ CHECK(!Url.empty());
+ CHECK(Url.find("X-Amz-Signature") != std::string::npos);
+
+ // Fetch via the pre-signed URL (no auth headers needed)
+ HttpClient Hc(Minio.Endpoint());
+ // Extract the path+query from the full URL
+ std::string_view UrlView = Url;
+ size_t PathStart = UrlView.find('/', UrlView.find("://") + 3);
+ std::string PathAndQuery(UrlView.substr(PathStart));
+ HttpClient::Response Resp = Hc.Get(PathAndQuery);
+ REQUIRE(Resp.IsSuccess());
+ CHECK(Resp.AsText() == TestData);
+
+ // Cleanup
+ Client.DeleteObject("presigned/test.txt");
+ }
+
+ Minio.StopMinioServer();
+}
+
+TEST_SUITE_END();
+
+#endif
+
+} // namespace zen
diff --git a/src/zenutil/cloud/sigv4.cpp b/src/zenutil/cloud/sigv4.cpp
new file mode 100644
index 000000000..055ccb2ad
--- /dev/null
+++ b/src/zenutil/cloud/sigv4.cpp
@@ -0,0 +1,531 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/cloud/sigv4.h>
+
+#include <zencore/string.h>
+#include <zencore/testing.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cstring>
+#include <ctime>
+
+// Platform-specific crypto backends
+#if ZEN_PLATFORM_WINDOWS
+# define ZEN_S3_USE_BCRYPT 1
+#else
+# define ZEN_S3_USE_BCRYPT 0
+#endif
+
+#ifndef ZEN_S3_USE_OPENSSL
+# if ZEN_S3_USE_BCRYPT
+# define ZEN_S3_USE_OPENSSL 0
+# else
+# define ZEN_S3_USE_OPENSSL 1
+# endif
+#endif
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+
+#if ZEN_S3_USE_OPENSSL
+# include <openssl/evp.h>
+#elif ZEN_S3_USE_BCRYPT
+# include <zencore/windows.h>
+# include <bcrypt.h>
+#endif
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+// SHA-256
+
+#if ZEN_S3_USE_OPENSSL
+
+Sha256Digest
+ComputeSha256(const void* Data, size_t Size)
+{
+ Sha256Digest Result;
+ unsigned int Len = 0;
+ EVP_Digest(Data, Size, Result.data(), &Len, EVP_sha256(), nullptr);
+ ZEN_ASSERT(Len == 32);
+ return Result;
+}
+
+Sha256Digest
+ComputeHmacSha256(const void* Key, size_t KeySize, const void* Data, size_t DataSize)
+{
+ Sha256Digest Result;
+
+ EVP_MAC* Mac = EVP_MAC_fetch(nullptr, "HMAC", nullptr);
+ ZEN_ASSERT(Mac != nullptr);
+
+ EVP_MAC_CTX* Ctx = EVP_MAC_CTX_new(Mac);
+ ZEN_ASSERT(Ctx != nullptr);
+
+ OSSL_PARAM Params[] = {
+ OSSL_PARAM_construct_utf8_string("digest", const_cast<char*>("SHA256"), 0),
+ OSSL_PARAM_construct_end(),
+ };
+
+ int Rc = EVP_MAC_init(Ctx, reinterpret_cast<const unsigned char*>(Key), KeySize, Params);
+ ZEN_ASSERT(Rc == 1);
+
+ Rc = EVP_MAC_update(Ctx, reinterpret_cast<const unsigned char*>(Data), DataSize);
+ ZEN_ASSERT(Rc == 1);
+
+ size_t OutLen = 0;
+ Rc = EVP_MAC_final(Ctx, Result.data(), &OutLen, Result.size());
+ ZEN_ASSERT(Rc == 1);
+ ZEN_ASSERT(OutLen == 32);
+
+ EVP_MAC_CTX_free(Ctx);
+ EVP_MAC_free(Mac);
+
+ return Result;
+}
+
+#elif ZEN_S3_USE_BCRYPT
+
+namespace {
+
+# define NT_SUCCESS(Status) (((NTSTATUS)(Status)) >= 0)
+
+ Sha256Digest BcryptHash(BCRYPT_ALG_HANDLE Algorithm, const void* Data, size_t DataSize)
+ {
+ Sha256Digest Result;
+ BCRYPT_HASH_HANDLE HashHandle = nullptr;
+ NTSTATUS Status;
+
+ Status = BCryptCreateHash(Algorithm, &HashHandle, nullptr, 0, nullptr, 0, 0);
+ ZEN_ASSERT(NT_SUCCESS(Status));
+
+ Status = BCryptHashData(HashHandle, (PUCHAR)Data, (ULONG)DataSize, 0);
+ ZEN_ASSERT(NT_SUCCESS(Status));
+
+ Status = BCryptFinishHash(HashHandle, Result.data(), (ULONG)Result.size(), 0);
+ ZEN_ASSERT(NT_SUCCESS(Status));
+
+ BCryptDestroyHash(HashHandle);
+ return Result;
+ }
+
+ Sha256Digest BcryptHmac(BCRYPT_ALG_HANDLE Algorithm, const void* Key, size_t KeySize, const void* Data, size_t DataSize)
+ {
+ Sha256Digest Result;
+ BCRYPT_HASH_HANDLE HashHandle = nullptr;
+ NTSTATUS Status;
+
+ Status = BCryptCreateHash(Algorithm, &HashHandle, nullptr, 0, (PUCHAR)Key, (ULONG)KeySize, 0);
+ ZEN_ASSERT(NT_SUCCESS(Status));
+
+ Status = BCryptHashData(HashHandle, (PUCHAR)Data, (ULONG)DataSize, 0);
+ ZEN_ASSERT(NT_SUCCESS(Status));
+
+ Status = BCryptFinishHash(HashHandle, Result.data(), (ULONG)Result.size(), 0);
+ ZEN_ASSERT(NT_SUCCESS(Status));
+
+ BCryptDestroyHash(HashHandle);
+ return Result;
+ }
+
+ struct BcryptAlgorithmHandles
+ {
+ BCRYPT_ALG_HANDLE Sha256 = nullptr;
+ BCRYPT_ALG_HANDLE HmacSha256 = nullptr;
+
+ BcryptAlgorithmHandles()
+ {
+ NTSTATUS Status;
+ Status = BCryptOpenAlgorithmProvider(&Sha256, BCRYPT_SHA256_ALGORITHM, nullptr, 0);
+ ZEN_ASSERT(NT_SUCCESS(Status));
+ Status = BCryptOpenAlgorithmProvider(&HmacSha256, BCRYPT_SHA256_ALGORITHM, nullptr, BCRYPT_ALG_HANDLE_HMAC_FLAG);
+ ZEN_ASSERT(NT_SUCCESS(Status));
+ }
+
+ ~BcryptAlgorithmHandles()
+ {
+ if (Sha256)
+ {
+ BCryptCloseAlgorithmProvider(Sha256, 0);
+ }
+ if (HmacSha256)
+ {
+ BCryptCloseAlgorithmProvider(HmacSha256, 0);
+ }
+ }
+ };
+
+ BcryptAlgorithmHandles& GetBcryptHandles()
+ {
+ static BcryptAlgorithmHandles s_Handles;
+ return s_Handles;
+ }
+
+} // namespace
+
+Sha256Digest
+ComputeSha256(const void* Data, size_t Size)
+{
+ return BcryptHash(GetBcryptHandles().Sha256, Data, Size);
+}
+
+Sha256Digest
+ComputeHmacSha256(const void* Key, size_t KeySize, const void* Data, size_t DataSize)
+{
+ return BcryptHmac(GetBcryptHandles().HmacSha256, Key, KeySize, Data, DataSize);
+}
+
+#endif
+
+Sha256Digest
+ComputeSha256(std::string_view Data)
+{
+ return ComputeSha256(Data.data(), Data.size());
+}
+
+Sha256Digest
+ComputeHmacSha256(const Sha256Digest& Key, std::string_view Data)
+{
+ return ComputeHmacSha256(Key.data(), Key.size(), Data.data(), Data.size());
+}
+
+std::string
+Sha256ToHex(const Sha256Digest& Digest)
+{
+ std::string Result;
+ Result.reserve(64);
+ for (uint8_t Byte : Digest)
+ {
+ fmt::format_to(std::back_inserter(Result), "{:02x}", Byte);
+ }
+ return Result;
+}
+
+void
+SecureZeroSecret(void* Data, size_t Size)
+{
+#if ZEN_PLATFORM_WINDOWS
+ SecureZeroMemory(Data, Size);
+#elif ZEN_PLATFORM_LINUX
+ explicit_bzero(Data, Size);
+#else
+ // Portable fallback: volatile pointer prevents the compiler from optimizing away the memset
+ static void* (*const volatile VolatileMemset)(void*, int, size_t) = memset;
+ VolatileMemset(Data, 0, Size);
+#endif
+}
+
+//////////////////////////////////////////////////////////////////////////
+// SigV4 signing
+
+namespace {
+
+ std::string GetDateStamp(std::string_view AmzDate)
+ {
+ // AmzDate is "YYYYMMDDTHHMMSSZ", date stamp is first 8 chars
+ return std::string(AmzDate.substr(0, 8));
+ }
+
+} // namespace
+
+std::string
+GetAmzTimestamp()
+{
+ auto Now = std::chrono::system_clock::now();
+ std::time_t NowTime = std::chrono::system_clock::to_time_t(Now);
+
+ struct tm Tm;
+#if ZEN_PLATFORM_WINDOWS
+ gmtime_s(&Tm, &NowTime);
+#else
+ gmtime_r(&NowTime, &Tm);
+#endif
+
+ char Buf[32];
+ std::strftime(Buf, sizeof(Buf), "%Y%m%dT%H%M%SZ", &Tm);
+ return std::string(Buf);
+}
+
+std::string
+AwsUriEncode(std::string_view Input, bool EncodeSlash)
+{
+ ExtendableStringBuilder<256> Result;
+ for (char C : Input)
+ {
+ if ((C >= 'A' && C <= 'Z') || (C >= 'a' && C <= 'z') || (C >= '0' && C <= '9') || C == '_' || C == '-' || C == '~' || C == '.')
+ {
+ Result.Append(C);
+ }
+ else if (C == '/' && !EncodeSlash)
+ {
+ Result.Append(C);
+ }
+ else
+ {
+ Result.Append(fmt::format("%{:02X}", static_cast<unsigned char>(C)));
+ }
+ }
+ return std::string(Result.ToView());
+}
+
+std::string
+BuildCanonicalQueryString(std::vector<std::pair<std::string, std::string>> Parameters)
+{
+ if (Parameters.empty())
+ {
+ return {};
+ }
+
+ // Sort by key name, then by value (as required by SigV4)
+ std::sort(Parameters.begin(), Parameters.end());
+
+ ExtendableStringBuilder<512> Result;
+ for (size_t i = 0; i < Parameters.size(); ++i)
+ {
+ if (i > 0)
+ {
+ Result.Append('&');
+ }
+ Result.Append(AwsUriEncode(Parameters[i].first));
+ Result.Append('=');
+ Result.Append(AwsUriEncode(Parameters[i].second));
+ }
+ return std::string(Result.ToView());
+}
+
+SigV4SignedHeaders
+SignRequestV4(const SigV4Credentials& Credentials,
+ std::string_view Method,
+ std::string_view Url,
+ std::string_view CanonicalQueryString,
+ std::string_view Region,
+ std::string_view Service,
+ std::string_view AmzDate,
+ const std::vector<std::pair<std::string, std::string>>& Headers,
+ std::string_view PayloadHash,
+ const Sha256Digest* SigningKeyPtr)
+{
+ SigV4SignedHeaders Result;
+ Result.AmzDate = std::string(AmzDate);
+ Result.PayloadHash = std::string(PayloadHash);
+
+ std::string DateStamp = GetDateStamp(Result.AmzDate);
+
+ // Step 1: Create canonical request
+ // CanonicalRequest =
+ // HTTPRequestMethod + '\n' +
+ // CanonicalURI + '\n' +
+ // CanonicalQueryString + '\n' +
+ // CanonicalHeaders + '\n' +
+ // SignedHeaders + '\n' +
+ // HexEncode(Hash(RequestPayload))
+
+ std::string CanonicalUri = AwsUriEncode(Url, false);
+
+ // Build canonical headers and signed headers (headers must be sorted by lowercase name)
+ ExtendableStringBuilder<512> CanonicalHeadersSb;
+ ExtendableStringBuilder<256> SignedHeadersSb;
+
+ for (size_t i = 0; i < Headers.size(); ++i)
+ {
+ CanonicalHeadersSb.Append(Headers[i].first);
+ CanonicalHeadersSb.Append(':');
+ CanonicalHeadersSb.Append(Headers[i].second);
+ CanonicalHeadersSb.Append('\n');
+
+ if (i > 0)
+ {
+ SignedHeadersSb.Append(';');
+ }
+ SignedHeadersSb.Append(Headers[i].first);
+ }
+
+ std::string SignedHeaders = std::string(SignedHeadersSb.ToView());
+
+ std::string CanonicalRequest = fmt::format("{}\n{}\n{}\n{}\n{}\n{}",
+ Method,
+ CanonicalUri,
+ CanonicalQueryString,
+ CanonicalHeadersSb.ToView(),
+ SignedHeaders,
+ PayloadHash);
+
+ // Step 2: Create the string to sign
+ std::string CredentialScope = fmt::format("{}/{}/{}/aws4_request", DateStamp, Region, Service);
+
+ Sha256Digest CanonicalRequestHash = ComputeSha256(CanonicalRequest);
+ std::string CanonicalRequestHex = Sha256ToHex(CanonicalRequestHash);
+
+ std::string StringToSign = fmt::format("AWS4-HMAC-SHA256\n{}\n{}\n{}", Result.AmzDate, CredentialScope, CanonicalRequestHex);
+
+ // Step 3: Calculate the signing key
+ // kDate = HMAC("AWS4" + SecretKey, DateStamp)
+ // kRegion = HMAC(kDate, Region)
+ // kService = HMAC(kRegion, Service)
+ // kSigning = HMAC(kService, "aws4_request")
+
+ Sha256Digest DerivedSigningKey;
+ if (!SigningKeyPtr)
+ {
+ std::string SecretPrefix = fmt::format("AWS4{}", Credentials.SecretAccessKey);
+
+ Sha256Digest DateKey = ComputeHmacSha256(SecretPrefix.data(), SecretPrefix.size(), DateStamp.data(), DateStamp.size());
+ SecureZeroSecret(SecretPrefix.data(), SecretPrefix.size());
+
+ Sha256Digest RegionKey = ComputeHmacSha256(DateKey, Region);
+ Sha256Digest ServiceKey = ComputeHmacSha256(RegionKey, Service);
+ DerivedSigningKey = ComputeHmacSha256(ServiceKey, "aws4_request");
+ SigningKeyPtr = &DerivedSigningKey;
+ }
+
+ // Step 4: Calculate the signature
+ Sha256Digest Signature = ComputeHmacSha256(*SigningKeyPtr, StringToSign);
+ std::string SignatureHex = Sha256ToHex(Signature);
+
+ // Step 5: Build the Authorization header
+ Result.Authorization = fmt::format("AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}",
+ Credentials.AccessKeyId,
+ CredentialScope,
+ SignedHeaders,
+ SignatureHex);
+
+ return Result;
+}
+
+std::string
+GeneratePresignedUrl(const SigV4Credentials& Credentials,
+ std::string_view Method,
+ std::string_view Scheme,
+ std::string_view Host,
+ std::string_view Path,
+ std::string_view Region,
+ std::string_view Service,
+ std::chrono::seconds ExpiresIn,
+ const std::vector<std::pair<std::string, std::string>>& ExtraQueryParams)
+{
+ // Pre-signed URLs use query string authentication:
+ // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+
+ std::string AmzDate = GetAmzTimestamp();
+ std::string DateStamp = GetDateStamp(AmzDate);
+
+ std::string CredentialScope = fmt::format("{}/{}/{}/aws4_request", DateStamp, Region, Service);
+ std::string Credential = fmt::format("{}/{}", Credentials.AccessKeyId, CredentialScope);
+
+ // The only signed header for pre-signed URLs is "host"
+ constexpr std::string_view SignedHeaders = "host";
+
+ // Build query parameters that will be part of the canonical request.
+ // These are the auth params (minus X-Amz-Signature which is added after signing).
+ std::vector<std::pair<std::string, std::string>> QueryParams = ExtraQueryParams;
+ QueryParams.emplace_back("X-Amz-Algorithm", "AWS4-HMAC-SHA256");
+ QueryParams.emplace_back("X-Amz-Credential", Credential);
+ QueryParams.emplace_back("X-Amz-Date", AmzDate);
+ QueryParams.emplace_back("X-Amz-Expires", fmt::format("{}", ExpiresIn.count()));
+ if (!Credentials.SessionToken.empty())
+ {
+ QueryParams.emplace_back("X-Amz-Security-Token", Credentials.SessionToken);
+ }
+ QueryParams.emplace_back("X-Amz-SignedHeaders", std::string(SignedHeaders));
+
+ std::string CanonicalQueryString = BuildCanonicalQueryString(QueryParams);
+ std::string CanonicalUri = AwsUriEncode(Path, false);
+
+ // For pre-signed URLs, the payload is always UNSIGNED-PAYLOAD
+ constexpr std::string_view PayloadHash = "UNSIGNED-PAYLOAD";
+
+ // Build the canonical request
+ // Only "host" is in the canonical headers for pre-signed URLs
+ std::string CanonicalHeaders = fmt::format("host:{}\n", Host);
+
+ std::string CanonicalRequest =
+ fmt::format("{}\n{}\n{}\n{}\n{}\n{}", Method, CanonicalUri, CanonicalQueryString, CanonicalHeaders, SignedHeaders, PayloadHash);
+
+ // Create the string to sign
+ Sha256Digest CanonicalRequestHash = ComputeSha256(CanonicalRequest);
+ std::string CanonicalRequestHex = Sha256ToHex(CanonicalRequestHash);
+
+ std::string StringToSign = fmt::format("AWS4-HMAC-SHA256\n{}\n{}\n{}", AmzDate, CredentialScope, CanonicalRequestHex);
+
+ // Calculate the signing key
+ std::string SecretPrefix = fmt::format("AWS4{}", Credentials.SecretAccessKey);
+ Sha256Digest DateKey = ComputeHmacSha256(SecretPrefix.data(), SecretPrefix.size(), DateStamp.data(), DateStamp.size());
+ SecureZeroSecret(SecretPrefix.data(), SecretPrefix.size());
+ Sha256Digest RegionKey = ComputeHmacSha256(DateKey, Region);
+ Sha256Digest ServiceKey = ComputeHmacSha256(RegionKey, Service);
+ Sha256Digest SigningKey = ComputeHmacSha256(ServiceKey, "aws4_request");
+
+ // Calculate the signature
+ std::string SignatureHex = Sha256ToHex(ComputeHmacSha256(SigningKey, StringToSign));
+
+ // Build the final URL (use the URI-encoded path so special characters are properly escaped)
+ return fmt::format("{}://{}{}?{}&X-Amz-Signature={}", Scheme, Host, CanonicalUri, CanonicalQueryString, SignatureHex);
+}
+
+//////////////////////////////////////////////////////////////////////////
+// Tests
+
+#if ZEN_WITH_TESTS
+
+void
+sigv4_forcelink()
+{
+}
+
+TEST_SUITE_BEGIN("util.cloud.sigv4");
+
+TEST_CASE("sigv4.sha256")
+{
+ // Test with known test vector (empty string)
+ Sha256Digest Empty = ComputeSha256("", 0);
+ std::string Hex = Sha256ToHex(Empty);
+ CHECK(Hex == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+
+ // Test with "hello"
+ Sha256Digest Hello = ComputeSha256("hello");
+ std::string HelloHex = Sha256ToHex(Hello);
+ CHECK(HelloHex == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824");
+}
+
+TEST_CASE("sigv4.hmac_sha256")
+{
+ // RFC 4231 Test Case 2
+ std::string_view Key = "Jefe";
+ std::string_view Data = "what do ya want for nothing?";
+
+ Sha256Digest Result = ComputeHmacSha256(Key.data(), Key.size(), Data.data(), Data.size());
+ std::string Hex = Sha256ToHex(Result);
+ CHECK(Hex == "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843");
+}
+
+TEST_CASE("sigv4.signing")
+{
+ // Based on the AWS SigV4 test suite example
+ // https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+
+ SigV4Credentials Creds;
+ Creds.AccessKeyId = "AKIDEXAMPLE";
+ Creds.SecretAccessKey = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY";
+
+ // We can't test with a fixed timestamp since SignRequestV4 uses current time,
+ // but we can verify the crypto primitives produce correct results by testing
+ // the signing key derivation manually.
+
+ // Test signing key derivation: HMAC chain for "20150830" / "us-east-1" / "iam"
+ std::string SecretPrefix = "AWS4wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY";
+ Sha256Digest DateKey = ComputeHmacSha256(SecretPrefix.data(), SecretPrefix.size(), "20150830", 8);
+ Sha256Digest RegionKey = ComputeHmacSha256(DateKey, "us-east-1");
+ Sha256Digest ServiceKey = ComputeHmacSha256(RegionKey, "iam");
+ Sha256Digest SigningKey = ComputeHmacSha256(ServiceKey, "aws4_request");
+
+ std::string SigningKeyHex = Sha256ToHex(SigningKey);
+ CHECK(SigningKeyHex == "c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9");
+}
+
+TEST_SUITE_END();
+
+#endif
+
+} // namespace zen