Simple S3 client (#836)

This functionality is intended to be used to manage datasets for test cases, but may be useful elsewhere in the future.

- **Add S3 client with AWS Signature V4 (SigV4) signing** — new `S3Client` in `zenutil/cloud/` supporting `GetObject`, `PutObject`, `DeleteObject`, `HeadObject`, and `ListObjects` operations
- **Add EC2 IMDS credential provider** — automatically fetches and refreshes temporary AWS credentials from the EC2 Instance Metadata Service (IMDSv2) for use by the S3 client
- **Add SigV4 signing library** — standalone implementation of AWS Signature Version 4 request signing (headers and query-string presigning)
- **Add path-style addressing support** — enables compatibility with S3-compatible stores like MinIO (in addition to virtual-hosted style)
- **Add S3 integration tests** — includes a `MinioProcess` test helper that spins up a local MinIO server, plus integration tests exercising the S3 client end-to-end
- **Add S3-backed `HttpObjectStoreService` tests** — integration tests verifying the zenserver object store works against an S3 backend
- **Refactor mock IMDS into `zenutil/cloud/`** — moved and generalized the mock IMDS server from `zencompute` so it can be reused by both compute and S3 credential tests

1 files changed, 237 insertions, 0 deletions

diff --git a/src/zenutil/cloud/mockimds.cpp b/src/zenutil/cloud/mockimds.cpp
new file mode 100644
index 000000000..6919fab4d
--- /dev/null
+++ b/src/zenutil/cloud/mockimds.cpp
@@ -0,0 +1,237 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/cloud/mockimds.h>
+
+#include <zencore/fmtutils.h>
+
+#if ZEN_WITH_TESTS
+
+namespace zen::compute {
+
+const char*
+MockImdsService::BaseUri() const
+{
+	return "/";
+}
+
+void
+MockImdsService::HandleRequest(HttpServerRequest& Request)
+{
+	std::string_view Uri = Request.RelativeUri();
+
+	// AWS endpoints live under /latest/
+	if (Uri.starts_with("latest/"))
+	{
+		if (ActiveProvider == CloudProvider::AWS)
+		{
+			HandleAwsRequest(Request);
+			return;
+		}
+		Request.WriteResponse(HttpResponseCode::NotFound);
+		return;
+	}
+
+	// Azure endpoints live under /metadata/
+	if (Uri.starts_with("metadata/"))
+	{
+		if (ActiveProvider == CloudProvider::Azure)
+		{
+			HandleAzureRequest(Request);
+			return;
+		}
+		Request.WriteResponse(HttpResponseCode::NotFound);
+		return;
+	}
+
+	// GCP endpoints live under /computeMetadata/
+	if (Uri.starts_with("computeMetadata/"))
+	{
+		if (ActiveProvider == CloudProvider::GCP)
+		{
+			HandleGcpRequest(Request);
+			return;
+		}
+		Request.WriteResponse(HttpResponseCode::NotFound);
+		return;
+	}
+
+	Request.WriteResponse(HttpResponseCode::NotFound);
+}
+
+// ---------------------------------------------------------------------------
+// AWS
+// ---------------------------------------------------------------------------
+
+void
+MockImdsService::HandleAwsRequest(HttpServerRequest& Request)
+{
+	std::string_view Uri = Request.RelativeUri();
+
+	// IMDSv2 token acquisition (PUT only)
+	if (Uri == "latest/api/token" && Request.RequestVerb() == HttpVerb::kPut)
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.Token);
+		return;
+	}
+
+	// Instance identity
+	if (Uri == "latest/meta-data/instance-id")
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.InstanceId);
+		return;
+	}
+
+	if (Uri == "latest/meta-data/placement/availability-zone")
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.AvailabilityZone);
+		return;
+	}
+
+	if (Uri == "latest/meta-data/instance-life-cycle")
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.LifeCycle);
+		return;
+	}
+
+	// Autoscaling lifecycle state — 404 when not in an ASG
+	if (Uri == "latest/meta-data/autoscaling/target-lifecycle-state")
+	{
+		if (Aws.AutoscalingState.empty())
+		{
+			Request.WriteResponse(HttpResponseCode::NotFound);
+			return;
+		}
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.AutoscalingState);
+		return;
+	}
+
+	// Spot interruption notice — 404 when no interruption pending
+	if (Uri == "latest/meta-data/spot/instance-action")
+	{
+		if (Aws.SpotAction.empty())
+		{
+			Request.WriteResponse(HttpResponseCode::NotFound);
+			return;
+		}
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.SpotAction);
+		return;
+	}
+
+	// IAM role discovery — returns the role name
+	if (Uri == "latest/meta-data/iam/security-credentials/")
+	{
+		if (Aws.IamRoleName.empty())
+		{
+			Request.WriteResponse(HttpResponseCode::NotFound);
+			return;
+		}
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.IamRoleName);
+		return;
+	}
+
+	// IAM credentials for a specific role
+	constexpr std::string_view kIamCredPrefix = "latest/meta-data/iam/security-credentials/";
+	if (Uri.starts_with(kIamCredPrefix) && Uri.size() > kIamCredPrefix.size())
+	{
+		std::string_view RequestedRole = Uri.substr(kIamCredPrefix.size());
+		if (RequestedRole == Aws.IamRoleName)
+		{
+			std::string Json =
+				fmt::format(R"({{"Code":"Success","AccessKeyId":"{}","SecretAccessKey":"{}","Token":"{}","Expiration":"{}"}})",
+							Aws.IamAccessKeyId,
+							Aws.IamSecretAccessKey,
+							Aws.IamSessionToken,
+							Aws.IamExpiration);
+			Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Json);
+			return;
+		}
+		Request.WriteResponse(HttpResponseCode::NotFound);
+		return;
+	}
+
+	Request.WriteResponse(HttpResponseCode::NotFound);
+}
+
+// ---------------------------------------------------------------------------
+// Azure
+// ---------------------------------------------------------------------------
+
+void
+MockImdsService::HandleAzureRequest(HttpServerRequest& Request)
+{
+	std::string_view Uri = Request.RelativeUri();
+
+	// Instance metadata (single JSON document)
+	if (Uri == "metadata/instance")
+	{
+		std::string Json = fmt::format(R"({{"compute":{{"vmId":"{}","location":"{}","priority":"{}","vmScaleSetName":"{}"}}}})",
+									   Azure.VmId,
+									   Azure.Location,
+									   Azure.Priority,
+									   Azure.VmScaleSetName);
+
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Json);
+		return;
+	}
+
+	// Scheduled events for termination monitoring
+	if (Uri == "metadata/scheduledevents")
+	{
+		std::string Json;
+		if (Azure.ScheduledEventType.empty())
+		{
+			Json = R"({"Events":[]})";
+		}
+		else
+		{
+			Json = fmt::format(R"({{"Events":[{{"EventType":"{}","EventStatus":"{}"}}]}})",
+							   Azure.ScheduledEventType,
+							   Azure.ScheduledEventStatus);
+		}
+
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Json);
+		return;
+	}
+
+	Request.WriteResponse(HttpResponseCode::NotFound);
+}
+
+// ---------------------------------------------------------------------------
+// GCP
+// ---------------------------------------------------------------------------
+
+void
+MockImdsService::HandleGcpRequest(HttpServerRequest& Request)
+{
+	std::string_view Uri = Request.RelativeUri();
+
+	if (Uri == "computeMetadata/v1/instance/id")
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.InstanceId);
+		return;
+	}
+
+	if (Uri == "computeMetadata/v1/instance/zone")
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.Zone);
+		return;
+	}
+
+	if (Uri == "computeMetadata/v1/instance/scheduling/preemptible")
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.Preemptible);
+		return;
+	}
+
+	if (Uri == "computeMetadata/v1/instance/maintenance-event")
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.MaintenanceEvent);
+		return;
+	}
+
+	Request.WriteResponse(HttpResponseCode::NotFound);
+}
+
+}  // namespace zen::compute
+
+#endif	// ZEN_WITH_TESTS