From 81e7101fa4e84e7bd86287ea76a63e088941287c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 23 Apr 2024 16:59:27 +0200 Subject: added some simple test code to exercise SDK bits --- src/zencloud/zencloudmain.cpp | 441 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 406 insertions(+), 35 deletions(-) (limited to 'src/zencloud/zencloudmain.cpp') diff --git a/src/zencloud/zencloudmain.cpp b/src/zencloud/zencloudmain.cpp index 4fd771e68..e87850047 100644 --- a/src/zencloud/zencloudmain.cpp +++ b/src/zencloud/zencloudmain.cpp @@ -1,7 +1,9 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#include #include #include +#include #if ZEN_WITH_TESTS # define ZEN_TEST_WITH_RUNNER 1 @@ -17,8 +19,13 @@ ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + ZEN_THIRD_PARTY_INCLUDES_START #include +#include #include #include #include @@ -31,10 +38,307 @@ ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END +#include +#include +#include + +static const char ALLOCATION_TAG[] = "s3-crt-demo"; + +// List all Amazon Simple Storage Service (Amazon S3) buckets under the account. +bool +ListBuckets(const Aws::S3Crt::S3CrtClient& s3CrtClient) +{ + Aws::S3Crt::Model::ListBucketsOutcome outcome = s3CrtClient.ListBuckets(); + + if (outcome.IsSuccess()) + { + std::cout << "All buckets under my account:" << std::endl; + + for (auto const& bucket : outcome.GetResult().GetBuckets()) + { + std::cout << " * " << bucket.GetName() << std::endl; + } + std::cout << std::endl; + + return true; + } + else + { + std::cout << "ListBuckets error:\n" << outcome.GetError() << std::endl << std::endl; + + return false; + } +} + +// Create an Amazon Simple Storage Service (Amazon S3) bucket. +bool +CreateBucket(const Aws::S3Crt::S3CrtClient& s3CrtClient, + const Aws::String& bucketName, + const Aws::S3Crt::Model::BucketLocationConstraint& locConstraint) +{ + std::cout << "Creating bucket: \"" << bucketName << "\" ..." << std::endl; + + Aws::S3Crt::Model::CreateBucketRequest request; + request.SetBucket(bucketName); + + // If you don't specify an AWS Region, the bucket is created in the US East (N. Virginia) Region (us-east-1) + if (locConstraint != Aws::S3Crt::Model::BucketLocationConstraint::us_east_1) + { + Aws::S3Crt::Model::CreateBucketConfiguration bucket_config; + bucket_config.SetLocationConstraint(locConstraint); + + request.SetCreateBucketConfiguration(bucket_config); + } + + Aws::S3Crt::Model::CreateBucketOutcome outcome = s3CrtClient.CreateBucket(request); + + if (outcome.IsSuccess()) + { + std::cout << "Bucket created." << std::endl << std::endl; + + return true; + } + else + { + std::cout << "CreateBucket error:\n" << outcome.GetError() << std::endl << std::endl; + + return false; + } +} + +// Delete an existing Amazon S3 bucket. +bool +DeleteBucket(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName) +{ + std::cout << "Deleting bucket: \"" << bucketName << "\" ..." << std::endl; + + Aws::S3Crt::Model::DeleteBucketRequest request; + request.SetBucket(bucketName); + + Aws::S3Crt::Model::DeleteBucketOutcome outcome = s3CrtClient.DeleteBucket(request); + + if (outcome.IsSuccess()) + { + std::cout << "Bucket deleted." << std::endl << std::endl; + + return true; + } + else + { + std::cout << "DeleteBucket error:\n" << outcome.GetError() << std::endl << std::endl; + + return false; + } +} + +// Put an Amazon S3 object to the bucket. +bool +PutObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, + const Aws::String& bucketName, + const Aws::String& objectKey, + const Aws::String& fileName) +{ + std::cout << "Putting object: \"" << objectKey << "\" to bucket: \"" << bucketName << "\" ..." << std::endl; + + Aws::S3Crt::Model::PutObjectRequest request; + request.SetBucket(bucketName); + request.SetKey(objectKey); + + // zen::SHA1Stream Sha1; + // zen::ScanFile(fileName, 16 * 1024 * 1024, [&](const void* Data, size_t Size) { Sha1.Append(Data, Size); }); + // zen::SHA1 Sha1Hash = Sha1.GetHash(); + // zen::StringBuilder<64> Sha1String; + // Sha1Hash.ToHexString(Sha1String); + + // request.SetChecksumSHA1(Sha1String.c_str()); + + std::shared_ptr bodyStream = + Aws::MakeShared(ALLOCATION_TAG, fileName.c_str(), std::ios_base::in | std::ios_base::binary); + if (!bodyStream->good()) + { + std::cout << "Failed to open file: \"" << fileName << "\"." << std::endl << std::endl; + return false; + } + + // auto Sha1Hash = Aws::Utils::HashingUtils::HexEncode(Aws::Utils::HashingUtils::CalculateSHA1(*bodyStream)); + // request.SetChecksumSHA1(Sha1Hash); + // bodyStream->seekg(0); + + request.SetBody(bodyStream); + + // A PUT operation turns into a multipart upload using the s3-crt client. + // https://github.com/aws/aws-sdk-cpp/wiki/Improving-S3-Throughput-with-AWS-SDK-for-CPP-v1.9 + Aws::S3Crt::Model::PutObjectOutcome outcome = s3CrtClient.PutObject(request); + + if (outcome.IsSuccess()) + { + std::cout << "Object added." << std::endl << std::endl; + + return true; + } + else + { + std::cout << "PutObject error:\n" << outcome.GetError() << std::endl << std::endl; + + return false; + } +} + +// Get the Amazon S3 object from the bucket. +bool +GetObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::String& objectKey) +{ + std::cout << "Getting object: \"" << objectKey << "\" from bucket: \"" << bucketName << "\" ..." << std::endl; + + Aws::S3Crt::Model::GetObjectRequest request; + request.SetBucket(bucketName); + request.SetKey(objectKey); + + Aws::S3Crt::Model::GetObjectOutcome outcome = s3CrtClient.GetObject(request); + + if (outcome.IsSuccess()) + { + // Uncomment this line if you wish to have the contents of the file displayed. Not recommended for large files + // because it takes a while. + // std::cout << "Object content: " << outcome.GetResult().GetBody().rdbuf() << std::endl << std::endl; + + return true; + } + else + { + std::cout << "GetObject error:\n" << outcome.GetError() << std::endl << std::endl; + + return false; + } +} + +// Delete the Amazon S3 object from the bucket. +bool +DeleteObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::String& objectKey) +{ + std::cout << "Deleting object: \"" << objectKey << "\" from bucket: \"" << bucketName << "\" ..." << std::endl; + + Aws::S3Crt::Model::DeleteObjectRequest request; + request.SetBucket(bucketName); + request.SetKey(objectKey); + + Aws::S3Crt::Model::DeleteObjectOutcome outcome = s3CrtClient.DeleteObject(request); + + if (outcome.IsSuccess()) + { + std::cout << "Object deleted." << std::endl << std::endl; + + return true; + } + else + { + std::cout << "DeleteObject error:\n" << outcome.GetError() << std::endl << std::endl; + + return false; + } +} + ////////////////////////////////////////////////////////////////////////// // TODO: should make this Unicode-aware so we can pass anything in on the // command line. +struct ZenCloudOptions +{ + bool IsDebug = false; + bool IsVerbose = false; + bool IsLocal = false; + bool IsTest = false; + std::string TestDataDirectory; + + // Arguments after " -- " on command line are passed through and not parsed + std::string PassthroughCommandLine; + std::string PassthroughArgs; + std::vector PassthroughArgV; +}; + +int +DoWork(ZenCloudOptions& GlobalOptions) +{ + using namespace std::literals; + using namespace zen; + + // AWS SDK setup + + Aws::SDKOptions options; + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + Aws::InitAPI(options); + { + // TODO: Set to your account AWS Region. + Aws::String region = Aws::Region::EU_NORTH_1; // US_EAST_1; + + // The object_key is the unique identifier for the object in the bucket. + Aws::String object_key = "my-object"; + + // Create a globally unique name for the new bucket. + // Format: "my-bucket-" + lowercase UUID. + Aws::String uuid = Aws::Utils::UUID::RandomUUID(); + Aws::String bucket_name = "my-bucket-" + Aws::Utils::StringUtils::ToLower(uuid.c_str()); + + const double throughput_target_gbps = 10; + const uint64_t part_size = 32 * 1024 * 1024; + + Aws::S3Crt::ClientConfiguration config; + config.region = region; + config.throughputTargetGbps = throughput_target_gbps; + config.partSize = part_size; + + std::vector Tokens; + + const char* AwsAccessEnv = std::getenv("ZEN_AWS_ACCESS"); + + if (AwsAccessEnv && !GlobalOptions.IsLocal) + { + zen::ForEachStrTok(AwsAccessEnv, ':', [&](const std::string_view& Token) { + Tokens.push_back(Token); + return true; + }); + } + else + { + Tokens.push_back("zencloud-test"sv); + Tokens.push_back("misterblobby"sv); + + config.endpointOverride = "http://127.0.0.1:9000"; + } + + ZEN_ASSERT(Tokens.size() == 2); + + Aws::Auth::AWSCredentials credentials; + credentials.SetAWSAccessKeyId(Aws::String(Tokens[0])); + credentials.SetAWSSecretKey(Aws::String(Tokens[1])); + + ZEN_CONSOLE("using credentials: {}/{}", Tokens[0], Tokens[1]); + + Aws::S3Crt::S3CrtClient s3_crt_client(credentials, config); + + // Use BucketLocationConstraintMapper to get the BucketLocationConstraint enum from the region string. + // https://sdk.amazonaws.com/cpp/api/0.14.3/namespace_aws_1_1_s3_1_1_model_1_1_bucket_location_constraint_mapper.html#a50d4503d3f481022f969eff1085cfbb0 + Aws::S3Crt::Model::BucketLocationConstraint locConstraint = + Aws::S3Crt::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(region); + + ListBuckets(s3_crt_client); + CreateBucket(s3_crt_client, bucket_name, locConstraint); + + // TODO: Add a large file to your executable folder, and update file_name to the name of that file. + // File "ny.json" (1940 census data; https://www.archives.gov/developer/1940-census#accessportiondataset) + // is an example data file large enough to demonstrate multipart upload. + // Download "ny.json" from https://nara-1940-census.s3.us-east-2.amazonaws.com/metadata/json/ny.json + Aws::String file_name = GlobalOptions.TestDataDirectory; + + PutObject(s3_crt_client, bucket_name, object_key, file_name); + GetObject(s3_crt_client, bucket_name, object_key); + DeleteObject(s3_crt_client, bucket_name, object_key); + DeleteBucket(s3_crt_client, bucket_name); + } + Aws::ShutdownAPI(options); + return 0; +} + int main(int argc, char** argv) { @@ -62,56 +366,123 @@ main(int argc, char** argv) zen::MaximizeOpenFileCount(); - // AWS SDK setup + // Split command line into options, commands and any pass-through arguments - Aws::SDKOptions options; - options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; - Aws::InitAPI(options); + std::string Passthrough; + std::string PassthroughArgs; + std::vector PassthroughArgV; + + for (int i = 1; i < argc; ++i) { - // TODO: Add a large file to your executable folder, and update file_name to the name of that file. - // File "ny.json" (1940 census data; https://www.archives.gov/developer/1940-census#accessportiondataset) - // is an example data file large enough to demonstrate multipart upload. - // Download "ny.json" from https://nara-1940-census.s3.us-east-2.amazonaws.com/metadata/json/ny.json - Aws::String file_name = "ny.json"; + if ("--"sv == argv[i]) + { + bool IsFirst = true; + ExtendableStringBuilder<256> Line; + ExtendableStringBuilder<256> Arguments; - // TODO: Set to your account AWS Region. - Aws::String region = Aws::Region::US_EAST_1; + for (int j = i + 1; j < argc; ++j) + { + auto AppendAscii = [&](auto X) { + Line.Append(X); + if (!IsFirst) + { + Arguments.Append(X); + } + }; - // The object_key is the unique identifier for the object in the bucket. - Aws::String object_key = "my-object"; + if (!IsFirst) + { + AppendAscii(" "); + } - // Create a globally unique name for the new bucket. - // Format: "my-bucket-" + lowercase UUID. - Aws::String uuid = Aws::Utils::UUID::RandomUUID(); - Aws::String bucket_name = "my-bucket-" + Aws::Utils::StringUtils::ToLower(uuid.c_str()); + std::string_view ThisArg(argv[j]); + PassthroughArgV.push_back(std::string(ThisArg)); - const double throughput_target_gbps = 5; - const uint64_t part_size = 8 * 1024 * 1024; // 8 MB. + const bool NeedsQuotes = (ThisArg.find(' ') != std::string_view::npos); - Aws::S3Crt::ClientConfiguration config; - config.region = region; - config.throughputTargetGbps = throughput_target_gbps; - config.partSize = part_size; + if (NeedsQuotes) + { + AppendAscii("\""); + } - Aws::S3Crt::S3CrtClient s3_crt_client(config); + AppendAscii(ThisArg); - // Use BucketLocationConstraintMapper to get the BucketLocationConstraint enum from the region string. - // https://sdk.amazonaws.com/cpp/api/0.14.3/namespace_aws_1_1_s3_1_1_model_1_1_bucket_location_constraint_mapper.html#a50d4503d3f481022f969eff1085cfbb0 - Aws::S3Crt::Model::BucketLocationConstraint locConstraint = - Aws::S3Crt::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(region); + if (NeedsQuotes) + { + AppendAscii("\""); + } + + IsFirst = false; + } + + Passthrough = Line.c_str(); + PassthroughArgs = Arguments.c_str(); + + // This will "truncate" the arg vector and terminate the loop + argc = i; + } + } - // ListBuckets(s3_crt_client, bucket_name); + ZenCloudOptions GlobalOptions; + cxxopts::Options Options("zencloud", "Zen cloud interface"); - // CreateBucket(s3_crt_client, bucket_name, locConstraint); + Options.add_options()("d, debug", "Enable debugging", cxxopts::value(GlobalOptions.IsDebug)); + Options.add_options()("v, verbose", "Enable verbose logging", cxxopts::value(GlobalOptions.IsVerbose)); + Options.add_options()("help", "Show command line help"); + Options.add_options()("local", "Use local server (such as minio)", cxxopts::value(GlobalOptions.IsLocal)); + Options.add_options()("t, test", "Run tests", cxxopts::value(GlobalOptions.IsTest)); + Options.add_options()("data", "Test data path", cxxopts::value(GlobalOptions.TestDataDirectory)); - // PutObject(s3_crt_client, bucket_name, object_key, file_name); + const bool IsNullInvoke = (argc == 1); // If no arguments are passed we want to print usage information - // GetObject(s3_crt_client, bucket_name, object_key); + try + { + cxxopts::ParseResult ParseResult = Options.parse(argc, argv); + + if (ParseResult.count("help") || IsNullInvoke == 1) + { + std::string Help = Options.help(); + + printf("%s\n", Help.c_str()); - // DeleteObject(s3_crt_client, bucket_name, object_key); + exit(0); + } - // DeleteBucket(s3_crt_client, bucket_name); + if (GlobalOptions.IsDebug) + { + logging::SetLogLevel(logging::level::Debug); + } + + if (GlobalOptions.IsVerbose) + { + logging::SetLogLevel(logging::level::Trace); + } + + if (GlobalOptions.IsTest) + { + DoWork(GlobalOptions); + } } - Aws::ShutdownAPI(options); + catch (const OptionParseException& Ex) + { + std::string HelpMessage = Options.help(); + + printf("Error parsing program arguments: %s\n\n%s", Ex.what(), HelpMessage.c_str()); + + return 9; + } + catch (const std::system_error& Ex) + { + printf("System Error: %s\n", Ex.what()); + + return Ex.code() ? Ex.code().value() : 10; + } + catch (const std::exception& Ex) + { + printf("Error: %s\n", Ex.what()); + + return 11; + } + return 0; } -- cgit v1.2.3