// Copyright Epic Games, Inc. All Rights Reserved. #include #include #include #include #if ZEN_WITH_TESTS # define ZEN_TEST_WITH_RUNNER 1 # include #endif #if ZEN_USE_MIMALLOC # include #endif ZEN_THIRD_PARTY_INCLUDES_START #include #include ZEN_THIRD_PARTY_INCLUDES_END ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END ZEN_THIRD_PARTY_INCLUDES_START #include #include #include #include #include #include #include #include #include #include #include #include ZEN_THIRD_PARTY_INCLUDES_END #include #include #include static const char ALLOCATION_TAG[] = "s3-crt-demo"; // List all Amazon Simple Storage Service (Amazon S3) buckets under the account. bool ListBuckets(const Aws::S3Crt::S3CrtClient& s3CrtClient) { Aws::S3Crt::Model::ListBucketsOutcome outcome = s3CrtClient.ListBuckets(); if (outcome.IsSuccess()) { std::cout << "All buckets under my account:" << std::endl; for (auto const& bucket : outcome.GetResult().GetBuckets()) { std::cout << " * " << bucket.GetName() << std::endl; } std::cout << std::endl; return true; } else { std::cout << "ListBuckets error:\n" << outcome.GetError() << std::endl << std::endl; return false; } } // Create an Amazon Simple Storage Service (Amazon S3) bucket. bool CreateBucket(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::S3Crt::Model::BucketLocationConstraint& locConstraint) { std::cout << "Creating bucket: \"" << bucketName << "\" ..." << std::endl; Aws::S3Crt::Model::CreateBucketRequest request; request.SetBucket(bucketName); // If you don't specify an AWS Region, the bucket is created in the US East (N. Virginia) Region (us-east-1) if (locConstraint != Aws::S3Crt::Model::BucketLocationConstraint::us_east_1) { Aws::S3Crt::Model::CreateBucketConfiguration bucket_config; bucket_config.SetLocationConstraint(locConstraint); request.SetCreateBucketConfiguration(bucket_config); } Aws::S3Crt::Model::CreateBucketOutcome outcome = s3CrtClient.CreateBucket(request); if (outcome.IsSuccess()) { std::cout << "Bucket created." << std::endl << std::endl; return true; } else { std::cout << "CreateBucket error:\n" << outcome.GetError() << std::endl << std::endl; return false; } } // Delete an existing Amazon S3 bucket. bool DeleteBucket(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName) { std::cout << "Deleting bucket: \"" << bucketName << "\" ..." << std::endl; Aws::S3Crt::Model::DeleteBucketRequest request; request.SetBucket(bucketName); Aws::S3Crt::Model::DeleteBucketOutcome outcome = s3CrtClient.DeleteBucket(request); if (outcome.IsSuccess()) { std::cout << "Bucket deleted." << std::endl << std::endl; return true; } else { std::cout << "DeleteBucket error:\n" << outcome.GetError() << std::endl << std::endl; return false; } } // Put an Amazon S3 object to the bucket. bool PutObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::String& objectKey, const Aws::String& fileName) { std::cout << "Putting object: \"" << objectKey << "\" to bucket: \"" << bucketName << "\" ..." << std::endl; Aws::S3Crt::Model::PutObjectRequest request; std::shared_ptr bodyStream = Aws::MakeShared(ALLOCATION_TAG, fileName.c_str(), std::ios_base::in | std::ios_base::binary); if (!bodyStream->good()) { std::cout << "Failed to open file: \"" << fileName << "\"." << std::endl << std::endl; return false; } request.SetBucket(bucketName); request.SetKey(objectKey); request.SetBody(bodyStream); // A PUT operation turns into a multipart upload using the s3-crt client. // https://github.com/aws/aws-sdk-cpp/wiki/Improving-S3-Throughput-with-AWS-SDK-for-CPP-v1.9 Aws::S3Crt::Model::PutObjectOutcome outcome = s3CrtClient.PutObject(request); if (outcome.IsSuccess()) { std::cout << "Object added." << std::endl << std::endl; return true; } else { std::cout << "PutObject error:\n" << outcome.GetError() << std::endl << std::endl; return false; } } // Get the Amazon S3 object from the bucket. bool GetObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::String& objectKey) { std::cout << "Getting object: \"" << objectKey << "\" from bucket: \"" << bucketName << "\" ..." << std::endl; Aws::S3Crt::Model::GetObjectRequest request; request.SetBucket(bucketName); request.SetKey(objectKey); Aws::S3Crt::Model::GetObjectOutcome outcome = s3CrtClient.GetObject(request); if (outcome.IsSuccess()) { // Uncomment this line if you wish to have the contents of the file displayed. Not recommended for large files // because it takes a while. // std::cout << "Object content: " << outcome.GetResult().GetBody().rdbuf() << std::endl << std::endl; return true; } else { std::cout << "GetObject error:\n" << outcome.GetError() << std::endl << std::endl; return false; } } // Delete the Amazon S3 object from the bucket. bool DeleteObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::String& objectKey) { std::cout << "Deleting object: \"" << objectKey << "\" from bucket: \"" << bucketName << "\" ..." << std::endl; Aws::S3Crt::Model::DeleteObjectRequest request; request.SetBucket(bucketName); request.SetKey(objectKey); Aws::S3Crt::Model::DeleteObjectOutcome outcome = s3CrtClient.DeleteObject(request); if (outcome.IsSuccess()) { std::cout << "Object deleted." << std::endl << std::endl; return true; } else { std::cout << "DeleteObject error:\n" << outcome.GetError() << std::endl << std::endl; return false; } } ////////////////////////////////////////////////////////////////////////// // TODO: should make this Unicode-aware so we can pass anything in on the // command line. struct ZenCloudOptions { bool IsDebug = false; bool IsVerbose = false; bool IsLocal = false; bool IsTest = false; std::string TestDataDirectory; // Arguments after " -- " on command line are passed through and not parsed std::string PassthroughCommandLine; std::string PassthroughArgs; std::vector PassthroughArgV; }; int DoWork(ZenCloudOptions& GlobalOptions) { using namespace std::literals; using namespace zen; // AWS SDK setup Aws::SDKOptions options; options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Info; Aws::InitAPI(options); { // TODO: Set to your account AWS Region. Aws::String region = Aws::Region::EU_NORTH_1; // US_EAST_1; // The object_key is the unique identifier for the object in the bucket. Aws::String object_key = "my-object"; // Create a globally unique name for the new bucket. // Format: "my-bucket-" + lowercase UUID. Aws::String uuid = Aws::Utils::UUID::RandomUUID(); Aws::String bucket_name = "my-bucket-" + Aws::Utils::StringUtils::ToLower(uuid.c_str()); const double throughput_target_gbps = 10; const uint64_t part_size = 32 * 1024 * 1024; Aws::S3Crt::ClientConfiguration config; config.region = region; config.throughputTargetGbps = throughput_target_gbps; config.partSize = part_size; // config.httpLibOverride = Aws::Http::TransferLibType::CURL_CLIENT; std::vector Tokens; const char* AwsAccessEnv = std::getenv("ZEN_AWS_ACCESS"); if (AwsAccessEnv && !GlobalOptions.IsLocal) { zen::ForEachStrTok(AwsAccessEnv, ':', [&](const std::string_view& Token) { Tokens.push_back(Token); return true; }); } else { Tokens.push_back("zencloud-test"sv); Tokens.push_back("misterblobby"sv); config.endpointOverride = "http://127.0.0.1:9000"; } ZEN_ASSERT(Tokens.size() == 2); Aws::Auth::AWSCredentials credentials; credentials.SetAWSAccessKeyId(Aws::String(Tokens[0])); credentials.SetAWSSecretKey(Aws::String(Tokens[1])); ZEN_CONSOLE("using credentials: {}/{}", Tokens[0], Tokens[1]); Aws::S3Crt::S3CrtClient s3_crt_client(credentials, config); // Use BucketLocationConstraintMapper to get the BucketLocationConstraint enum from the region string. // https://sdk.amazonaws.com/cpp/api/0.14.3/namespace_aws_1_1_s3_1_1_model_1_1_bucket_location_constraint_mapper.html#a50d4503d3f481022f969eff1085cfbb0 Aws::S3Crt::Model::BucketLocationConstraint locConstraint = Aws::S3Crt::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(region); ListBuckets(s3_crt_client); CreateBucket(s3_crt_client, bucket_name, locConstraint); Aws::String file_name = GlobalOptions.TestDataDirectory; PutObject(s3_crt_client, bucket_name, object_key, file_name); GetObject(s3_crt_client, bucket_name, object_key); DeleteObject(s3_crt_client, bucket_name, object_key); DeleteBucket(s3_crt_client, bucket_name); } Aws::ShutdownAPI(options); return 0; } int main(int argc, char** argv) { using namespace zen; using namespace std::literals; ZEN_UNUSED(argc, argv); #if ZEN_USE_MIMALLOC mi_version(); #endif zen::logging::InitializeLogging(); // Set output mode to handle virtual terminal sequences zen::logging::EnableVTMode(); std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); }); LoggerRef DefaultLogger = zen::logging::Default(); auto& Sinks = DefaultLogger.SpdLogger->sinks(); Sinks.clear(); auto ConsoleSink = std::make_shared(); Sinks.push_back(ConsoleSink); zen::MaximizeOpenFileCount(); // Split command line into options, commands and any pass-through arguments std::string Passthrough; std::string PassthroughArgs; std::vector PassthroughArgV; for (int i = 1; i < argc; ++i) { if ("--"sv == argv[i]) { bool IsFirst = true; ExtendableStringBuilder<256> Line; ExtendableStringBuilder<256> Arguments; for (int j = i + 1; j < argc; ++j) { auto AppendAscii = [&](auto X) { Line.Append(X); if (!IsFirst) { Arguments.Append(X); } }; if (!IsFirst) { AppendAscii(" "); } std::string_view ThisArg(argv[j]); PassthroughArgV.push_back(std::string(ThisArg)); const bool NeedsQuotes = (ThisArg.find(' ') != std::string_view::npos); if (NeedsQuotes) { AppendAscii("\""); } AppendAscii(ThisArg); if (NeedsQuotes) { AppendAscii("\""); } IsFirst = false; } Passthrough = Line.c_str(); PassthroughArgs = Arguments.c_str(); // This will "truncate" the arg vector and terminate the loop argc = i; } } ZenCloudOptions GlobalOptions; cxxopts::Options Options("zencloud", "Zen cloud interface"); Options.add_options()("d, debug", "Enable debugging", cxxopts::value(GlobalOptions.IsDebug)); Options.add_options()("v, verbose", "Enable verbose logging", cxxopts::value(GlobalOptions.IsVerbose)); Options.add_options()("help", "Show command line help"); Options.add_options()("local", "Use local server (such as minio)", cxxopts::value(GlobalOptions.IsLocal)); Options.add_options()("t, test", "Run tests", cxxopts::value(GlobalOptions.IsTest)); Options.add_options()("data", "Test data path", cxxopts::value(GlobalOptions.TestDataDirectory)); const bool IsNullInvoke = (argc == 1); // If no arguments are passed we want to print usage information try { cxxopts::ParseResult ParseResult = Options.parse(argc, argv); if (ParseResult.count("help") || IsNullInvoke == 1) { std::string Help = Options.help(); printf("%s\n", Help.c_str()); exit(0); } if (GlobalOptions.IsDebug) { logging::SetLogLevel(logging::level::Debug); } if (GlobalOptions.IsVerbose) { logging::SetLogLevel(logging::level::Trace); } if (GlobalOptions.IsTest) { DoWork(GlobalOptions); } } catch (const OptionParseException& Ex) { std::string HelpMessage = Options.help(); printf("Error parsing program arguments: %s\n\n%s", Ex.what(), HelpMessage.c_str()); return 9; } catch (const std::system_error& Ex) { printf("System Error: %s\n", Ex.what()); return Ex.code() ? Ex.code().value() : 10; } catch (const std::exception& Ex) { printf("Error: %s\n", Ex.what()); return 11; } return 0; }