added some simple test code to exercise SDK bits

1 files changed, 406 insertions, 35 deletions

diff --git a/src/zencloud/zencloudmain.cpp b/src/zencloud/zencloudmain.cpp
index 4fd771e68..e87850047 100644
--- a/src/zencloud/zencloudmain.cpp
+++ b/src/zencloud/zencloudmain.cpp
@@ -1,7 +1,9 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
+#include <zencore/except.h>
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
+#include <zencore/sha1.h>
 
 #if ZEN_WITH_TESTS
 #	define ZEN_TEST_WITH_RUNNER 1
@@ -18,7 +20,12 @@ ZEN_THIRD_PARTY_INCLUDES_START
 ZEN_THIRD_PARTY_INCLUDES_END
 
 ZEN_THIRD_PARTY_INCLUDES_START
+#include <cxxopts.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+ZEN_THIRD_PARTY_INCLUDES_START
 #include <aws/core/Aws.h>
+#include <aws/core/utils/HashingUtils.h>
 #include <aws/core/utils/UUID.h>
 #include <aws/core/utils/logging/CRTLogSystem.h>
 #include <aws/core/utils/memory/stl/AWSStringStream.h>
@@ -31,10 +38,307 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <aws/s3-crt/model/PutObjectRequest.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+
+static const char ALLOCATION_TAG[] = "s3-crt-demo";
+
+// List all Amazon Simple Storage Service (Amazon S3) buckets under the account.
+bool
+ListBuckets(const Aws::S3Crt::S3CrtClient& s3CrtClient)
+{
+	Aws::S3Crt::Model::ListBucketsOutcome outcome = s3CrtClient.ListBuckets();
+
+	if (outcome.IsSuccess())
+	{
+		std::cout << "All buckets under my account:" << std::endl;
+
+		for (auto const& bucket : outcome.GetResult().GetBuckets())
+		{
+			std::cout << "  * " << bucket.GetName() << std::endl;
+		}
+		std::cout << std::endl;
+
+		return true;
+	}
+	else
+	{
+		std::cout << "ListBuckets error:\n" << outcome.GetError() << std::endl << std::endl;
+
+		return false;
+	}
+}
+
+// Create an Amazon Simple Storage Service (Amazon S3) bucket.
+bool
+CreateBucket(const Aws::S3Crt::S3CrtClient&						s3CrtClient,
+			 const Aws::String&									bucketName,
+			 const Aws::S3Crt::Model::BucketLocationConstraint& locConstraint)
+{
+	std::cout << "Creating bucket: \"" << bucketName << "\" ..." << std::endl;
+
+	Aws::S3Crt::Model::CreateBucketRequest request;
+	request.SetBucket(bucketName);
+
+	//  If you don't specify an AWS Region, the bucket is created in the US East (N. Virginia) Region (us-east-1)
+	if (locConstraint != Aws::S3Crt::Model::BucketLocationConstraint::us_east_1)
+	{
+		Aws::S3Crt::Model::CreateBucketConfiguration bucket_config;
+		bucket_config.SetLocationConstraint(locConstraint);
+
+		request.SetCreateBucketConfiguration(bucket_config);
+	}
+
+	Aws::S3Crt::Model::CreateBucketOutcome outcome = s3CrtClient.CreateBucket(request);
+
+	if (outcome.IsSuccess())
+	{
+		std::cout << "Bucket created." << std::endl << std::endl;
+
+		return true;
+	}
+	else
+	{
+		std::cout << "CreateBucket error:\n" << outcome.GetError() << std::endl << std::endl;
+
+		return false;
+	}
+}
+
+// Delete an existing Amazon S3 bucket.
+bool
+DeleteBucket(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName)
+{
+	std::cout << "Deleting bucket: \"" << bucketName << "\" ..." << std::endl;
+
+	Aws::S3Crt::Model::DeleteBucketRequest request;
+	request.SetBucket(bucketName);
+
+	Aws::S3Crt::Model::DeleteBucketOutcome outcome = s3CrtClient.DeleteBucket(request);
+
+	if (outcome.IsSuccess())
+	{
+		std::cout << "Bucket deleted." << std::endl << std::endl;
+
+		return true;
+	}
+	else
+	{
+		std::cout << "DeleteBucket error:\n" << outcome.GetError() << std::endl << std::endl;
+
+		return false;
+	}
+}
+
+// Put an Amazon S3 object to the bucket.
+bool
+PutObject(const Aws::S3Crt::S3CrtClient& s3CrtClient,
+		  const Aws::String&			 bucketName,
+		  const Aws::String&			 objectKey,
+		  const Aws::String&			 fileName)
+{
+	std::cout << "Putting object: \"" << objectKey << "\" to bucket: \"" << bucketName << "\" ..." << std::endl;
+
+	Aws::S3Crt::Model::PutObjectRequest request;
+	request.SetBucket(bucketName);
+	request.SetKey(objectKey);
+
+	// zen::SHA1Stream Sha1;
+	// zen::ScanFile(fileName, 16 * 1024 * 1024, [&](const void* Data, size_t Size) { Sha1.Append(Data, Size); });
+	// zen::SHA1 Sha1Hash = Sha1.GetHash();
+	// zen::StringBuilder<64> Sha1String;
+	// Sha1Hash.ToHexString(Sha1String);
+
+	// request.SetChecksumSHA1(Sha1String.c_str());
+
+	std::shared_ptr<Aws::IOStream> bodyStream =
+		Aws::MakeShared<Aws::FStream>(ALLOCATION_TAG, fileName.c_str(), std::ios_base::in | std::ios_base::binary);
+	if (!bodyStream->good())
+	{
+		std::cout << "Failed to open file: \"" << fileName << "\"." << std::endl << std::endl;
+		return false;
+	}
+
+	// auto Sha1Hash = Aws::Utils::HashingUtils::HexEncode(Aws::Utils::HashingUtils::CalculateSHA1(*bodyStream));
+	// request.SetChecksumSHA1(Sha1Hash);
+	// bodyStream->seekg(0);
+
+	request.SetBody(bodyStream);
+
+	// A PUT operation turns into a multipart upload using the s3-crt client.
+	// https://github.com/aws/aws-sdk-cpp/wiki/Improving-S3-Throughput-with-AWS-SDK-for-CPP-v1.9
+	Aws::S3Crt::Model::PutObjectOutcome outcome = s3CrtClient.PutObject(request);
+
+	if (outcome.IsSuccess())
+	{
+		std::cout << "Object added." << std::endl << std::endl;
+
+		return true;
+	}
+	else
+	{
+		std::cout << "PutObject error:\n" << outcome.GetError() << std::endl << std::endl;
+
+		return false;
+	}
+}
+
+// Get the Amazon S3 object from the bucket.
+bool
+GetObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::String& objectKey)
+{
+	std::cout << "Getting object: \"" << objectKey << "\" from bucket: \"" << bucketName << "\" ..." << std::endl;
+
+	Aws::S3Crt::Model::GetObjectRequest request;
+	request.SetBucket(bucketName);
+	request.SetKey(objectKey);
+
+	Aws::S3Crt::Model::GetObjectOutcome outcome = s3CrtClient.GetObject(request);
+
+	if (outcome.IsSuccess())
+	{
+		// Uncomment this line if you wish to have the contents of the file displayed. Not recommended for large files
+		//  because it takes a while.
+		//  std::cout << "Object content: " << outcome.GetResult().GetBody().rdbuf() << std::endl << std::endl;
+
+		return true;
+	}
+	else
+	{
+		std::cout << "GetObject error:\n" << outcome.GetError() << std::endl << std::endl;
+
+		return false;
+	}
+}
+
+// Delete the Amazon S3 object from the bucket.
+bool
+DeleteObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::String& objectKey)
+{
+	std::cout << "Deleting object: \"" << objectKey << "\" from bucket: \"" << bucketName << "\" ..." << std::endl;
+
+	Aws::S3Crt::Model::DeleteObjectRequest request;
+	request.SetBucket(bucketName);
+	request.SetKey(objectKey);
+
+	Aws::S3Crt::Model::DeleteObjectOutcome outcome = s3CrtClient.DeleteObject(request);
+
+	if (outcome.IsSuccess())
+	{
+		std::cout << "Object deleted." << std::endl << std::endl;
+
+		return true;
+	}
+	else
+	{
+		std::cout << "DeleteObject error:\n" << outcome.GetError() << std::endl << std::endl;
+
+		return false;
+	}
+}
+
 //////////////////////////////////////////////////////////////////////////
 // TODO: should make this Unicode-aware so we can pass anything in on the
 // command line.
 
+struct ZenCloudOptions
+{
+	bool		IsDebug	  = false;
+	bool		IsVerbose = false;
+	bool		IsLocal	  = false;
+	bool		IsTest	  = false;
+	std::string TestDataDirectory;
+
+	// Arguments after " -- " on command line are passed through and not parsed
+	std::string				 PassthroughCommandLine;
+	std::string				 PassthroughArgs;
+	std::vector<std::string> PassthroughArgV;
+};
+
+int
+DoWork(ZenCloudOptions& GlobalOptions)
+{
+	using namespace std::literals;
+	using namespace zen;
+
+	// AWS SDK setup
+
+	Aws::SDKOptions options;
+	options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
+	Aws::InitAPI(options);
+	{
+		// TODO: Set to your account AWS Region.
+		Aws::String region = Aws::Region::EU_NORTH_1;  // US_EAST_1;
+
+		// The object_key is the unique identifier for the object in the bucket.
+		Aws::String object_key = "my-object";
+
+		// Create a globally unique name for the new bucket.
+		// Format: "my-bucket-" + lowercase UUID.
+		Aws::String uuid		= Aws::Utils::UUID::RandomUUID();
+		Aws::String bucket_name = "my-bucket-" + Aws::Utils::StringUtils::ToLower(uuid.c_str());
+
+		const double   throughput_target_gbps = 10;
+		const uint64_t part_size			  = 32 * 1024 * 1024;
+
+		Aws::S3Crt::ClientConfiguration config;
+		config.region				= region;
+		config.throughputTargetGbps = throughput_target_gbps;
+		config.partSize				= part_size;
+
+		std::vector<std::string_view> Tokens;
+
+		const char* AwsAccessEnv = std::getenv("ZEN_AWS_ACCESS");
+
+		if (AwsAccessEnv && !GlobalOptions.IsLocal)
+		{
+			zen::ForEachStrTok(AwsAccessEnv, ':', [&](const std::string_view& Token) {
+				Tokens.push_back(Token);
+				return true;
+			});
+		}
+		else
+		{
+			Tokens.push_back("zencloud-test"sv);
+			Tokens.push_back("misterblobby"sv);
+
+			config.endpointOverride = "http://127.0.0.1:9000";
+		}
+
+		ZEN_ASSERT(Tokens.size() == 2);
+
+		Aws::Auth::AWSCredentials credentials;
+		credentials.SetAWSAccessKeyId(Aws::String(Tokens[0]));
+		credentials.SetAWSSecretKey(Aws::String(Tokens[1]));
+
+		ZEN_CONSOLE("using credentials: {}/{}", Tokens[0], Tokens[1]);
+
+		Aws::S3Crt::S3CrtClient s3_crt_client(credentials, config);
+
+		// Use BucketLocationConstraintMapper to get the BucketLocationConstraint enum from the region string.
+		// https://sdk.amazonaws.com/cpp/api/0.14.3/namespace_aws_1_1_s3_1_1_model_1_1_bucket_location_constraint_mapper.html#a50d4503d3f481022f969eff1085cfbb0
+		Aws::S3Crt::Model::BucketLocationConstraint locConstraint =
+			Aws::S3Crt::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(region);
+
+		ListBuckets(s3_crt_client);
+		CreateBucket(s3_crt_client, bucket_name, locConstraint);
+
+		// TODO: Add a large file to your executable folder, and update file_name to the name of that file.
+		//    File "ny.json" (1940 census data; https://www.archives.gov/developer/1940-census#accessportiondataset)
+		//    is an example data file large enough to demonstrate multipart upload.
+		// Download "ny.json" from https://nara-1940-census.s3.us-east-2.amazonaws.com/metadata/json/ny.json
+		Aws::String file_name = GlobalOptions.TestDataDirectory;
+
+		PutObject(s3_crt_client, bucket_name, object_key, file_name);
+		GetObject(s3_crt_client, bucket_name, object_key);
+		DeleteObject(s3_crt_client, bucket_name, object_key);
+		DeleteBucket(s3_crt_client, bucket_name);
+	}
+	Aws::ShutdownAPI(options);
+	return 0;
+}
+
 int
 main(int argc, char** argv)
 {
@@ -62,56 +366,123 @@ main(int argc, char** argv)
 
 	zen::MaximizeOpenFileCount();
 
-	// AWS SDK setup
+	// Split command line into options, commands and any pass-through arguments
 
-	Aws::SDKOptions options;
-	options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
-	Aws::InitAPI(options);
+	std::string				 Passthrough;
+	std::string				 PassthroughArgs;
+	std::vector<std::string> PassthroughArgV;
+
+	for (int i = 1; i < argc; ++i)
 	{
-		// TODO: Add a large file to your executable folder, and update file_name to the name of that file.
-		//    File "ny.json" (1940 census data; https://www.archives.gov/developer/1940-census#accessportiondataset)
-		//    is an example data file large enough to demonstrate multipart upload.
-		// Download "ny.json" from https://nara-1940-census.s3.us-east-2.amazonaws.com/metadata/json/ny.json
-		Aws::String file_name = "ny.json";
+		if ("--"sv == argv[i])
+		{
+			bool						 IsFirst = true;
+			ExtendableStringBuilder<256> Line;
+			ExtendableStringBuilder<256> Arguments;
 
-		// TODO: Set to your account AWS Region.
-		Aws::String region = Aws::Region::US_EAST_1;
+			for (int j = i + 1; j < argc; ++j)
+			{
+				auto AppendAscii = [&](auto X) {
+					Line.Append(X);
+					if (!IsFirst)
+					{
+						Arguments.Append(X);
+					}
+				};
 
-		// The object_key is the unique identifier for the object in the bucket.
-		Aws::String object_key = "my-object";
+				if (!IsFirst)
+				{
+					AppendAscii(" ");
+				}
 
-		// Create a globally unique name for the new bucket.
-		// Format: "my-bucket-" + lowercase UUID.
-		Aws::String uuid		= Aws::Utils::UUID::RandomUUID();
-		Aws::String bucket_name = "my-bucket-" + Aws::Utils::StringUtils::ToLower(uuid.c_str());
+				std::string_view ThisArg(argv[j]);
+				PassthroughArgV.push_back(std::string(ThisArg));
 
-		const double   throughput_target_gbps = 5;
-		const uint64_t part_size			  = 8 * 1024 * 1024;  // 8 MB.
+				const bool NeedsQuotes = (ThisArg.find(' ') != std::string_view::npos);
 
-		Aws::S3Crt::ClientConfiguration config;
-		config.region				= region;
-		config.throughputTargetGbps = throughput_target_gbps;
-		config.partSize				= part_size;
+				if (NeedsQuotes)
+				{
+					AppendAscii("\"");
+				}
 
-		Aws::S3Crt::S3CrtClient s3_crt_client(config);
+				AppendAscii(ThisArg);
 
-		// Use BucketLocationConstraintMapper to get the BucketLocationConstraint enum from the region string.
-		// https://sdk.amazonaws.com/cpp/api/0.14.3/namespace_aws_1_1_s3_1_1_model_1_1_bucket_location_constraint_mapper.html#a50d4503d3f481022f969eff1085cfbb0
-		Aws::S3Crt::Model::BucketLocationConstraint locConstraint =
-			Aws::S3Crt::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(region);
+				if (NeedsQuotes)
+				{
+					AppendAscii("\"");
+				}
+
+				IsFirst = false;
+			}
+
+			Passthrough		= Line.c_str();
+			PassthroughArgs = Arguments.c_str();
+
+			// This will "truncate" the arg vector and terminate the loop
+			argc = i;
+		}
+	}
 
-		// ListBuckets(s3_crt_client, bucket_name);
+	ZenCloudOptions	 GlobalOptions;
+	cxxopts::Options Options("zencloud", "Zen cloud interface");
 
-		// CreateBucket(s3_crt_client, bucket_name, locConstraint);
+	Options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(GlobalOptions.IsDebug));
+	Options.add_options()("v, verbose", "Enable verbose logging", cxxopts::value<bool>(GlobalOptions.IsVerbose));
+	Options.add_options()("help", "Show command line help");
+	Options.add_options()("local", "Use local server (such as minio)", cxxopts::value<bool>(GlobalOptions.IsLocal));
+	Options.add_options()("t, test", "Run tests", cxxopts::value<bool>(GlobalOptions.IsTest));
+	Options.add_options()("data", "Test data path", cxxopts::value<std::string>(GlobalOptions.TestDataDirectory));
 
-		// PutObject(s3_crt_client, bucket_name, object_key, file_name);
+	const bool IsNullInvoke = (argc == 1);	// If no arguments are passed we want to print usage information
 
-		// GetObject(s3_crt_client, bucket_name, object_key);
+	try
+	{
+		cxxopts::ParseResult ParseResult = Options.parse(argc, argv);
+
+		if (ParseResult.count("help") || IsNullInvoke == 1)
+		{
+			std::string Help = Options.help();
+
+			printf("%s\n", Help.c_str());
 
-		// DeleteObject(s3_crt_client, bucket_name, object_key);
+			exit(0);
+		}
 
-		// DeleteBucket(s3_crt_client, bucket_name);
+		if (GlobalOptions.IsDebug)
+		{
+			logging::SetLogLevel(logging::level::Debug);
+		}
+
+		if (GlobalOptions.IsVerbose)
+		{
+			logging::SetLogLevel(logging::level::Trace);
+		}
+
+		if (GlobalOptions.IsTest)
+		{
+			DoWork(GlobalOptions);
+		}
 	}
-	Aws::ShutdownAPI(options);
+	catch (const OptionParseException& Ex)
+	{
+		std::string HelpMessage = Options.help();
+
+		printf("Error parsing program arguments: %s\n\n%s", Ex.what(), HelpMessage.c_str());
+
+		return 9;
+	}
+	catch (const std::system_error& Ex)
+	{
+		printf("System Error: %s\n", Ex.what());
+
+		return Ex.code() ? Ex.code().value() : 10;
+	}
+	catch (const std::exception& Ex)
+	{
+		printf("Error: %s\n", Ex.what());
+
+		return 11;
+	}
+
 	return 0;
 }