path: root/src/zencloud/zencloudmain.cpp

// Copyright Epic Games, Inc. All Rights Reserved.

#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/logging.h>
#include <zencore/sha1.h>

#if ZEN_WITH_TESTS
#	define ZEN_TEST_WITH_RUNNER 1
#	include <zencore/testing.h>
#endif

#if ZEN_USE_MIMALLOC
#	include <mimalloc-new-delete.h>
#endif

ZEN_THIRD_PARTY_INCLUDES_START
#include <spdlog/sinks/ansicolor_sink.h>
#include <spdlog/spdlog.h>
ZEN_THIRD_PARTY_INCLUDES_END

ZEN_THIRD_PARTY_INCLUDES_START
#include <cxxopts.hpp>
ZEN_THIRD_PARTY_INCLUDES_END

ZEN_THIRD_PARTY_INCLUDES_START
#include <aws/core/Aws.h>
#include <aws/core/utils/HashingUtils.h>
#include <aws/core/utils/UUID.h>
#include <aws/core/utils/logging/CRTLogSystem.h>
#include <aws/core/utils/memory/stl/AWSStringStream.h>
#include <aws/s3-crt/S3CrtClient.h>
#include <aws/s3-crt/model/BucketLocationConstraint.h>
#include <aws/s3-crt/model/CreateBucketRequest.h>
#include <aws/s3-crt/model/DeleteBucketRequest.h>
#include <aws/s3-crt/model/DeleteObjectRequest.h>
#include <aws/s3-crt/model/GetObjectRequest.h>
#include <aws/s3-crt/model/PutObjectRequest.h>
ZEN_THIRD_PARTY_INCLUDES_END

#include <cstdlib>
#include <fstream>
#include <iostream>

static const char ALLOCATION_TAG[] = "s3-crt-demo";

// List all Amazon Simple Storage Service (Amazon S3) buckets under the account.
bool
ListBuckets(const Aws::S3Crt::S3CrtClient& s3CrtClient)
{
	Aws::S3Crt::Model::ListBucketsOutcome outcome = s3CrtClient.ListBuckets();

	if (outcome.IsSuccess())
	{
		std::cout << "All buckets under my account:" << std::endl;

		for (auto const& bucket : outcome.GetResult().GetBuckets())
		{
			std::cout << "  * " << bucket.GetName() << std::endl;
		}
		std::cout << std::endl;

		return true;
	}
	else
	{
		std::cout << "ListBuckets error:\n" << outcome.GetError() << std::endl << std::endl;

		return false;
	}
}

// Create an Amazon Simple Storage Service (Amazon S3) bucket.
bool
CreateBucket(const Aws::S3Crt::S3CrtClient&						s3CrtClient,
			 const Aws::String&									bucketName,
			 const Aws::S3Crt::Model::BucketLocationConstraint& locConstraint)
{
	std::cout << "Creating bucket: \"" << bucketName << "\" ..." << std::endl;

	Aws::S3Crt::Model::CreateBucketRequest request;
	request.SetBucket(bucketName);

	//  If you don't specify an AWS Region, the bucket is created in the US East (N. Virginia) Region (us-east-1)
	if (locConstraint != Aws::S3Crt::Model::BucketLocationConstraint::us_east_1)
	{
		Aws::S3Crt::Model::CreateBucketConfiguration bucket_config;
		bucket_config.SetLocationConstraint(locConstraint);

		request.SetCreateBucketConfiguration(bucket_config);
	}

	Aws::S3Crt::Model::CreateBucketOutcome outcome = s3CrtClient.CreateBucket(request);

	if (outcome.IsSuccess())
	{
		std::cout << "Bucket created." << std::endl << std::endl;

		return true;
	}
	else
	{
		std::cout << "CreateBucket error:\n" << outcome.GetError() << std::endl << std::endl;

		return false;
	}
}

// Delete an existing Amazon S3 bucket.
bool
DeleteBucket(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName)
{
	std::cout << "Deleting bucket: \"" << bucketName << "\" ..." << std::endl;

	Aws::S3Crt::Model::DeleteBucketRequest request;
	request.SetBucket(bucketName);

	Aws::S3Crt::Model::DeleteBucketOutcome outcome = s3CrtClient.DeleteBucket(request);

	if (outcome.IsSuccess())
	{
		std::cout << "Bucket deleted." << std::endl << std::endl;

		return true;
	}
	else
	{
		std::cout << "DeleteBucket error:\n" << outcome.GetError() << std::endl << std::endl;

		return false;
	}
}

// Put an Amazon S3 object to the bucket.
bool
PutObject(const Aws::S3Crt::S3CrtClient& s3CrtClient,
		  const Aws::String&			 bucketName,
		  const Aws::String&			 objectKey,
		  const Aws::String&			 fileName)
{
	std::cout << "Putting object: \"" << objectKey << "\" to bucket: \"" << bucketName << "\" ..." << std::endl;

	Aws::S3Crt::Model::PutObjectRequest request;

	std::shared_ptr<Aws::IOStream> bodyStream =
		Aws::MakeShared<Aws::FStream>(ALLOCATION_TAG, fileName.c_str(), std::ios_base::in | std::ios_base::binary);
	if (!bodyStream->good())
	{
		std::cout << "Failed to open file: \"" << fileName << "\"." << std::endl << std::endl;
		return false;
	}

	request.SetBucket(bucketName);
	request.SetKey(objectKey);
	request.SetBody(bodyStream);

	// A PUT operation turns into a multipart upload using the s3-crt client.
	// https://github.com/aws/aws-sdk-cpp/wiki/Improving-S3-Throughput-with-AWS-SDK-for-CPP-v1.9
	Aws::S3Crt::Model::PutObjectOutcome outcome = s3CrtClient.PutObject(request);

	if (outcome.IsSuccess())
	{
		std::cout << "Object added." << std::endl << std::endl;

		return true;
	}
	else
	{
		std::cout << "PutObject error:\n" << outcome.GetError() << std::endl << std::endl;

		return false;
	}
}

// Get the Amazon S3 object from the bucket.
bool
GetObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::String& objectKey)
{
	std::cout << "Getting object: \"" << objectKey << "\" from bucket: \"" << bucketName << "\" ..." << std::endl;

	Aws::S3Crt::Model::GetObjectRequest request;
	request.SetBucket(bucketName);
	request.SetKey(objectKey);
	Aws::S3Crt::Model::GetObjectOutcome outcome = s3CrtClient.GetObject(request);

	if (outcome.IsSuccess())
	{
		// Uncomment this line if you wish to have the contents of the file displayed. Not recommended for large files
		//  because it takes a while.
		//  std::cout << "Object content: " << outcome.GetResult().GetBody().rdbuf() << std::endl << std::endl;

		return true;
	}
	else
	{
		std::cout << "GetObject error:\n" << outcome.GetError() << std::endl << std::endl;

		return false;
	}
}

// Delete the Amazon S3 object from the bucket.
bool
DeleteObject(const Aws::S3Crt::S3CrtClient& s3CrtClient, const Aws::String& bucketName, const Aws::String& objectKey)
{
	std::cout << "Deleting object: \"" << objectKey << "\" from bucket: \"" << bucketName << "\" ..." << std::endl;

	Aws::S3Crt::Model::DeleteObjectRequest request;
	request.SetBucket(bucketName);
	request.SetKey(objectKey);

	Aws::S3Crt::Model::DeleteObjectOutcome outcome = s3CrtClient.DeleteObject(request);

	if (outcome.IsSuccess())
	{
		std::cout << "Object deleted." << std::endl << std::endl;

		return true;
	}
	else
	{
		std::cout << "DeleteObject error:\n" << outcome.GetError() << std::endl << std::endl;

		return false;
	}
}

//////////////////////////////////////////////////////////////////////////
// TODO: should make this Unicode-aware so we can pass anything in on the
// command line.

struct ZenCloudOptions
{
	bool		IsDebug	  = false;
	bool		IsVerbose = false;
	bool		IsLocal	  = false;
	bool		IsTest	  = false;
	std::string TestDataDirectory;

	// Arguments after " -- " on command line are passed through and not parsed
	std::string				 PassthroughCommandLine;
	std::string				 PassthroughArgs;
	std::vector<std::string> PassthroughArgV;
};

int
DoWork(ZenCloudOptions& GlobalOptions)
{
	using namespace std::literals;
	using namespace zen;

	// AWS SDK setup

	Aws::SDKOptions options;
	options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Info;

	Aws::InitAPI(options);
	{
		// TODO: Set to your account AWS Region.
		Aws::String region = Aws::Region::EU_NORTH_1;  // US_EAST_1;

		// The object_key is the unique identifier for the object in the bucket.
		Aws::String object_key = "my-object";

		// Create a globally unique name for the new bucket.
		// Format: "my-bucket-" + lowercase UUID.
		Aws::String uuid		= Aws::Utils::UUID::RandomUUID();
		Aws::String bucket_name = "my-bucket-" + Aws::Utils::StringUtils::ToLower(uuid.c_str());

		const double   throughput_target_gbps = 10;
		const uint64_t part_size			  = 32 * 1024 * 1024;

		Aws::S3Crt::ClientConfiguration config;
		config.region				= region;
		config.throughputTargetGbps = throughput_target_gbps;
		config.partSize				= part_size;
		//		config.httpLibOverride		= Aws::Http::TransferLibType::CURL_CLIENT;

		std::vector<std::string_view> Tokens;

		const char* AwsAccessEnv = std::getenv("ZEN_AWS_ACCESS");

		if (AwsAccessEnv && !GlobalOptions.IsLocal)
		{
			zen::ForEachStrTok(AwsAccessEnv, ':', [&](const std::string_view& Token) {
				Tokens.push_back(Token);
				return true;
			});
		}
		else
		{
			Tokens.push_back("zencloud-test"sv);
			Tokens.push_back("misterblobby"sv);

			config.endpointOverride = "http://127.0.0.1:9000";
		}

		ZEN_ASSERT(Tokens.size() == 2);

		Aws::Auth::AWSCredentials credentials;
		credentials.SetAWSAccessKeyId(Aws::String(Tokens[0]));
		credentials.SetAWSSecretKey(Aws::String(Tokens[1]));

		ZEN_CONSOLE("using credentials: {}/{}", Tokens[0], Tokens[1]);

		Aws::S3Crt::S3CrtClient s3_crt_client(credentials, config);

		// Use BucketLocationConstraintMapper to get the BucketLocationConstraint enum from the region string.
		// https://sdk.amazonaws.com/cpp/api/0.14.3/namespace_aws_1_1_s3_1_1_model_1_1_bucket_location_constraint_mapper.html#a50d4503d3f481022f969eff1085cfbb0
		Aws::S3Crt::Model::BucketLocationConstraint locConstraint =
			Aws::S3Crt::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(region);

		ListBuckets(s3_crt_client);
		CreateBucket(s3_crt_client, bucket_name, locConstraint);

		Aws::String file_name = GlobalOptions.TestDataDirectory;

		PutObject(s3_crt_client, bucket_name, object_key, file_name);
		GetObject(s3_crt_client, bucket_name, object_key);
		DeleteObject(s3_crt_client, bucket_name, object_key);
		DeleteBucket(s3_crt_client, bucket_name);
	}
	Aws::ShutdownAPI(options);
	return 0;
}

int
main(int argc, char** argv)
{
	using namespace zen;
	using namespace std::literals;

	ZEN_UNUSED(argc, argv);

#if ZEN_USE_MIMALLOC
	mi_version();
#endif

	zen::logging::InitializeLogging();

	// Set output mode to handle virtual terminal sequences
	zen::logging::EnableVTMode();
	std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); });

	LoggerRef DefaultLogger = zen::logging::Default();
	auto&	  Sinks			= DefaultLogger.SpdLogger->sinks();

	Sinks.clear();
	auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>();
	Sinks.push_back(ConsoleSink);

	zen::MaximizeOpenFileCount();

	// Split command line into options, commands and any pass-through arguments

	std::string				 Passthrough;
	std::string				 PassthroughArgs;
	std::vector<std::string> PassthroughArgV;

	for (int i = 1; i < argc; ++i)
	{
		if ("--"sv == argv[i])
		{
			bool						 IsFirst = true;
			ExtendableStringBuilder<256> Line;
			ExtendableStringBuilder<256> Arguments;

			for (int j = i + 1; j < argc; ++j)
			{
				auto AppendAscii = [&](auto X) {
					Line.Append(X);
					if (!IsFirst)
					{
						Arguments.Append(X);
					}
				};

				if (!IsFirst)
				{
					AppendAscii(" ");
				}

				std::string_view ThisArg(argv[j]);
				PassthroughArgV.push_back(std::string(ThisArg));

				const bool NeedsQuotes = (ThisArg.find(' ') != std::string_view::npos);

				if (NeedsQuotes)
				{
					AppendAscii("\"");
				}

				AppendAscii(ThisArg);

				if (NeedsQuotes)
				{
					AppendAscii("\"");
				}

				IsFirst = false;
			}

			Passthrough		= Line.c_str();
			PassthroughArgs = Arguments.c_str();

			// This will "truncate" the arg vector and terminate the loop
			argc = i;
		}
	}

	ZenCloudOptions	 GlobalOptions;
	cxxopts::Options Options("zencloud", "Zen cloud interface");

	Options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(GlobalOptions.IsDebug));
	Options.add_options()("v, verbose", "Enable verbose logging", cxxopts::value<bool>(GlobalOptions.IsVerbose));
	Options.add_options()("help", "Show command line help");
	Options.add_options()("local", "Use local server (such as minio)", cxxopts::value<bool>(GlobalOptions.IsLocal));
	Options.add_options()("t, test", "Run tests", cxxopts::value<bool>(GlobalOptions.IsTest));
	Options.add_options()("data", "Test data path", cxxopts::value<std::string>(GlobalOptions.TestDataDirectory));

	const bool IsNullInvoke = (argc == 1);	// If no arguments are passed we want to print usage information

	try
	{
		cxxopts::ParseResult ParseResult = Options.parse(argc, argv);

		if (ParseResult.count("help") || IsNullInvoke == 1)
		{
			std::string Help = Options.help();

			printf("%s\n", Help.c_str());

			exit(0);
		}

		if (GlobalOptions.IsDebug)
		{
			logging::SetLogLevel(logging::level::Debug);
		}

		if (GlobalOptions.IsVerbose)
		{
			logging::SetLogLevel(logging::level::Trace);
		}

		if (GlobalOptions.IsTest)
		{
			DoWork(GlobalOptions);
		}
	}
	catch (const OptionParseException& Ex)
	{
		std::string HelpMessage = Options.help();

		printf("Error parsing program arguments: %s\n\n%s", Ex.what(), HelpMessage.c_str());

		return 9;
	}
	catch (const std::system_error& Ex)
	{
		printf("System Error: %s\n", Ex.what());

		return Ex.code() ? Ex.code().value() : 10;
	}
	catch (const std::exception& Ex)
	{
		printf("Error: %s\n", Ex.what());

		return 11;
	}

	return 0;
}