builds download url (#419)

* RemoveQuotes helper
* `--url` option for `zen builds` command has been reworked to accept a "Cloud Artifact URL", removing the need to specify "host", "namespace" and "bucket" separately

4 files changed, 154 insertions, 64 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 372291fcc..1c1193a3f 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -37,6 +37,7 @@
 
 #include <signal.h>
 #include <memory>
+#include <regex>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_map.h>
@@ -9434,12 +9435,7 @@ BuildsCommand::BuildsCommand()
 	auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) {
 		AddAuthOptions(Ops);
 		Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>");
-		Ops.add_option("cloud build",
-					   "",
-					   "url",
-					   "Cloud Builds host url (legacy - use --override-host)",
-					   cxxopts::value(m_OverrideHost),
-					   "<url>");
+		Ops.add_option("cloud build", "", "url", "Cloud Artifact URL", cxxopts::value(m_Url), "<url>");
 		Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>");
 		Ops.add_option("cloud build",
 					   "",
@@ -9829,6 +9825,55 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	ParseSystemOptions();
 
 	auto ParseStorageOptions = [&](bool RequireBucket) {
+		m_Host		   = RemoveQuotes(m_Host);
+		m_OverrideHost = RemoveQuotes(m_OverrideHost);
+		m_Url		   = RemoveQuotes(m_Url);
+		m_Namespace	   = RemoveQuotes(m_Url);
+		m_Bucket	   = RemoveQuotes(m_Bucket);
+		if (!m_Url.empty())
+		{
+			if (!m_Host.empty())
+			{
+				throw zen::OptionParseException(fmt::format("host is not compatible with the url option\n{}", m_Options.help()));
+			}
+			if (!m_Bucket.empty())
+			{
+				throw zen::OptionParseException(fmt::format("bucket is not compatible with the url option\n{}", m_Options.help()));
+			}
+			if (!m_BuildId.empty())
+			{
+				throw zen::OptionParseException(fmt::format("buildid is not compatible with the url option\n{}", m_Options.help()));
+			}
+			const std::string ArtifactURLRegExString = R"((.*?:\/\/.*?)\/api\/v2\/builds\/(.*?)\/(.*?)\/(.*))";
+			const std::regex  ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript);
+			std::match_results<std::string_view::const_iterator> MatchResults;
+			const std::string_view								 Url(RemoveQuotes(m_Url));
+			if (regex_match(begin(Url), end(Url), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5)
+			{
+				auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view {
+					ZEN_ASSERT(Index < MatchResults.size());
+
+					const auto& Match = MatchResults[Index];
+
+					return std::string_view(&*Match.first, Match.second - Match.first);
+				};
+
+				const std::string_view Host		 = GetMatch(1);
+				const std::string_view Namespace = GetMatch(2);
+				const std::string_view Bucket	 = GetMatch(3);
+				const std::string_view BuildId	 = GetMatch(4);
+
+				m_Host		= Host;
+				m_Namespace = Namespace;
+				m_Bucket	= Bucket;
+				m_BuildId	= BuildId;
+			}
+			else
+			{
+				throw zen::OptionParseException(fmt::format("url does not match the Cloud Artifact URL format\n{}", m_Options.help()));
+			}
+		}
+
 		if (!m_OverrideHost.empty() || !m_Host.empty())
 		{
 			if (!m_StoragePath.empty())
@@ -9890,6 +9935,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 
 	auto ParseAuthOptions = [&]() {
+		m_OpenIdProviderUrl = RemoveQuotes(m_OpenIdProviderUrl);
+		m_OpenIdClientId	= RemoveQuotes(m_OpenIdClientId);
+
+		m_AccessToken	= RemoveQuotes(m_AccessToken);
+		m_EncryptionKey = RemoveQuotes(m_EncryptionKey);
+		m_EncryptionIV	= RemoveQuotes(m_EncryptionIV);
+
+		m_OAuthUrl			= RemoveQuotes(m_OAuthUrl);
+		m_OAuthClientId		= RemoveQuotes(m_OAuthClientId);
+		m_OAuthClientSecret = RemoveQuotes(m_OAuthClientSecret);
+
+		m_OidcTokenAuthExecutablePath = RemoveQuotes(m_OidcTokenAuthExecutablePath);
+
 		if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty())
 		{
 			CreateAuthMgr();
@@ -9993,6 +10051,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 								  bool							 RequireBucket) -> StorageInstance {
 		ParseStorageOptions(RequireBucket);
 
+		m_ZenCacheHost = RemoveQuotes(m_ZenCacheHost);
+
 		StorageInstance Result;
 
 		std::string BuildStorageName = ZEN_CLOUD_STORAGE;
@@ -10259,6 +10319,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 
 	auto ParseBlobHash = [&]() -> IoHash {
+		m_BlobHash = RemoveQuotes(m_BlobHash);
 		if (m_BlobHash.empty())
 		{
 			throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help()));
@@ -10274,6 +10335,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 
 	auto ParseBuildId = [&]() -> Oid {
+		m_BuildId = RemoveQuotes(m_BuildId);
 		if (m_BuildId.length() != Oid::StringLength)
 		{
 			throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help()));
@@ -10289,6 +10351,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 
 	auto ParseBuildPartId = [&]() -> Oid {
+		m_BuildPartId = RemoveQuotes(m_BuildPartId);
 		if (m_BuildPartId.length() != Oid::StringLength)
 		{
 			throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help()));
@@ -10307,7 +10370,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		std::vector<Oid> BuildPartIds;
 		for (const std::string& BuildPartId : m_BuildPartIds)
 		{
-			BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId));
+			BuildPartIds.push_back(Oid::TryFromHexString(RemoveQuotes(BuildPartId)));
 			if (BuildPartIds.back() == Oid::Zero)
 			{
 				throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help()));
@@ -10316,6 +10379,20 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		return BuildPartIds;
 	};
 
+	auto ParseBuildPartNames = [&]() -> std::vector<std::string> {
+		std::vector<std::string> BuildPartNames;
+		for (const std::string& BuildPartName : m_BuildPartNames)
+		{
+			BuildPartNames.push_back(std::string(RemoveQuotes(BuildPartName)));
+			if (BuildPartNames.back().empty())
+			{
+				throw zen::OptionParseException(
+					fmt::format("build-part-names '{}' is invalid\n{}", BuildPartName, m_DownloadOptions.help()));
+			}
+		}
+		return BuildPartNames;
+	};
+
 	auto ParseBuildMetadata = [&]() -> CbObject {
 		if (m_CreateBuild)
 		{
@@ -10342,6 +10419,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				}
 				return MetaData;
 			}
+			m_BuildMetadata = RemoveQuotes(m_BuildMetadata);
 			if (!m_BuildMetadata.empty())
 			{
 				CbObjectWriter MetaDataWriter(1024);
@@ -10520,22 +10598,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			ParsePath();
 
-			if (m_BuildPartName.empty())
-			{
-				m_BuildPartName = m_Path.filename().string();
-			}
-
-			const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId();
-			if (m_BuildId.empty())
-			{
-				m_BuildId = BuildId.ToString();
-			}
-			const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId();
-			if (m_BuildPartId.empty())
-			{
-				m_BuildPartId = BuildPartId.ToString();
-			}
-
 			BuildStorage::Statistics	  StorageStats;
 			BuildStorageCache::Statistics StorageCacheStats;
 
@@ -10557,10 +10619,28 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			StorageInstance Storage =
 				CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true);
 
+			if (m_BuildPartName.empty())
+			{
+				m_BuildPartName = m_Path.filename().string();
+			}
+
+			const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId();
+			if (m_BuildId.empty())
+			{
+				m_BuildId = BuildId.ToString();
+			}
+			const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId();
+			if (m_BuildPartId.empty())
+			{
+				m_BuildPartId = BuildPartId.ToString();
+			}
+
 			CbObject MetaData = ParseBuildMetadata();
 
 			const std::filesystem::path TempDir = UploadTempDirectory(m_Path);
 
+			m_ManifestPath = RemoveQuotes(m_ManifestPath);
+
 			UploadFolder(Storage,
 						 BuildId,
 						 BuildPartId,
@@ -10605,6 +10685,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			ParsePath();
 
+			if (m_ZenFolderPath.empty())
+			{
+				m_ZenFolderPath = m_Path / ZenFolderName;
+			}
+			MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			StorageInstance Storage =
+				CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true);
+
 			const Oid BuildId = ParseBuildId();
 
 			if (m_PostDownloadVerify && m_PrimeCacheOnly)
@@ -10623,24 +10715,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled");
 			}
 
-			std::vector<Oid> BuildPartIds = ParseBuildPartIds();
-
-			if (m_ZenFolderPath.empty())
-			{
-				m_ZenFolderPath = m_Path / ZenFolderName;
-			}
-			MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
-
-			BuildStorage::Statistics	  StorageStats;
-			BuildStorageCache::Statistics StorageCacheStats;
-
-			StorageInstance Storage =
-				CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true);
+			std::vector<Oid>		 BuildPartIds	= ParseBuildPartIds();
+			std::vector<std::string> BuildPartNames = ParseBuildPartNames();
 
 			DownloadFolder(Storage,
 						   BuildId,
 						   BuildPartIds,
-						   m_BuildPartNames,
+						   BuildPartNames,
 						   m_Path,
 						   m_ZenFolderPath,
 						   m_SystemRootDir,
@@ -10665,10 +10746,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		if (SubOption == &m_FetchBlobOptions)
 		{
 			ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
-			IoHash BlobHash = ParseBlobHash();
-
-			const Oid BuildId = Oid::FromHexString(m_BuildId);
-
 			BuildStorage::Statistics	  StorageStats;
 			BuildStorageCache::Statistics StorageCacheStats;
 
@@ -10690,6 +10767,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			StorageInstance Storage =
 				CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true);
 
+			IoHash BlobHash = ParseBlobHash();
+
+			const Oid BuildId = Oid::FromHexString(m_BuildId);
+
 			uint64_t CompressedSize;
 			uint64_t DecompressedSize;
 			ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
@@ -10708,13 +10789,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		{
 			ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
 
-			Oid BuildId = ParseBuildId();
-
-			if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
-			{
-				throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help()));
-			}
-
 			BuildStorage::Statistics	  StorageStats;
 			BuildStorageCache::Statistics StorageCacheStats;
 
@@ -10736,6 +10810,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			StorageInstance Storage =
 				CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true);
 
+			Oid BuildId = ParseBuildId();
+
+			if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
+			{
+				throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help()));
+			}
+
 			const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId();
 
 			ValidateStatistics ValidateStats;
@@ -10769,7 +10850,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			Stopwatch Timer;
 			for (const std::string& BuildIdString : m_BuildIds)
 			{
-				Oid BuildId = Oid::FromHexString(BuildIdString);
+				Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString));
 				if (BuildId == Oid::Zero)
 				{
 					throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help()));
@@ -10807,14 +10888,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			ParsePath();
 
-			m_BuildId		= Oid::NewOid().ToString();
-			m_BuildPartName = m_Path.filename().string();
-			m_BuildPartId	= Oid::NewOid().ToString();
-			m_CreateBuild	= true;
-
-			const Oid BuildId	  = Oid::FromHexString(m_BuildId);
-			const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
-
 			if (m_OverrideHost.empty() && m_StoragePath.empty())
 			{
 				m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
@@ -10852,6 +10925,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			StorageInstance Storage =
 				CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true);
 
+			m_BuildId		= Oid::NewOid().ToString();
+			m_BuildPartName = m_Path.filename().string();
+			m_BuildPartId	= Oid::NewOid().ToString();
+			m_CreateBuild	= true;
+
+			const Oid BuildId	  = Oid::FromHexString(m_BuildId);
+			const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
+
 			auto MakeMetaData = [](const Oid& BuildId) -> CbObject {
 				CbObjectWriter BuildMetaDataWriter;
 				{
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 41ed65105..378810155 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -38,6 +38,7 @@ private:
 	// cloud builds
 	std::string m_OverrideHost;
 	std::string m_Host;
+	std::string m_Url;
 	bool		m_AssumeHttp2	= false;
 	bool		m_AllowRedirect = false;
 	std::string m_Namespace;
@@ -88,7 +89,6 @@ private:
 	std::string m_Verb;	 // list, upload, download
 
 	cxxopts::Options m_ListNamespacesOptions{"list-namespaces", "List available build namespaces"};
-	std::string		 m_ListNamespacesResultPath;
 	bool			 m_ListNamespacesRecursive = false;
 
 	cxxopts::Options	  m_ListOptions{"list", "List available builds"};
diff --git a/src/zenutil/commandlineoptions.cpp b/src/zenutil/commandlineoptions.cpp
index 0dffa42f0..afef7f6f2 100644
--- a/src/zenutil/commandlineoptions.cpp
+++ b/src/zenutil/commandlineoptions.cpp
@@ -157,12 +157,7 @@ MakeSafeAbsolutePath(const std::filesystem::path& Path)
 std::filesystem::path
 StringToPath(const std::string_view& Path)
 {
-	std::string_view UnquotedPath = Path;
-
-	if (UnquotedPath.length() > 2 && UnquotedPath.front() == '\"' && UnquotedPath.back() == '\"')
-	{
-		UnquotedPath = UnquotedPath.substr(1, UnquotedPath.length() - 2);
-	}
+	std::string_view UnquotedPath = RemoveQuotes(Path);
 
 	if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator))
 	{
@@ -172,6 +167,19 @@ StringToPath(const std::string_view& Path)
 	return std::filesystem::path(UnquotedPath).make_preferred();
 }
 
+std::string_view
+RemoveQuotes(const std::string_view& Arg)
+{
+	if (Arg.length() > 2)
+	{
+		if (Arg[0] == '"' && Arg[Arg.length() - 1] == '"')
+		{
+			return Arg.substr(1, Arg.length() - 2);
+		}
+	}
+	return Arg;
+}
+
 #if ZEN_WITH_TESTS
 
 void
diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h
index b7581f6cd..f927d41e5 100644
--- a/src/zenutil/include/zenutil/commandlineoptions.h
+++ b/src/zenutil/include/zenutil/commandlineoptions.h
@@ -22,6 +22,7 @@ std::vector<char*>					StripCommandlineQuotes(std::vector<std::string>& InOutArg
 void								MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path);
 [[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path);
 std::filesystem::path				StringToPath(const std::string_view& Path);
+std::string_view					RemoveQuotes(const std::string_view& Arg);
 
 void commandlineoptions_forcelink();  // internal