Merge pull request #360 from ue-foundation/zs/zencli-list-namespaces-buckets

Zs/zencli list namespaces buckets

7 files changed, 176 insertions, 4 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index f50f6205b..15c635594 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -8876,12 +8876,34 @@ BuildsCommand::BuildsCommand()
 	m_Options.add_option("",
 						 "v",
 						 "verb",
-						 "Verb for build - list, upload, download, diff, fetch-blob, validate-part",
+						 "Verb for build - list-namespaces, list, upload, download, diff, fetch-blob, validate-part",
 						 cxxopts::value(m_Verb),
 						 "<verb>");
 	m_Options.parse_positional({"verb"});
 	m_Options.positional_help("verb");
 
+	// list-namespaces
+	AddSystemOptions(m_ListNamespacesOptions);
+	AddCloudOptions(m_ListNamespacesOptions);
+	AddFileOptions(m_ListNamespacesOptions);
+	AddOutputOptions(m_ListNamespacesOptions);
+	AddZenFolderOptions(m_ListNamespacesOptions);
+	m_ListNamespacesOptions.add_options()("h,help", "Print help");
+	m_ListNamespacesOptions.add_option("",
+									   "",
+									   "recursive",
+									   "Enable fetch of buckets within namespaces also",
+									   cxxopts::value(m_ListNamespacesRecursive),
+									   "<recursive>");
+	m_ListNamespacesOptions.add_option("",
+									   "",
+									   "result-path",
+									   "Path to json or compactbinary to write query result to",
+									   cxxopts::value(m_ListResultPath),
+									   "<result-path>");
+	m_ListNamespacesOptions.parse_positional({"result-path"});
+	m_ListNamespacesOptions.positional_help("result-path");
+
 	// list
 	AddSystemOptions(m_ListOptions);
 	AddCloudOptions(m_ListOptions);
@@ -9181,7 +9203,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			{
 				throw zen::OptionParseException(fmt::format("url is not compatible with the storage-path option\n{}", m_Options.help()));
 			}
-			if (m_Namespace.empty() || m_Bucket.empty())
+			if (SubOption != &m_ListNamespacesOptions && (m_Namespace.empty() || m_Bucket.empty()))
 			{
 				throw zen::OptionParseException(
 					fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help()));
@@ -9702,6 +9724,57 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	try
 	{
+		if (SubOption == &m_ListNamespacesOptions)
+		{
+			if (!m_ListResultPath.empty())
+			{
+				ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
+			}
+
+			BuildStorage::Statistics	  StorageStats;
+			BuildStorageCache::Statistics StorageCacheStats;
+
+			const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
+															? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
+															: MakeSafeAbsolutePath(m_ZenFolderPath);
+			CreateDirectories(ZenFolderPath);
+			auto _ = MakeGuard([ZenFolderPath]() {
+				if (CleanDirectory(ZenFolderPath, {}))
+				{
+					std::error_code DummyEc;
+					RemoveDir(ZenFolderPath, DummyEc);
+				}
+			});
+
+			StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+
+			CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive);
+			ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None);
+			if (m_ListResultPath.empty())
+			{
+				ExtendableStringBuilder<1024> SB;
+				CompactBinaryToJson(Response.GetView(), SB);
+				ZEN_CONSOLE("{}", SB.ToView());
+			}
+			else
+			{
+				std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath);
+				if (ToLower(ListResultPath.extension().string()) == ".cbo")
+				{
+					MemoryView ResponseView = Response.GetView();
+					WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
+				}
+				else
+				{
+					ExtendableStringBuilder<1024> SB;
+					CompactBinaryToJson(Response.GetView(), SB);
+					WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+				}
+			}
+
+			return 0;
+		}
+
 		if (SubOption == &m_ListOptions)
 		{
 			MakeSafeAbsolutePathÍnPlace(m_ListQueryPath);
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index cd2ae1f08..01d510e1b 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -84,6 +84,10 @@ private:
 
 	std::string m_Verb;	 // list, upload, download
 
+	cxxopts::Options m_ListNamespacesOptions{"list-namespaces", "List available build namespaces"};
+	std::string		 m_ListNamespacesResultPath;
+	bool			 m_ListNamespacesRecursive = false;
+
 	cxxopts::Options	  m_ListOptions{"list", "List available builds"};
 	std::filesystem::path m_ListQueryPath;
 	std::filesystem::path m_ListResultPath;
@@ -114,7 +118,8 @@ private:
 	cxxopts::Options		 m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"};
 	std::vector<std::string> m_BuildIds;
 
-	cxxopts::Options* m_SubCommands[8] = {&m_ListOptions,
+	cxxopts::Options* m_SubCommands[9] = {&m_ListNamespacesOptions,
+										  &m_ListOptions,
 										  &m_UploadOptions,
 										  &m_DownloadOptions,
 										  &m_DiffOptions,
diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
index f335a03a3..badfb4840 100644
--- a/src/zenutil/filebuildstorage.cpp
+++ b/src/zenutil/filebuildstorage.cpp
@@ -35,6 +35,27 @@ public:
 
 	virtual ~FileBuildStorage() {}
 
+	virtual CbObject ListNamespaces(bool bRecursive) override
+	{
+		ZEN_TRACE_CPU("FileBuildStorage::ListNamespaces");
+		ZEN_UNUSED(bRecursive);
+
+		SimulateLatency(0, 0);
+
+		Stopwatch ExecutionTimer;
+		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		m_Stats.TotalRequestCount++;
+
+		CbObjectWriter Writer;
+		Writer.BeginArray("results");
+		{
+		}
+		Writer.EndArray();	// results
+		Writer.Save();
+		SimulateLatency(Writer.GetSaveSize(), 0);
+		return Writer.Save();
+	}
+
 	virtual CbObject ListBuilds(CbObject Query) override
 	{
 		ZEN_TRACE_CPU("FileBuildStorage::ListBuilds");
@@ -66,7 +87,7 @@ public:
 				}
 			}
 		}
-		Writer.EndArray();	// builds
+		Writer.EndArray();	// results
 		Writer.Save();
 		SimulateLatency(Writer.GetSaveSize(), 0);
 		return Writer.Save();
diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h
index 05e3ca22d..5422c837c 100644
--- a/src/zenutil/include/zenutil/buildstorage.h
+++ b/src/zenutil/include/zenutil/buildstorage.h
@@ -25,6 +25,7 @@ public:
 
 	virtual ~BuildStorage() {}
 
+	virtual CbObject ListNamespaces(bool bRecursive = false)				= 0;
 	virtual CbObject ListBuilds(CbObject Query)								= 0;
 	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0;
 	virtual CbObject GetBuild(const Oid& BuildId)							= 0;
diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h
index c2886ca4c..32bfd50f4 100644
--- a/src/zenutil/include/zenutil/jupiter/jupitersession.h
+++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h
@@ -102,6 +102,8 @@ public:
 
 	std::vector<IoHash> Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes);
 
+	JupiterResult	   ListBuildNamespaces();
+	JupiterResult	   ListBuildBuckets(std::string_view Namespace);
 	JupiterResult	   ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload);
 	JupiterResult	   PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload);
 	JupiterResult	   GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index 24e062c7b..6ada72e1e 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -35,6 +35,61 @@ public:
 	}
 	virtual ~JupiterBuildStorage() {}
 
+	virtual CbObject ListNamespaces(bool bRecursive) override
+	{
+		ZEN_TRACE_CPU("Jupiter::ListNamespaces");
+
+		Stopwatch	  ExecutionTimer;
+		auto		  _			 = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+		JupiterResult ListResult = m_Session.ListBuildNamespaces();
+		AddStatistic(ListResult);
+		if (!ListResult.Success)
+		{
+			throw std::runtime_error(fmt::format("Failed listing namespaces: {} ({})", ListResult.Reason, ListResult.ErrorCode));
+		}
+		CbObject NamespaceResponse = PayloadToCbObject("Failed listing namespaces"sv, ListResult.Response);
+
+		CbObjectWriter Response;
+		Response.BeginArray("results"sv);
+		for (CbFieldView NamespaceField : NamespaceResponse["namespaces"])
+		{
+			std::string_view Namespace = NamespaceField.AsString();
+			if (!Namespace.empty())
+			{
+				Response.BeginObject();
+				Response.AddString("name", Namespace);
+
+				if (bRecursive)
+				{
+					JupiterResult BucketsResult = m_Session.ListBuildBuckets(Namespace);
+					AddStatistic(BucketsResult);
+					if (!BucketsResult.Success)
+					{
+						throw std::runtime_error(
+							fmt::format("Failed listing namespaces: {} ({})", BucketsResult.Reason, BucketsResult.ErrorCode));
+					}
+					CbObject BucketResponse = PayloadToCbObject("Failed listing namespaces"sv, BucketsResult.Response);
+
+					Response.BeginArray("items");
+					for (CbFieldView BucketField : BucketResponse["buckets"])
+					{
+						std::string_view Bucket = BucketField.AsString();
+						if (!Bucket.empty())
+						{
+							Response.AddString(Bucket);
+						}
+					}
+					Response.EndArray();
+				}
+
+				Response.EndObject();
+			}
+		}
+		Response.EndArray();
+
+		return Response.Save();
+	}
+
 	virtual CbObject ListBuilds(CbObject Query) override
 	{
 		ZEN_TRACE_CPU("Jupiter::ListBuilds");
diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index 1f71c29b7..d3076d36b 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -357,6 +357,21 @@ JupiterSession::CacheTypeExists(std::string_view Namespace, std::string_view Typ
 }
 
 JupiterResult
+JupiterSession::ListBuildNamespaces()
+{
+	HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds"), {HttpClient::Accept(ZenContentType::kJSON)});
+	return detail::ConvertResponse(Response, "JupiterSession::ListBuildNamespaces"sv);
+}
+
+JupiterResult
+JupiterSession::ListBuildBuckets(std::string_view Namespace)
+{
+	HttpClient::Response Response =
+		m_HttpClient.Get(fmt::format("/api/v2/builds/{}", Namespace), {HttpClient::Accept(ZenContentType::kJSON)});
+	return detail::ConvertResponse(Response, "JupiterSession::ListBuildBuckets"sv);
+}
+
+JupiterResult
 JupiterSession::ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload)
 {
 	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);