structured output for builds ls (#709)

* make ResolveBuildStore respect Verbose flag
* add structured output to zen builds ls command

3 files changed, 188 insertions, 70 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 99f29376e..d7980cc24 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -2008,81 +2008,148 @@ namespace {
 				   const std::vector<Oid>&		BuildPartIds,
 				   std::span<const std::string> BuildPartNames,
 				   std::span<const std::string> IncludeWildcards,
-				   std::span<const std::string> ExcludeWildcards)
+				   std::span<const std::string> ExcludeWildcards,
+				   CbObjectWriter*				OptionalStructuredOutput)
 	{
 		std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
 
 		CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);
 
+		if (OptionalStructuredOutput != nullptr)
+		{
+			OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId);
+			OptionalStructuredOutput->AddObject("build"sv, BuildObject);
+		}
+
 		std::vector<std::pair<Oid, std::string>> AllBuildParts =
 			ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
 
-		Stopwatch GetBuildPartTimer;
-
-		for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++)
+		if (!AllBuildParts.empty())
 		{
-			const Oid			   BuildPartId		 = AllBuildParts[BuildPartIndex].first;
-			const std::string_view BuildPartName	 = AllBuildParts[BuildPartIndex].second;
-			CbObject			   BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
+			Stopwatch GetBuildPartTimer;
 
-			if (!IsQuiet)
+			if (OptionalStructuredOutput != nullptr)
 			{
-				ZEN_CONSOLE("{}Part: {} ('{}'):\n",
-							BuildPartIndex > 0 ? "\n" : "",
-							BuildPartId,
-							BuildPartName,
-							NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
-							NiceBytes(BuildPartManifest.GetSize()));
+				OptionalStructuredOutput->BeginArray("parts"sv);
 			}
 
-			std::vector<std::filesystem::path> Paths;
-			std::vector<IoHash>				   RawHashes;
-			std::vector<uint64_t>			   RawSizes;
-			std::vector<uint32_t>			   Attributes;
-
-			SourcePlatform		  Platform;
-			std::vector<IoHash>	  SequenceRawHashes;
-			std::vector<uint32_t> ChunkCounts;
-			std::vector<uint32_t> AbsoluteChunkOrders;
-			std::vector<IoHash>	  LooseChunkHashes;
-			std::vector<uint64_t> LooseChunkRawSizes;
-			std::vector<IoHash>	  BlockRawHashes;
+			for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++)
+			{
+				const Oid			   BuildPartId		 = AllBuildParts[BuildPartIndex].first;
+				const std::string_view BuildPartName	 = AllBuildParts[BuildPartIndex].second;
+				CbObject			   BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
 
-			ReadBuildContentFromCompactBinary(BuildPartManifest,
-											  Platform,
-											  Paths,
-											  RawHashes,
-											  RawSizes,
-											  Attributes,
-											  SequenceRawHashes,
-											  ChunkCounts,
-											  AbsoluteChunkOrders,
-											  LooseChunkHashes,
-											  LooseChunkRawSizes,
-											  BlockRawHashes);
+				if (OptionalStructuredOutput != nullptr)
+				{
+					OptionalStructuredOutput->BeginObject();
+					OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId);
+					OptionalStructuredOutput->AddString("partName"sv, BuildPartName);
+				}
+				{
+					if (OptionalStructuredOutput != nullptr)
+					{
+					}
+					else if (!IsQuiet)
+					{
+						ZEN_CONSOLE("{}Part: {} ('{}'):\n",
+									BuildPartIndex > 0 ? "\n" : "",
+									BuildPartId,
+									BuildPartName,
+									NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+									NiceBytes(BuildPartManifest.GetSize()));
+					}
 
-			std::vector<size_t> Order(Paths.size());
-			std::iota(Order.begin(), Order.end(), 0);
+					std::vector<std::filesystem::path> Paths;
+					std::vector<IoHash>				   RawHashes;
+					std::vector<uint64_t>			   RawSizes;
+					std::vector<uint32_t>			   Attributes;
+
+					SourcePlatform		  Platform;
+					std::vector<IoHash>	  SequenceRawHashes;
+					std::vector<uint32_t> ChunkCounts;
+					std::vector<uint32_t> AbsoluteChunkOrders;
+					std::vector<IoHash>	  LooseChunkHashes;
+					std::vector<uint64_t> LooseChunkRawSizes;
+					std::vector<IoHash>	  BlockRawHashes;
+
+					ReadBuildContentFromCompactBinary(BuildPartManifest,
+													  Platform,
+													  Paths,
+													  RawHashes,
+													  RawSizes,
+													  Attributes,
+													  SequenceRawHashes,
+													  ChunkCounts,
+													  AbsoluteChunkOrders,
+													  LooseChunkHashes,
+													  LooseChunkRawSizes,
+													  BlockRawHashes);
+
+					std::vector<size_t> Order(Paths.size());
+					std::iota(Order.begin(), Order.end(), 0);
+
+					std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) {
+						const std::filesystem::path& LhsPath = Paths[Lhs];
+						const std::filesystem::path& RhsPath = Paths[Rhs];
+						return LhsPath < RhsPath;
+					});
 
-			std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) {
-				const std::filesystem::path& LhsPath = Paths[Lhs];
-				const std::filesystem::path& RhsPath = Paths[Rhs];
-				return LhsPath < RhsPath;
-			});
+					if (OptionalStructuredOutput != nullptr)
+					{
+						OptionalStructuredOutput->BeginArray("files"sv);
+					}
+					{
+						for (size_t Index : Order)
+						{
+							const std::filesystem::path& Path = Paths[Index];
+							if (IncludePath(IncludeWildcards, ExcludeWildcards, Path))
+							{
+								const IoHash&  RawHash	 = RawHashes[Index];
+								const uint64_t RawSize	 = RawSizes[Index];
+								const uint32_t Attribute = Attributes[Index];
 
-			for (size_t Index : Order)
-			{
-				const std::filesystem::path& Path = Paths[Index];
-				if (IncludePath(IncludeWildcards, ExcludeWildcards, Path))
+								if (OptionalStructuredOutput != nullptr)
+								{
+									OptionalStructuredOutput->BeginObject();
+									{
+										OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path));
+										OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize);
+										switch (Platform)
+										{
+											case SourcePlatform::Windows:
+												OptionalStructuredOutput->AddInteger("attributes"sv, Attribute);
+												break;
+											case SourcePlatform::MacOS:
+											case SourcePlatform::Linux:
+												OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute));
+												break;
+											default:
+												throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform));
+										}
+									}
+									OptionalStructuredOutput->EndObject();
+								}
+								else
+								{
+									ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash);
+								}
+							}
+						}
+					}
+					if (OptionalStructuredOutput != nullptr)
+					{
+						OptionalStructuredOutput->EndArray();  // "files"
+					}
+				}
+				if (OptionalStructuredOutput != nullptr)
 				{
-					const IoHash&  RawHash	 = RawHashes[Index];
-					const uint64_t RawSize	 = RawSizes[Index];
-					const uint32_t Attribute = Attributes[Index];
-					ZEN_UNUSED(Attribute);
-
-					ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash);
+					OptionalStructuredOutput->EndObject();
 				}
 			}
+			if (OptionalStructuredOutput != nullptr)
+			{
+				OptionalStructuredOutput->EndArray();  // parts
+			}
 		}
 	}
 
@@ -2493,12 +2560,13 @@ BuildsCommand::BuildsCommand()
 									   "Enable fetch of buckets within namespaces also",
 									   cxxopts::value(m_ListNamespacesRecursive),
 									   "<recursive>");
-	m_ListNamespacesOptions.add_option("",
-									   "",
-									   "result-path",
-									   "Path to json or compactbinary to write query result to",
-									   cxxopts::value(m_ListResultPath),
-									   "<result-path>");
+	m_ListNamespacesOptions.add_option(
+		"",
+		"",
+		"result-path",
+		"Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
+		cxxopts::value(m_ListResultPath),
+		"<result-path>");
 	m_ListNamespacesOptions.parse_positional({"result-path"});
 	m_ListNamespacesOptions.positional_help("result-path");
 
@@ -2518,7 +2586,7 @@ BuildsCommand::BuildsCommand()
 	m_ListOptions.add_option("",
 							 "",
 							 "result-path",
-							 "Path to json or compactbinary to write query result to",
+							 "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
 							 cxxopts::value(m_ListResultPath),
 							 "<result-path>");
 	m_ListOptions.parse_positional({"query-path", "result-path"});
@@ -2533,7 +2601,7 @@ BuildsCommand::BuildsCommand()
 	m_ListBlocksOptions.add_option("",
 								   "",
 								   "result-path",
-								   "Path to json or compactbinary to write query result to",
+								   "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
 								   cxxopts::value(m_ListResultPath),
 								   "<result-path>");
 
@@ -2724,6 +2792,20 @@ BuildsCommand::BuildsCommand()
 						   cxxopts::value(m_BuildPartNames),
 						   "<name>");
 
+	m_LsOptions.add_option("",
+						   "",
+						   "result-path",
+						   "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
+						   cxxopts::value(m_LsResultPath),
+						   "<result-path>");
+
+	m_LsOptions.add_option("",
+						   "o",
+						   "output-path",
+						   "Path to output, extension .json or .cb (compac binary). Default is output to console",
+						   cxxopts::value(m_LsResultPath),
+						   "<output-path>");
+
 	m_LsOptions.parse_positional({"build-id", "wildcard"});
 	m_LsOptions.positional_help("build-id wildcard");
 
@@ -3837,9 +3919,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	if (SubOption == &m_LsOptions)
 	{
-		if (!IsQuiet)
+		if (!m_LsResultPath.empty())
 		{
-			LogExecutableVersionAndPid();
+			if (!IsQuiet)
+			{
+				LogExecutableVersionAndPid();
+			}
 		}
 
 		ZenState InstanceState;
@@ -3869,7 +3954,30 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		std::vector<Oid>		 BuildPartIds	= ParseBuildPartIds();
 		std::vector<std::string> BuildPartNames = ParseBuildPartNames();
 
-		ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards);
+		std::unique_ptr<CbObjectWriter> StructuredOutput;
+		if (!m_LsResultPath.empty())
+		{
+			MakeSafeAbsolutePathÍnPlace(m_LsResultPath);
+			StructuredOutput = std::make_unique<CbObjectWriter>();
+		}
+
+		ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get());
+
+		if (StructuredOutput)
+		{
+			CbObject Response = StructuredOutput->Save();
+			if (ToLower(m_LsResultPath.extension().string()) == ".cbo")
+			{
+				MemoryView ResponseView = Response.GetView();
+				WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
+			}
+			else
+			{
+				ExtendableStringBuilder<1024> SB;
+				CompactBinaryToJson(Response.GetView(), SB);
+				WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+			}
+		}
 
 		if (AbortFlag)
 		{
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 4f6209f93..80c64c48d 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -106,7 +106,8 @@ private:
 	bool					 m_PostDownloadVerify = false;
 	bool					 m_EnableScavenging	  = true;
 
-	cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"};
+	cxxopts::Options	  m_LsOptions{"ls", "List the content of uploaded build"};
+	std::filesystem::path m_LsResultPath;
 
 	cxxopts::Options	  m_DiffOptions{"diff", "Compare two local folders"};
 	std::filesystem::path m_DiffPath;
diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp
index 15ece2edd..36b45e800 100644
--- a/src/zenremotestore/builds/buildstorageutil.cpp
+++ b/src/zenremotestore/builds/buildstorageutil.cpp
@@ -129,7 +129,10 @@ ResolveBuildStorage(OperationLogOutput&		  Output,
 						TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose);
 					TestResult.Success)
 				{
-					ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl);
+					if (Verbose)
+					{
+						ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl);
+					}
 
 					HostUrl			= ServerEndpoint.BaseUrl;
 					HostAssumeHttp2 = ServerEndpoint.AssumeHttp2;
@@ -172,7 +175,10 @@ ResolveBuildStorage(OperationLogOutput&		  Output,
 							TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose);
 						TestResult.Success)
 					{
-						ZEN_OPERATION_LOG_INFO(Output, "Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl);
+						if (Verbose)
+						{
+							ZEN_OPERATION_LOG_INFO(Output, "Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl);
+						}
 
 						CacheUrl		 = CacheEndpoint.BaseUrl;
 						CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2;
@@ -391,7 +397,10 @@ GetBlockDescriptions(OperationLogOutput&	 Output,
 							 [BlockHash](const ChunkBlockDescription& Description) { return Description.BlockHash == BlockHash; });
 						 ListBlocksIt != FoundBlocks.end())
 				{
-					ZEN_OPERATION_LOG_INFO(Output, "Found block {} via context find successfully", BlockHash);
+					if (!IsQuiet)
+					{
+						ZEN_OPERATION_LOG_INFO(Output, "Found block {} via context find successfully", BlockHash);
+					}
 					AugmentedBlockDescriptions.emplace_back(std::move(*ListBlocksIt));
 				}
 				else