build list filters (#313)

 - Feature: `zen builds list` command has new options
	- `--query-path` - path to a .json (json format) or .cbo (compact binary object format) with the search query to use
	- `--result-path` - path to a .json (json format) or .cbo (compact binary object format) to write output result to, if omitted json format will be output to console

5 files changed, 96 insertions, 35 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 11efc42ec..a1fb1a94a 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -12,6 +12,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
+#include <zencore/stream.h>
 #include <zencore/string.h>
 #include <zencore/trace.h>
 #include <zencore/uid.h>
@@ -7133,7 +7134,12 @@ BuildsCommand::BuildsCommand()
 		Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
 	};
 
-	m_Options.add_option("", "v", "verb", "Verb for build - list, upload, download, diff", cxxopts::value(m_Verb), "<verb>");
+	m_Options.add_option("",
+						 "v",
+						 "verb",
+						 "Verb for build - list, upload, download, diff, fetch-blob, validate-part",
+						 cxxopts::value(m_Verb),
+						 "<verb>");
 	m_Options.parse_positional({"verb"});
 	m_Options.positional_help("verb");
 
@@ -7142,6 +7148,20 @@ BuildsCommand::BuildsCommand()
 	AddFileOptions(m_ListOptions);
 	AddOutputOptions(m_ListOptions);
 	m_ListOptions.add_options()("h,help", "Print help");
+	m_ListOptions.add_option("",
+							 "",
+							 "query-path",
+							 "Path to json or compactbinary file containing list query",
+							 cxxopts::value(m_ListQueryPath),
+							 "<query-path>");
+	m_ListOptions.add_option("",
+							 "",
+							 "result-path",
+							 "Path to json or compactbinary to write query result to",
+							 cxxopts::value(m_ListResultPath),
+							 "<result-path>");
+	m_ListOptions.parse_positional({"query-path", "result-path"});
+	m_ListOptions.positional_help("query-path result-path");
 
 	// upload
 	AddCloudOptions(m_UploadOptions);
@@ -7418,7 +7438,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		}
 		else if (!m_AccessTokenPath.empty())
 		{
-			std::string ResolvedAccessToken = ReadAccessTokenFromFile(m_AccessTokenPath);
+			std::string ResolvedAccessToken = ReadAccessTokenFromFile(StringToPath(m_AccessTokenPath));
 			if (!ResolvedAccessToken.empty())
 			{
 				ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken);
@@ -7469,32 +7489,50 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			HttpClient Http(m_BuildsUrl, ClientSettings);
 
-			CbObjectWriter QueryWriter;
-			QueryWriter.BeginObject("query");
+			CbObject QueryObject;
+			if (m_ListQueryPath.empty())
+			{
+				CbObjectWriter QueryWriter;
+				QueryWriter.BeginObject("query");
+				QueryWriter.EndObject();  // query
+				QueryObject = QueryWriter.Save();
+			}
+			else
 			{
-				//			QueryWriter.BeginObject("platform");
-				//			{
-				//				QueryWriter.AddString("$eq", "Windows");
-				//			}
-				//			QueryWriter.EndObject(); // changelist
+				std::filesystem::path ListQueryPath = StringToPath(m_ListQueryPath);
+				if (ToLower(ListQueryPath.extension().string()) == ".cbo")
+				{
+					QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath));
+				}
+				else
+				{
+					IoBuffer		 MetaDataJson = ReadFile(ListQueryPath).Flatten();
+					std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+					std::string		 JsonError;
+					QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+					if (!JsonError.empty())
+					{
+						throw std::runtime_error(
+							fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError));
+					}
+				}
 			}
-			QueryWriter.EndObject();  // query
 
 			BuildStorage::Statistics	  StorageStats;
 			std::unique_ptr<BuildStorage> Storage;
 			if (!m_BuildsUrl.empty())
 			{
-				ZEN_CONSOLE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'",
-							m_BuildsUrl,
-							Http.GetSessionId(),
-							m_Namespace,
-							m_Bucket);
+				ZEN_CONSOLE_VERBOSE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'",
+									m_BuildsUrl,
+									Http.GetSessionId(),
+									m_Namespace,
+									m_Bucket);
 				Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, std::filesystem::path{});
 			}
 			else if (!m_StoragePath.empty())
 			{
 				std::filesystem::path StoragePath = StringToPath(m_StoragePath);
-				ZEN_CONSOLE("Querying builds in folder '{}'.", StoragePath);
+				ZEN_CONSOLE_VERBOSE("Querying builds in folder '{}'.", StoragePath);
 				Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
 			}
 			else
@@ -7502,10 +7540,30 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help()));
 			}
 
-			CbObject					  Response = Storage->ListBuilds(QueryWriter.Save());
-			ExtendableStringBuilder<1024> SB;
-			CompactBinaryToJson(Response.GetView(), SB);
-			ZEN_CONSOLE("{}", SB.ToView());
+			CbObject Response = Storage->ListBuilds(QueryObject);
+			ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None);
+			if (m_ListResultPath.empty())
+			{
+				ExtendableStringBuilder<1024> SB;
+				CompactBinaryToJson(Response.GetView(), SB);
+				ZEN_CONSOLE("{}", SB.ToView());
+			}
+			else
+			{
+				std::filesystem::path ListResultPath = StringToPath(m_ListResultPath);
+				if (ToLower(ListResultPath.extension().string()) == ".cbo")
+				{
+					MemoryView ResponseView = Response.GetView();
+					WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
+				}
+				else
+				{
+					ExtendableStringBuilder<1024> SB;
+					CompactBinaryToJson(Response.GetView(), SB);
+					WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+				}
+			}
+
 			return 0;
 		}
 
@@ -7622,7 +7680,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			{
 				if (!m_BuildMetadataPath.empty())
 				{
-					std::filesystem::path MetadataPath(m_BuildMetadataPath);
+					std::filesystem::path MetadataPath = StringToPath(m_BuildMetadataPath);
 					IoBuffer			  MetaDataJson = ReadFile(MetadataPath).Flatten();
 					std::string_view	  Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
 					std::string			  JsonError;
@@ -7654,7 +7712,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 						 BuildPartId,
 						 m_BuildPartName,
 						 Path,
-						 m_ManifestPath,
+						 StringToPath(m_ManifestPath),
 						 m_BlockReuseMinPercentLimit,
 						 m_AllowMultiparts,
 						 MetaData,
@@ -7795,8 +7853,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			{
 				throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help()));
 			}
-			std::filesystem::path Path = StringToPath(m_Path);
-			DiffFolders(Path, m_DiffPath, m_OnlyChunked);
+			std::filesystem::path Path	   = StringToPath(m_Path);
+			std::filesystem::path DiffPath = StringToPath(m_DiffPath);
+			DiffFolders(Path, DiffPath, m_OnlyChunked);
 			return AbortFlag ? 11 : 0;
 		}
 
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 60953efad..1634975c1 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -75,6 +75,8 @@ private:
 	std::string m_Verb;	 // list, upload, download
 
 	cxxopts::Options m_ListOptions{"list", "List available builds"};
+	std::string		 m_ListQueryPath;
+	std::string		 m_ListResultPath;
 
 	std::string m_Path;
 
@@ -90,23 +92,23 @@ private:
 	std::string		 m_DiffPath;
 	bool			 m_OnlyChunked = false;
 
-	cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"};
-
-	cxxopts::Options		 m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"};
-	std::vector<std::string> m_BuildIds;
-
 	cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"};
 	std::string		 m_BlobHash;
 
 	cxxopts::Options m_ValidateBuildPartOptions{"validate-part", "Fetch a build part and validate all referenced attachments"};
 
+	cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"};
+
+	cxxopts::Options		 m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"};
+	std::vector<std::string> m_BuildIds;
+
 	cxxopts::Options* m_SubCommands[8] = {&m_ListOptions,
 										  &m_UploadOptions,
 										  &m_DownloadOptions,
 										  &m_DiffOptions,
-										  &m_TestOptions,
 										  &m_FetchBlobOptions,
 										  &m_ValidateBuildPartOptions,
+										  &m_TestOptions,
 										  &m_MultiTestDownloadOptions};
 };
 
diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h
index a46b9fd83..4d901bdb5 100644
--- a/src/zenhttp/include/zenhttp/httpclient.h
+++ b/src/zenhttp/include/zenhttp/httpclient.h
@@ -97,7 +97,7 @@ public:
 		HttpResponseCode StatusCode = HttpResponseCode::ImATeapot;
 		IoBuffer		 ResponsePayload;  // Note: this also includes the content type
 
-		// Contains the reponse headers
+		// Contains the response headers
 		KeyValueMap Header;
 
 		// The number of bytes sent as part of the request
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index 3bdcdf098..a315ea2df 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -684,7 +684,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
 		);
 	}
 
-	auto EmitReponseDetails = [&](StringBuilderBase& ResponseDetails) -> void {
+	auto EmitResponseDetails = [&](StringBuilderBase& ResponseDetails) -> void {
 		for (int i = 0; i < ThisRequestChunkCount; ++i)
 		{
 			const HTTP_DATA_CHUNK Chunk = m_HttpDataChunks[ThisRequestChunkOffset + i];
@@ -767,7 +767,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
 		// Emit diagnostics
 
 		ExtendableStringBuilder<256> ResponseDetails;
-		EmitReponseDetails(ResponseDetails);
+		EmitResponseDetails(ResponseDetails);
 
 		ZEN_WARN("failed to send HTTP response (error {}: '{}'), request URL: '{}', ({}.{}) response: {}",
 				 SendResult,
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 6259c0f37..027a35998 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -3017,9 +3017,9 @@ TEST_CASE("project.remote")
 		{
 			cpr::Response StatusResponse = Session.Get();
 			CHECK(IsHttpSuccessCode(StatusResponse.status_code));
-			CbObject ReponseObject =
+			CbObject ResponseObject =
 				LoadCompactBinaryObject(IoBuffer(IoBuffer::Wrap, StatusResponse.text.data(), StatusResponse.text.size()));
-			std::string_view Status = ReponseObject["Status"sv].AsString();
+			std::string_view Status = ResponseObject["Status"sv].AsString();
 			CHECK(Status != "Aborted"sv);
 			if (Status == "Complete"sv)
 			{