aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2026-03-16 12:03:00 +0100
committerGitHub Enterprise <[email protected]>2026-03-16 12:03:00 +0100
commitdfdcdcc4645f6aa6c9c8ef6fd2a0b2f1e5b38070 (patch)
treee058fcc16b9344f0809f8826f6cadd8a8ae3ee87 /src
parentFix WsHttpSysConnection to use WinIoThreadPool abstraction instead of raw PTP_IO (diff)
parentRestructure zen builds using subcommands (#834) (diff)
downloadzen-dfdcdcc4645f6aa6c9c8ef6fd2a0b2f1e5b38070.tar.xz
zen-dfdcdcc4645f6aa6c9c8ef6fd2a0b2f1e5b38070.zip
Merge branch 'main' into sb/threadpoolsb/threadpool
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/builds_cmd.cpp4548
-rw-r--r--src/zen/cmds/builds_cmd.h383
2 files changed, 2647 insertions, 2284 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index b4b4df7c9..93108dd47 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -2011,2156 +2011,2231 @@ namespace builds_impl {
} // namespace builds_impl
//////////////////////////////////////////////////////////////////////////////////////////////////////
+// BuildsCommand — Option-adding helpers
+//
-BuildsCommand::BuildsCommand()
+void
+BuildsCommand::AddSystemOptions(cxxopts::Options& Ops)
{
- using namespace builds_impl;
- m_Options.add_options()("h,help", "Print help");
-
- auto AddSystemOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>");
- Ops.add_option("",
- "",
- "use-sparse-files",
- "Enable use of sparse files when writing large files. Defaults to true.",
- cxxopts::value(m_UseSparseFiles),
- "<usesparsefiles>");
- };
-
- auto AddCloudOptions = [this](cxxopts::Options& Ops) {
- m_AuthOptions.AddOptions(Ops);
-
- Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>");
- Ops.add_option("cloud build",
- "",
- "url",
- "Cloud Builds host url (legacy - use --override-host)",
- cxxopts::value(m_OverrideHost),
- "<url>");
- Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>");
- Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>");
- Ops.add_option("cloud build",
- "",
- "assume-http2",
- "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake",
- cxxopts::value(m_AssumeHttp2),
- "<assumehttp2>");
- Ops.add_option("cloud build",
- "",
- "verbose-http",
- "Enable verbose option for http client",
- cxxopts::value(m_VerboseHttp),
- "<verbosehttp>");
-
- Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>");
- Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>");
- Ops.add_option("cloud build",
- "",
- "allow-redirect",
- "Allow redirect of requests",
- cxxopts::value(m_AllowRedirect),
- "<allow-redirect>");
- };
+ Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>");
+ Ops.add_option("",
+ "",
+ "use-sparse-files",
+ "Enable use of sparse files when writing large files. Defaults to true.",
+ cxxopts::value(m_UseSparseFiles),
+ "<usesparsefiles>");
+}
- auto AddFileOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>");
- Ops.add_option("filestorage",
- "",
- "json-metadata",
- "Write build, part and block metadata as .json files in addition to .cb files",
- cxxopts::value(m_WriteMetadataAsJson),
- "<jsonmetadata>");
- };
+void
+BuildsCommand::AddCloudOptions(cxxopts::Options& Ops)
+{
+ m_AuthOptions.AddOptions(Ops);
+
+ Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>");
+ Ops.add_option("cloud build",
+ "",
+ "url",
+ "Cloud Builds host url (legacy - use --override-host)",
+ cxxopts::value(m_OverrideHost),
+ "<url>");
+ Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>");
+ Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>");
+ Ops.add_option("cloud build",
+ "",
+ "assume-http2",
+ "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake",
+ cxxopts::value(m_AssumeHttp2),
+ "<assumehttp2>");
+ Ops.add_option("cloud build",
+ "",
+ "verbose-http",
+ "Enable verbose option for http client",
+ cxxopts::value(m_VerboseHttp),
+ "<verbosehttp>");
+
+ Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>");
+ Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>");
+ Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(m_AllowRedirect), "<allow-redirect>");
+}
- auto AddCacheOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>");
- };
+void
+BuildsCommand::AddFileOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>");
+ Ops.add_option("filestorage",
+ "",
+ "json-metadata",
+ "Write build, part and block metadata as .json files in addition to .cb files",
+ cxxopts::value(m_WriteMetadataAsJson),
+ "<jsonmetadata>");
+}
- auto AddOutputOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("output",
- "",
- "plain-progress",
- "Show progress using plain output",
- cxxopts::value(m_PlainProgress),
- "<plainprogress>");
- Ops.add_option("output",
- "",
- "log-progress",
- "Write @progress style progress to output",
- cxxopts::value(m_LogProgress),
- "<logprogress>");
- Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
- Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>");
- };
+void
+BuildsCommand::AddCacheOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>");
+}
- auto AddWorkerOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("",
- "",
- "boost-worker-count",
- "Increase the number of worker threads - may cause computer to be less responsive",
- cxxopts::value(m_BoostWorkerCount),
- "<boostworkercount>");
-
- Ops.add_option("",
- "",
- "boost-worker-memory",
- "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to "
- "be less responsive due to high memory usage",
- cxxopts::value(m_BoostWorkerMemory),
- "<boostworkermemory>");
-
- Ops.add_option("",
- "",
- "boost-workers",
- "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive",
- cxxopts::value(m_BoostWorkers),
- "<boostworkermemory>");
- };
+void
+BuildsCommand::AddOutputOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<plainprogress>");
+ Ops.add_option("output",
+ "",
+ "log-progress",
+ "Write @progress style progress to output",
+ cxxopts::value(m_LogProgress),
+ "<logprogress>");
+ Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
+ Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>");
+}
- auto AddZenFolderOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("",
- "",
- "zen-folder-path",
- fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName),
- cxxopts::value(m_ZenFolderPath),
- "<boostworkers>");
- };
+void
+BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("",
+ "",
+ "boost-worker-count",
+ "Increase the number of worker threads - may cause computer to be less responsive",
+ cxxopts::value(m_BoostWorkerCount),
+ "<boostworkercount>");
+
+ Ops.add_option("",
+ "",
+ "boost-worker-memory",
+ "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to "
+ "be less responsive due to high memory usage",
+ cxxopts::value(m_BoostWorkerMemory),
+ "<boostworkermemory>");
+
+ Ops.add_option("",
+ "",
+ "boost-workers",
+ "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive",
+ cxxopts::value(m_BoostWorkers),
+ "<boostworkermemory>");
+}
- auto AddChunkingCacheOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("",
- "",
- "chunking-cache-path",
- "Path to cache for chunking information of scanned files. Default is empty resulting in no caching",
- cxxopts::value(m_ChunkingCachePath),
- "<chunkingcachepath>");
- };
+void
+BuildsCommand::AddZenFolderOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("",
+ "",
+ "zen-folder-path",
+ fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", builds_impl::ZenFolderName),
+ cxxopts::value(m_ZenFolderPath),
+ "<boostworkers>");
+}
- auto AddWildcardOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("",
- "",
- "wildcard",
- "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;",
- cxxopts::value(m_IncludeWildcard),
- "<wildcard>");
-
- Ops.add_option("",
- "",
- "exclude-wildcard",
- "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard "
- "include filter",
- cxxopts::value(m_ExcludeWildcard),
- "<excludewildcard>");
- };
+void
+BuildsCommand::AddChunkingCacheOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("",
+ "",
+ "chunking-cache-path",
+ "Path to cache for chunking information of scanned files. Default is empty resulting in no caching",
+ cxxopts::value(m_ChunkingCachePath),
+ "<chunkingcachepath>");
+}
- auto AddExcludeFolderOption = [this](cxxopts::Options& Ops) {
- Ops.add_option("",
- "",
- "exclude-folders",
- "Names of folders to exclude, separated by ;",
- cxxopts::value(m_ExcludeFolders),
- "<excludefolders>");
- };
+void
+BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("",
+ "",
+ "wildcard",
+ "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;",
+ cxxopts::value(m_IncludeWildcard),
+ "<wildcard>");
+
+ Ops.add_option("",
+ "",
+ "exclude-wildcard",
+ "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard "
+ "include filter",
+ cxxopts::value(m_ExcludeWildcard),
+ "<excludewildcard>");
+}
- auto AddExcludeExtensionsOption = [this](cxxopts::Options& Ops) {
- Ops.add_option("",
- "",
- "exclude-extensions",
- "Extensions to exclude, separated by ;"
- "include filter",
- cxxopts::value(m_ExcludeExtensions),
- "<excludeextensions>");
- };
+void
+BuildsCommand::AddExcludeFolderOption(cxxopts::Options& Ops)
+{
+ Ops.add_option("",
+ "",
+ "exclude-folders",
+ "Names of folders to exclude, separated by ;",
+ cxxopts::value(m_ExcludeFolders),
+ "<excludefolders>");
+}
- auto AddMultipartOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("",
- "",
- "allow-multipart",
- "Allow large attachments to be transfered using multipart protocol. Defaults to true.",
- cxxopts::value(m_AllowMultiparts),
- "<allowmultipart>");
- };
+void
+BuildsCommand::AddExcludeExtensionsOption(cxxopts::Options& Ops)
+{
+ Ops.add_option("",
+ "",
+ "exclude-extensions",
+ "Extensions to exclude, separated by ;"
+ "include filter",
+ cxxopts::value(m_ExcludeExtensions),
+ "<excludeextensions>");
+}
- auto AddPartialBlockRequestOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("",
- "",
- "allow-partial-block-requests",
- "Allow request for partial chunk blocks.\n"
- " false = only full block requests allowed\n"
- " mixed = multiple partial block ranges requests per block allowed to zen cache, single partial block range "
- "request per block to host\n"
- " zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests "
- "allowed to host\n"
- " true = multiple partial block ranges requests per block allowed to zen cache and host\n"
- "Defaults to 'mixed'.",
- cxxopts::value(m_AllowPartialBlockRequests),
- "<allowpartialblockrequests>");
- };
+void
+BuildsCommand::AddMultipartOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("",
+ "",
+ "allow-multipart",
+ "Allow large attachments to be transfered using multipart protocol. Defaults to true.",
+ cxxopts::value(m_AllowMultiparts),
+ "<allowmultipart>");
+}
- auto AddAppendNewContentOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("",
- "",
- "append",
- "Decides if the remote data should replace or append to the current local data.\n"
- " false = the local content will be replaced by the remote content\n"
- " true = the remote data will be overlayed on top of local data\n"
- "Defaults to false.",
- cxxopts::value(m_AppendNewContent),
- "<append>");
- };
+void
+BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("",
+ "",
+ "allow-partial-block-requests",
+ "Allow request for partial chunk blocks.\n"
+ " false = only full block requests allowed\n"
+ " mixed = multiple partial block ranges requests per block allowed to zen cache, single partial block range "
+ "request per block to host\n"
+ " zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests "
+ "allowed to host\n"
+ " true = multiple partial block ranges requests per block allowed to zen cache and host\n"
+ "Defaults to 'mixed'.",
+ cxxopts::value(m_AllowPartialBlockRequests),
+ "<allowpartialblockrequests>");
+}
- m_Options.add_option("",
- "v",
- "verb",
- "Verb for build - list-namespaces, list, upload, download, diff, fetch-blob, validate-part",
- cxxopts::value(m_Verb),
- "<verb>");
- m_Options.parse_positional({"verb"});
- m_Options.positional_help("verb");
-
- // list-namespaces
- AddSystemOptions(m_ListNamespacesOptions);
- AddCloudOptions(m_ListNamespacesOptions);
- AddFileOptions(m_ListNamespacesOptions);
- AddOutputOptions(m_ListNamespacesOptions);
- AddZenFolderOptions(m_ListNamespacesOptions);
- m_ListNamespacesOptions.add_options()("h,help", "Print help");
- m_ListNamespacesOptions.add_option("",
- "",
- "recursive",
- "Enable fetch of buckets within namespaces also",
- cxxopts::value(m_ListNamespacesRecursive),
- "<recursive>");
- m_ListNamespacesOptions.add_option(
- "",
- "",
- "result-path",
- "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
- cxxopts::value(m_ListResultPath),
- "<result-path>");
- m_ListNamespacesOptions.parse_positional({"result-path"});
- m_ListNamespacesOptions.positional_help("result-path");
-
- // list
- AddSystemOptions(m_ListOptions);
- AddCloudOptions(m_ListOptions);
- AddFileOptions(m_ListOptions);
- AddOutputOptions(m_ListOptions);
- AddZenFolderOptions(m_ListOptions);
- m_ListOptions.add_options()("h,help", "Print help");
- m_ListOptions.add_option("",
- "",
- "query-path",
- "Path to json or compactbinary file containing list query",
- cxxopts::value(m_ListQueryPath),
- "<query-path>");
- m_ListOptions.add_option("",
- "",
- "result-path",
- "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
- cxxopts::value(m_ListResultPath),
- "<result-path>");
- m_ListOptions.parse_positional({"query-path", "result-path"});
- m_ListOptions.positional_help("query-path result-path");
-
- // list-blocks
- AddSystemOptions(m_ListBlocksOptions);
- AddCloudOptions(m_ListBlocksOptions);
- AddZenFolderOptions(m_ListBlocksOptions);
- m_ListBlocksOptions.add_options()("h,help", "Print help");
- m_ListBlocksOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- m_ListBlocksOptions.add_option("",
- "",
- "result-path",
- "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
- cxxopts::value(m_ListResultPath),
- "<result-path>");
-
- m_ListBlocksOptions
- .add_option("", "", "max-count", "Maximum number of blocks to list", cxxopts::value(m_ListBlocksMaxCount), "<maxcount>");
-
- m_ListBlocksOptions.parse_positional({"build-id"});
- m_ListBlocksOptions.positional_help("build-id");
-
- // upload
- AddSystemOptions(m_UploadOptions);
- AddCloudOptions(m_UploadOptions);
- AddFileOptions(m_UploadOptions);
- AddOutputOptions(m_UploadOptions);
- AddCacheOptions(m_UploadOptions);
- AddWorkerOptions(m_UploadOptions);
- AddZenFolderOptions(m_UploadOptions);
- AddExcludeFolderOption(m_UploadOptions);
- AddExcludeExtensionsOption(m_UploadOptions);
- AddChunkingCacheOptions(m_UploadOptions);
- m_UploadOptions.add_options()("h,help", "Print help");
- m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
- m_UploadOptions.add_option("",
- "",
- "create-build",
- "Set to true to create the containing build, if unset a builds-id must be given and the build already exist",
- cxxopts::value(m_CreateBuild),
- "<id>");
- m_UploadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- m_UploadOptions.add_option("",
- "",
- "build-part-id",
- "Build part Id, if not given it will be auto generated",
- cxxopts::value(m_BuildPartId),
- "<id>");
- m_UploadOptions.add_option("",
- "",
- "build-part-name",
- "Name of the build part, if not given it will be be named after the directory name at end of local-path",
- cxxopts::value(m_BuildPartName),
- "<name>");
- m_UploadOptions.add_option("",
- "",
- "metadata-path",
- "Path to json file that holds the metadata for the build. Requires the create-build option to be set",
- cxxopts::value(m_BuildMetadataPath),
- "<metadata-path>");
- m_UploadOptions.add_option(
- "",
- "",
- "metadata",
- "Key-value pairs separated by ';' with build meta data. (key1=value1;key2=value2). Requires the create-build option to be set",
- cxxopts::value(m_BuildMetadata),
- "<metadata>");
- m_UploadOptions.add_option("", "", "clean", "Ignore existing blocks", cxxopts::value(m_Clean), "<clean>");
- m_UploadOptions.add_option("",
- "",
- "block-min-reuse",
- "Percent of an existing block that must be relevant for it to be resused. Defaults to 85.",
- cxxopts::value(m_BlockReuseMinPercentLimit),
- "<minreuse>");
- m_UploadOptions.add_option("cache",
- "",
- "zen-cache-upload",
- "Upload data downloaded from remote host to zen cache",
- cxxopts::value(m_UploadToZenCache),
- "<uploadtozencache>");
-
- AddMultipartOptions(m_UploadOptions);
-
- m_UploadOptions.add_option("",
- "",
- "manifest-path",
- "Path to a text file with one line of <local path>[TAB]<modification date> per file to include or a "
- "structured .json file describing the parts",
- cxxopts::value(m_ManifestPath),
- "<manifestpath>");
-
- m_UploadOptions
- .add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), "<verify>");
- m_UploadOptions.add_option("",
- "",
- "find-max-block-count",
- "The maximum number of blocks we search for in the build context",
- cxxopts::value(m_FindBlockMaxCount),
- "<maxblockcount>");
-
- m_UploadOptions.parse_positional({"local-path", "build-id"});
- m_UploadOptions.positional_help("local-path build-id");
-
- // download
- AddSystemOptions(m_DownloadOptions);
- AddCloudOptions(m_DownloadOptions);
- AddFileOptions(m_DownloadOptions);
- AddOutputOptions(m_DownloadOptions);
- AddCacheOptions(m_DownloadOptions);
- AddZenFolderOptions(m_DownloadOptions);
- AddWorkerOptions(m_DownloadOptions);
- AddWildcardOptions(m_DownloadOptions);
- AddAppendNewContentOptions(m_DownloadOptions);
- AddExcludeFolderOption(m_DownloadOptions);
-
- m_DownloadOptions.add_option("cache",
- "",
- "cache-prime-only",
- "Only download blobs missing in cache and upload to cache",
- cxxopts::value(m_PrimeCacheOnly),
- "<cacheprimeonly>");
-
- m_DownloadOptions.add_options()("h,help", "Print help");
- m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
- m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- m_DownloadOptions.add_option(
- "",
- "",
- "build-part-id",
- "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
- cxxopts::value(m_BuildPartIds),
- "<id>");
- m_DownloadOptions.add_option("",
- "",
- "build-part-name",
- "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
- "all parts will be downloaded",
- cxxopts::value(m_BuildPartNames),
- "<name>");
- m_DownloadOptions.add_option("",
- "",
- "clean",
- "Delete all data in target folder that is not part of the downloaded content",
- cxxopts::value(m_Clean),
- "<clean>");
- m_DownloadOptions.add_option("",
- "",
- "force",
- "Force download of all content by ignoring any existing local content",
- cxxopts::value(m_Force),
- "<force>");
- m_DownloadOptions.add_option("cache",
- "",
- "zen-cache-upload",
- "Upload data downloaded from remote host to zen cache",
- cxxopts::value(m_UploadToZenCache),
- "<uploadtozencache>");
- AddMultipartOptions(m_DownloadOptions);
-
- AddPartialBlockRequestOptions(m_DownloadOptions);
-
- m_DownloadOptions.add_option(
- "",
- "",
- "download-spec-path",
- "Path to a text file with one line of <local path> per file to include or a structured .json file describing what to download.",
- cxxopts::value(m_DownloadSpecPath),
- "<downloadspecpath>");
+void
+BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops)
+{
+ Ops.add_option("",
+ "",
+ "append",
+ "Decides if the remote data should replace or append to the current local data.\n"
+ " false = the local content will be replaced by the remote content\n"
+ " true = the remote data will be overlayed on top of local data\n"
+ "Defaults to false.",
+ cxxopts::value(m_AppendNewContent),
+ "<append>");
+}
- m_DownloadOptions
- .add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), "<verify>");
- m_DownloadOptions.add_option("",
- "",
- "enable-scavenge",
- "Enable scavenging of data from previouse download locations",
- cxxopts::value(m_EnableScavenging),
- "<scavenge>");
- m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"});
- m_DownloadOptions.positional_help("local-path build-id build-part-name");
-
- m_DownloadOptions.add_option("",
- "",
- "allow-file-clone",
- "Enable use of block reference counting when copying files",
- cxxopts::value(m_AllowFileClone),
- "<allowclone>");
-
- // ls
- AddSystemOptions(m_LsOptions);
- AddCloudOptions(m_LsOptions);
- AddFileOptions(m_LsOptions);
- AddOutputOptions(m_LsOptions);
- AddCacheOptions(m_LsOptions);
- AddZenFolderOptions(m_LsOptions);
- AddWorkerOptions(m_LsOptions);
- AddWildcardOptions(m_LsOptions);
-
- m_LsOptions.add_options()("h,help", "Print help");
- m_LsOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- m_LsOptions.add_option(
- "",
- "",
- "build-part-id",
- "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
- cxxopts::value(m_BuildPartIds),
- "<id>");
- m_LsOptions.add_option("",
- "",
- "build-part-name",
- "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
- "all parts will be downloaded",
- cxxopts::value(m_BuildPartNames),
- "<name>");
-
- m_LsOptions.add_option("",
- "",
- "result-path",
- "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
- cxxopts::value(m_LsResultPath),
- "<result-path>");
-
- m_LsOptions.add_option("",
- "o",
- "output-path",
- "Path to output, extension .json or .cb (compac binary). Default is output to console",
- cxxopts::value(m_LsResultPath),
- "<output-path>");
-
- m_LsOptions.parse_positional({"build-id", "wildcard"});
- m_LsOptions.positional_help("build-id wildcard");
-
- // diff
- AddOutputOptions(m_DiffOptions);
- AddWorkerOptions(m_DiffOptions);
- AddExcludeFolderOption(m_DiffOptions);
- AddExcludeExtensionsOption(m_DiffOptions);
- AddChunkingCacheOptions(m_DiffOptions);
- m_DiffOptions.add_options()("h,help", "Print help");
- m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
- m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>");
- m_DiffOptions.add_option("",
- "",
- "only-chunked",
- "Skip files from diff summation that are not processed with chunking",
- cxxopts::value(m_OnlyChunked),
- "<only-chunked>");
- m_DiffOptions.parse_positional({"local-path", "compare-path"});
- m_DiffOptions.positional_help("local-path compare-path");
-
- // test
- AddSystemOptions(m_TestOptions);
- AddCloudOptions(m_TestOptions);
- AddFileOptions(m_TestOptions);
- AddOutputOptions(m_TestOptions);
- AddCacheOptions(m_TestOptions);
- AddWorkerOptions(m_TestOptions);
- m_TestOptions.add_options()("h,help", "Print help");
- m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
- AddMultipartOptions(m_TestOptions);
- AddPartialBlockRequestOptions(m_TestOptions);
- AddWildcardOptions(m_TestOptions);
- AddAppendNewContentOptions(m_TestOptions);
- AddChunkingCacheOptions(m_TestOptions);
-
- m_TestOptions.add_option("",
- "",
- "enable-scavenge",
- "Enable scavenging of data from previouse download locations",
- cxxopts::value(m_EnableScavenging),
- "<scavenge>");
- m_TestOptions.add_option("",
- "",
- "allow-file-clone",
- "Enable use of block reference counting when copying files",
- cxxopts::value(m_AllowFileClone),
- "<allowclone>");
- m_TestOptions.parse_positional({"local-path"});
- m_TestOptions.positional_help("local-path");
-
- // fetch-blob
- AddSystemOptions(m_FetchBlobOptions);
- AddCloudOptions(m_FetchBlobOptions);
- AddFileOptions(m_FetchBlobOptions);
- AddOutputOptions(m_FetchBlobOptions);
- AddCacheOptions(m_FetchBlobOptions);
- AddZenFolderOptions(m_FetchBlobOptions);
- m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- m_FetchBlobOptions
- .add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>");
- m_FetchBlobOptions.parse_positional({"build-id", "blob-hash"});
- m_FetchBlobOptions.positional_help("build-id blob-hash");
-
- // prime-cache
- AddSystemOptions(m_PrimeCacheOptions);
- AddCloudOptions(m_PrimeCacheOptions);
- AddFileOptions(m_PrimeCacheOptions);
- AddOutputOptions(m_PrimeCacheOptions);
- AddCacheOptions(m_PrimeCacheOptions);
- AddWorkerOptions(m_PrimeCacheOptions);
- AddZenFolderOptions(m_PrimeCacheOptions);
- m_PrimeCacheOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- m_PrimeCacheOptions.add_option(
- "",
- "",
- "build-part-id",
- "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
- cxxopts::value(m_BuildPartIds),
- "<id>");
- m_PrimeCacheOptions.add_option("",
- "",
- "build-part-name",
- "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
- "all parts will be downloaded",
- cxxopts::value(m_BuildPartNames),
- "<name>");
-
- m_PrimeCacheOptions.add_option("",
- "",
- "force",
- "Force download of all blobs by ignoring any existing blobs in cache",
- cxxopts::value(m_Force),
- "<force>");
-
- m_PrimeCacheOptions.parse_positional({"build-id"});
- m_PrimeCacheOptions.positional_help("build-id");
-
- auto AddZenProcessOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
- };
- AddZenProcessOptions(m_PauseOptions);
- m_PauseOptions.parse_positional({"process-id"});
- m_PauseOptions.positional_help("process-id");
-
- AddZenProcessOptions(m_ResumeOptions);
- m_ResumeOptions.parse_positional({"process-id"});
- m_ResumeOptions.positional_help("process-id");
-
- AddZenProcessOptions(m_AbortOptions);
- m_AbortOptions.parse_positional({"process-id"});
- m_AbortOptions.positional_help("process-id");
-
- // validate-part
- AddSystemOptions(m_ValidateBuildPartOptions);
- AddCloudOptions(m_ValidateBuildPartOptions);
- AddFileOptions(m_ValidateBuildPartOptions);
- AddOutputOptions(m_ValidateBuildPartOptions);
- AddWorkerOptions(m_ValidateBuildPartOptions);
- AddZenFolderOptions(m_ValidateBuildPartOptions);
- m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- m_ValidateBuildPartOptions.add_option("",
- "",
- "build-part-id",
- "Build part Id, if not given it will be auto generated",
- cxxopts::value(m_BuildPartId),
- "<id>");
- m_ValidateBuildPartOptions.add_option(
- "",
- "",
- "build-part-name",
- "Name of the build part, if not given it will be be named after the directory name at end of local-path",
- cxxopts::value(m_BuildPartName),
- "<name>");
- m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"});
- m_ValidateBuildPartOptions.positional_help("build-id build-part-id");
-
- // multi-test-download
- AddSystemOptions(m_MultiTestDownloadOptions);
- AddCloudOptions(m_MultiTestDownloadOptions);
- AddFileOptions(m_MultiTestDownloadOptions);
- AddOutputOptions(m_MultiTestDownloadOptions);
- AddCacheOptions(m_MultiTestDownloadOptions);
- AddWorkerOptions(m_MultiTestDownloadOptions);
- m_MultiTestDownloadOptions
- .add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
- m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>");
- m_MultiTestDownloadOptions.add_option("",
- "",
- "enable-scavenge",
- "Enable scavenging of data from previouse download locations",
- cxxopts::value(m_EnableScavenging),
- "<scavenge>");
- m_MultiTestDownloadOptions.add_option("",
- "",
- "allow-file-clone",
- "Enable use of block reference counting when copying files",
- cxxopts::value(m_AllowFileClone),
- "<allowclone>");
- m_MultiTestDownloadOptions.parse_positional({"local-path"});
- m_MultiTestDownloadOptions.positional_help("local-path");
+BuildsCommand::BuildsCommand()
+: m_ListNamespacesSubCmd(*this)
+, m_ListSubCmd(*this)
+, m_ListBlocksSubCmd(*this)
+, m_UploadSubCmd(*this)
+, m_DownloadSubCmd(*this)
+, m_LsSubCmd(*this)
+, m_DiffSubCmd(*this)
+, m_FetchBlobSubCmd(*this)
+, m_PrimeCacheSubCmd(*this)
+, m_PauseSubCmd(*this)
+, m_ResumeSubCmd(*this)
+, m_AbortSubCmd(*this)
+, m_ValidatePartSubCmd(*this)
+, m_TestSubCmd(*this)
+, m_MultiTestDownloadSubCmd(*this)
+{
+ m_Options.add_options()("h,help", "Print help");
+ m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), "");
+ m_Options.parse_positional({"subcommand"});
+
+ AddSubCommand(m_ListNamespacesSubCmd);
+ AddSubCommand(m_ListSubCmd);
+ AddSubCommand(m_ListBlocksSubCmd);
+ AddSubCommand(m_UploadSubCmd);
+ AddSubCommand(m_DownloadSubCmd);
+ AddSubCommand(m_LsSubCmd);
+ AddSubCommand(m_DiffSubCmd);
+ AddSubCommand(m_FetchBlobSubCmd);
+ AddSubCommand(m_PrimeCacheSubCmd);
+ AddSubCommand(m_PauseSubCmd);
+ AddSubCommand(m_ResumeSubCmd);
+ AddSubCommand(m_AbortSubCmd);
+ AddSubCommand(m_ValidatePartSubCmd);
+ AddSubCommand(m_TestSubCmd);
+ AddSubCommand(m_MultiTestDownloadSubCmd);
}
BuildsCommand::~BuildsCommand() = default;
-void
-BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+bool
+BuildsCommand::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/)
{
using namespace builds_impl;
- ZEN_UNUSED(GlobalOptions);
signal(SIGINT, SignalCallbackHandler);
#if ZEN_PLATFORM_WINDOWS
signal(SIGBREAK, SignalCallbackHandler);
#endif // ZEN_PLATFORM_WINDOWS
- using namespace std::literals;
-
- std::vector<char*> SubCommandArguments;
- cxxopts::Options* SubOption = nullptr;
- int ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments);
- if (!ParseOptions(ParentCommandArgCount, argv))
+ // Validate output options
+ if (m_Verbose && m_Quiet)
{
- return;
+ throw OptionParseException("'--verbose' conflicts with '--quiet'", {});
}
-
- if (SubOption == nullptr)
+ if (m_LogProgress && m_PlainProgress)
{
- throw OptionParseException("'verb' option is required", m_Options.help());
+ throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", {});
+ }
+ if (m_LogProgress && m_Quiet)
+ {
+ throw OptionParseException("'--quiet' conflicts with '--log-progress'", {});
+ }
+ if (m_PlainProgress && m_Quiet)
+ {
+ throw OptionParseException("'--quiet' conflicts with '--plain-progress'", {});
+ }
+ IsVerbose = m_Verbose;
+ IsQuiet = m_Quiet;
+ if (m_LogProgress)
+ {
+ ProgressMode = ProgressBar::Mode::Log;
+ }
+ else if (m_PlainProgress)
+ {
+ ProgressMode = ProgressBar::Mode::Plain;
+ }
+ else if (m_Verbose)
+ {
+ ProgressMode = ProgressBar::Mode::Plain;
+ }
+ else if (IsQuiet)
+ {
+ ProgressMode = ProgressBar::Mode::Quiet;
+ }
+ else
+ {
+ ProgressMode = ProgressBar::Mode::Pretty;
}
- if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data()))
+ if (m_BoostWorkers)
{
- return;
+ m_BoostWorkerCount = true;
+ m_BoostWorkerMemory = true;
}
- auto ParseSystemOptions = [&]() {
- if (m_SystemRootDir.empty())
- {
- m_SystemRootDir = PickDefaultSystemRootDirectory();
- }
- MakeSafeAbsolutePathInPlace(m_SystemRootDir);
- };
- ParseSystemOptions();
+ // Parse system options
+ if (m_SystemRootDir.empty())
+ {
+ m_SystemRootDir = PickDefaultSystemRootDirectory();
+ }
+ MakeSafeAbsolutePathInPlace(m_SystemRootDir);
- auto ParseStorageOptions = [&](bool RequireNamespace, bool RequireBucket) {
- if (!m_Url.empty())
- {
- if (!m_Host.empty())
- {
- throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOption->help());
- }
- if (!m_Bucket.empty())
- {
- throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url),
- SubOption->help());
- }
- if (!m_BuildId.empty())
- {
- throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", m_BuildId, m_Url),
- SubOption->help());
- }
- if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, m_BuildId))
- {
- throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format",
- SubOption->help());
- }
- }
+ UseSparseFiles = m_UseSparseFiles;
- if (!m_OverrideHost.empty() || !m_Host.empty())
- {
- if (!m_StoragePath.empty())
- {
- throw OptionParseException(
- fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath),
- SubOption->help());
- }
- if (RequireNamespace && m_Namespace.empty())
- {
- throw OptionParseException("'--namespace' is required", SubOption->help());
- }
- if (RequireBucket && m_Bucket.empty())
- {
- throw OptionParseException("'--bucket' is required", SubOption->help());
- }
- }
- else if (m_StoragePath.empty())
- {
- throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help());
- }
- MakeSafeAbsolutePathInPlace(m_StoragePath);
- };
+ return true;
+}
- auto ParseOutputOptions = [&]() {
- if (m_Verbose && m_Quiet)
- {
- throw OptionParseException("'--verbose' conflicts with '--quiet'", SubOption->help());
- }
- if (m_LogProgress && m_PlainProgress)
- {
- throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", SubOption->help());
- }
- if (m_LogProgress && m_Quiet)
+void
+BuildsCommand::ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts)
+{
+ if (!m_Url.empty())
+ {
+ if (!m_Host.empty())
{
- throw OptionParseException("'--quiet' conflicts with '--log-progress'", SubOption->help());
+ throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOpts.help());
}
- if (m_PlainProgress && m_Quiet)
+ if (!m_Bucket.empty())
{
- throw OptionParseException("'--quiet' conflicts with '--plain-progress'", SubOption->help());
+ throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), SubOpts.help());
}
- IsVerbose = m_Verbose;
- IsQuiet = m_Quiet;
- if (m_LogProgress)
+ if (!BuildId.empty())
{
- ProgressMode = ProgressBar::Mode::Log;
+ throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Url), SubOpts.help());
}
- else if (m_PlainProgress)
+ if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, BuildId))
{
- ProgressMode = ProgressBar::Mode::Plain;
+ throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOpts.help());
}
- else if (m_Verbose)
+ }
+
+ if (!m_OverrideHost.empty() || !m_Host.empty())
+ {
+ if (!m_StoragePath.empty())
{
- ProgressMode = ProgressBar::Mode::Plain;
+ throw OptionParseException(
+ fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath),
+ SubOpts.help());
}
- else if (IsQuiet)
+ if (RequireNamespace && m_Namespace.empty())
{
- ProgressMode = ProgressBar::Mode::Quiet;
+ throw OptionParseException("'--namespace' is required", SubOpts.help());
}
- else
+ if (RequireBucket && m_Bucket.empty())
{
- ProgressMode = ProgressBar::Mode::Pretty;
+ throw OptionParseException("'--bucket' is required", SubOpts.help());
}
- };
- ParseOutputOptions();
-
- if (m_BoostWorkers)
+ }
+ else if (m_StoragePath.empty())
{
- m_BoostWorkerCount = true;
- m_BoostWorkerMemory = true;
+ throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help());
}
+ MakeSafeAbsolutePathInPlace(m_StoragePath);
+}
- std::unique_ptr<AuthMgr> Auth;
-
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
-
- auto CreateBuildStorage = [&](BuildStorageBase::Statistics& StorageStats,
+StorageInstance
+BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
BuildStorageCache::Statistics& StorageCacheStats,
const std::filesystem::path& TempPath,
+ std::string& BuildId,
bool RequireNamespace,
bool RequireBucket,
- bool BoostCacheBackgroundWorkerPool) -> StorageInstance {
- ParseStorageOptions(RequireNamespace, RequireBucket);
+ bool BoostCacheBackgroundWorkerPool,
+ std::unique_ptr<AuthMgr>& Auth,
+ cxxopts::Options& SubOpts)
+{
+ using namespace builds_impl;
- HttpClientSettings ClientSettings{
- .LogCategory = "httpbuildsclient",
- .AssumeHttp2 = m_AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 2,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)};
+ ParseStorageOptions(BuildId, RequireNamespace, RequireBucket, SubOpts);
- std::string StorageDescription;
- std::string CacheDescription;
+ HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
+ .AssumeHttp2 = m_AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 2,
+ .Verbose = m_VerboseHttp,
+ .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)};
- StorageInstance Result;
+ std::string StorageDescription;
+ std::string CacheDescription;
- if (!m_Host.empty() || !m_OverrideHost.empty())
- {
- m_AuthOptions.ParseOptions(*SubOption,
- m_SystemRootDir,
- ClientSettings,
- m_Host.empty() ? m_OverrideHost : m_Host,
- Auth,
- IsQuiet,
- /*Hidden*/ false,
- m_Verbose);
-
- BuildStorageResolveResult ResolveRes =
- ResolveBuildStorage(*Output, ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose);
- if (!ResolveRes.Cloud.Address.empty())
+ StorageInstance Result;
+
+ if (!m_Host.empty() || !m_OverrideHost.empty())
+ {
+ m_AuthOptions.ParseOptions(SubOpts,
+ m_SystemRootDir,
+ ClientSettings,
+ m_Host.empty() ? m_OverrideHost : m_Host,
+ Auth,
+ IsQuiet,
+ /*Hidden*/ false,
+ m_Verbose);
+
+ BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*CreateConsoleLogOutput(ProgressMode),
+ ClientSettings,
+ m_Host,
+ m_OverrideHost,
+ m_ZenCacheHost,
+ ZenCacheResolveMode::All,
+ m_Verbose);
+ if (!ResolveRes.Cloud.Address.empty())
+ {
+ ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2;
+ Result.BuildStorageHttp =
+ std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); });
+
+ Result.BuildStorage = CreateJupiterBuildStorage(Log(),
+ *Result.BuildStorageHttp,
+ StorageStats,
+ m_Namespace,
+ m_Bucket,
+ m_AllowRedirect,
+ TempPath / "storage");
+ Result.BuildStorageHost = ResolveRes.Cloud;
+
+ uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0;
+
+ StorageDescription =
+ fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}",
+ ResolveRes.Cloud.Name,
+ (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address),
+ Result.BuildStorageHttp->GetSessionId(),
+ m_Namespace,
+ m_Bucket,
+ NiceLatencyNs(HostLatencyNs));
+
+ if (!ResolveRes.Cache.Address.empty())
{
- ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2;
- Result.BuildStorageHttp =
- std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); });
-
- Result.BuildStorage = CreateJupiterBuildStorage(Log(),
- *Result.BuildStorageHttp,
- StorageStats,
- m_Namespace,
- m_Bucket,
- m_AllowRedirect,
- TempPath / "storage");
- Result.BuildStorageHost = ResolveRes.Cloud;
-
- uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0;
-
- StorageDescription =
- fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}",
- ResolveRes.Cloud.Name,
- (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address),
- Result.BuildStorageHttp->GetSessionId(),
- m_Namespace,
- m_Bucket,
- NiceLatencyNs(HostLatencyNs));
-
- if (!ResolveRes.Cache.Address.empty())
+ Result.CacheHttp = std::make_unique<HttpClient>(
+ ResolveRes.Cache.Address,
+ HttpClientSettings{
+ .LogCategory = "httpcacheclient",
+ .ConnectTimeout = std::chrono::milliseconds{3000},
+ .Timeout = std::chrono::milliseconds{30000},
+ .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 0,
+ .Verbose = m_VerboseHttp,
+ .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
+ []() { return AbortFlag.load(); });
+ Result.CacheStorage =
+ CreateZenBuildStorageCache(*Result.CacheHttp,
+ StorageCacheStats,
+ m_Namespace,
+ m_Bucket,
+ TempPath / "zencache",
+ BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
+ : GetTinyWorkerPool(EWorkloadType::Background));
+ Result.CacheHost = ResolveRes.Cache;
+
+ uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0;
+
+ CacheDescription =
+ fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}",
+ ResolveRes.Cache.Name,
+ (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address),
+ Result.CacheHttp->GetSessionId(),
+ NiceLatencyNs(CacheLatencyNs));
+
+ if (!m_Namespace.empty())
{
- Result.CacheHttp = std::make_unique<HttpClient>(
- ResolveRes.Cache.Address,
- HttpClientSettings{
- .LogCategory = "httpcacheclient",
- .ConnectTimeout = std::chrono::milliseconds{3000},
- .Timeout = std::chrono::milliseconds{30000},
- .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 0,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
- []() { return AbortFlag.load(); });
- Result.CacheStorage =
- CreateZenBuildStorageCache(*Result.CacheHttp,
- StorageCacheStats,
- m_Namespace,
- m_Bucket,
- TempPath / "zencache",
- BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
- : GetTinyWorkerPool(EWorkloadType::Background));
- Result.CacheHost = ResolveRes.Cache;
-
- uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0;
-
- CacheDescription =
- fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}",
- ResolveRes.Cache.Name,
- (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address),
- Result.CacheHttp->GetSessionId(),
- NiceLatencyNs(CacheLatencyNs));
-
- if (!m_Namespace.empty())
- {
- CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
- }
- if (!m_Bucket.empty())
- {
- CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
- }
+ CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
}
- }
- }
- else if (!m_StoragePath.empty())
- {
- StorageDescription = fmt::format("folder {}", m_StoragePath);
- Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
-
- Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(),
- .Name = "Disk",
- .LatencySec = 1.0 / 100000, // 1 us
- .Caps = {.MaxRangeCountPerRequest = 2048u}};
-
- if (!m_ZenCacheHost.empty())
- {
- ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp);
-
- if (TestResult.Success)
+ if (!m_Bucket.empty())
{
- Result.CacheHttp = std::make_unique<HttpClient>(
- m_ZenCacheHost,
- HttpClientSettings{
- .LogCategory = "httpcacheclient",
- .ConnectTimeout = std::chrono::milliseconds{3000},
- .Timeout = std::chrono::milliseconds{30000},
- .AssumeHttp2 = m_AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 0,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
- []() { return AbortFlag.load(); });
-
- Result.CacheStorage =
- CreateZenBuildStorageCache(*Result.CacheHttp,
- StorageCacheStats,
- m_Namespace,
- m_Bucket,
- TempPath / "zencache",
- BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
- : GetTinyWorkerPool(EWorkloadType::Background));
- Result.CacheHost =
- BuildStorageResolveResult::Host{.Address = m_ZenCacheHost,
- .Name = m_ZenCacheHost,
- .AssumeHttp2 = m_AssumeHttp2,
- .LatencySec = TestResult.LatencySeconds,
- .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}};
-
- CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId());
-
- if (!m_Namespace.empty())
- {
- CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
- }
- if (!m_Bucket.empty())
- {
- CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
- }
+ CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
}
}
}
- else
- {
- throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help());
- }
+ }
+ else if (!m_StoragePath.empty())
+ {
+ StorageDescription = fmt::format("folder {}", m_StoragePath);
+ Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Remote: {}", StorageDescription);
- if (!Result.CacheHost.Name.empty())
- {
- ZEN_CONSOLE("Cache : {}", CacheDescription);
- }
- }
- return Result;
- };
+ Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(),
+ .Name = "Disk",
+ .LatencySec = 1.0 / 100000, // 1 us
+ .Caps = {.MaxRangeCountPerRequest = 2048u}};
- auto ParsePath = [&]() {
- if (m_Path.empty())
+ if (!m_ZenCacheHost.empty())
{
- throw OptionParseException("'--local-path' is required", SubOption->help());
- }
- MakeSafeAbsolutePathInPlace(m_Path);
- };
+ ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp);
- auto ParseFileFilters = [&](std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) {
- auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) {
- ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) {
- if (!Wildcard.empty())
+ if (TestResult.Success)
+ {
+ Result.CacheHttp = std::make_unique<HttpClient>(
+ m_ZenCacheHost,
+ HttpClientSettings{
+ .LogCategory = "httpcacheclient",
+ .ConnectTimeout = std::chrono::milliseconds{3000},
+ .Timeout = std::chrono::milliseconds{30000},
+ .AssumeHttp2 = m_AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 0,
+ .Verbose = m_VerboseHttp,
+ .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
+ []() { return AbortFlag.load(); });
+
+ Result.CacheStorage =
+ CreateZenBuildStorageCache(*Result.CacheHttp,
+ StorageCacheStats,
+ m_Namespace,
+ m_Bucket,
+ TempPath / "zencache",
+ BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
+ : GetTinyWorkerPool(EWorkloadType::Background));
+ Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_ZenCacheHost,
+ .Name = m_ZenCacheHost,
+ .AssumeHttp2 = m_AssumeHttp2,
+ .LatencySec = TestResult.LatencySeconds,
+ .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}};
+
+ CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId());
+
+ if (!m_Namespace.empty())
{
- std::string CleanWildcard(ToLower(Wildcard));
- for (auto It = begin(CleanWildcard); It != end(CleanWildcard); It++)
- {
- if (*It == '\\')
- {
- *It = '/';
- }
- }
-
- if (CleanWildcard.starts_with("./") || CleanWildcard.starts_with(".\\"))
- {
- CleanWildcard = CleanWildcard.substr(2);
- }
-
- Output.emplace_back(std::move(CleanWildcard));
+ CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
}
- return true;
- });
- };
-
- SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards);
- SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards);
- };
-
- auto ParseExcludeFolderAndExtension = [&](std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) {
- auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) {
- ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) {
- if (!Exclusion.empty())
+ if (!m_Bucket.empty())
{
- std::string CleanExclusion(ToLower(Exclusion));
- if (CleanExclusion.length() > 2 && CleanExclusion.front() == '"' && CleanExclusion.back() == '"')
- {
- CleanExclusion = CleanExclusion.substr(1, CleanExclusion.length() - 2);
- }
- Output.emplace_back(std::move(CleanExclusion));
+ CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
}
- return true;
- });
- };
-
- SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders);
- SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions);
- };
-
- auto ParseDiffPath = [&]() {
- if (m_DiffPath.empty())
- {
- throw OptionParseException("'--compare-path' is required", SubOption->help());
+ }
}
- MakeSafeAbsolutePathInPlace(m_DiffPath);
- };
+ }
+ else
+ {
+ throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help());
+ }
- auto ParseBlobHash = [&]() -> IoHash {
- if (m_BlobHash.empty())
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Remote: {}", StorageDescription);
+ if (!Result.CacheHost.Name.empty())
{
- throw OptionParseException("'--blob-hash' is required", SubOption->help());
+ ZEN_CONSOLE("Cache : {}", CacheDescription);
}
+ }
+ return Result;
+}
- if (m_BlobHash.length() != IoHash::StringLength)
- {
- throw OptionParseException(
- fmt::format("'--blob-hash' ('{}') is malformed, it must be {} characters long", m_BlobHash, IoHash::StringLength),
- SubOption->help());
- }
+Oid
+BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts)
+{
+ if (BuildIdStr.length() != Oid::StringLength)
+ {
+ throw OptionParseException(
+ fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildIdStr, Oid::StringLength),
+ SubOpts.help());
+ }
+ else if (Oid BuildId = Oid::FromHexString(BuildIdStr); BuildId == Oid::Zero)
+ {
+ throw OptionParseException(fmt::format("'--build-id' ('{}') is invalid", BuildIdStr), SubOpts.help());
+ }
+ else
+ {
+ return BuildId;
+ }
+}
+
+Oid
+BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts)
+{
+ if (BuildPartIdStr.length() != Oid::StringLength)
+ {
+ throw OptionParseException(
+ fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildPartIdStr, Oid::StringLength),
+ SubOpts.help());
+ }
+ else if (Oid BuildPartId = Oid::FromHexString(BuildPartIdStr); BuildPartId == Oid::Zero)
+ {
+ throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildPartIdStr), SubOpts.help());
+ }
+ else
+ {
+ return BuildPartId;
+ }
+}
- IoHash BlobHash;
- if (!IoHash::TryParse(m_BlobHash, BlobHash))
+std::vector<Oid>
+BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts)
+{
+ std::vector<Oid> BuildPartIds;
+ for (const std::string& BuildPartId : BuildPartIdStrs)
+ {
+ BuildPartIds.push_back(Oid::TryFromHexString(RemoveQuotes(BuildPartId)));
+ if (BuildPartIds.back() == Oid::Zero)
{
- throw OptionParseException(fmt::format("'--blob-hash' ('{}') is malformed", m_BlobHash), SubOption->help());
+ throw OptionParseException(fmt::format("'--build-part-id' ('{}') is malformed", BuildPartId), SubOpts.help());
}
+ }
+ return BuildPartIds;
+}
- return BlobHash;
- };
-
- auto ParseBuildId = [&]() -> Oid {
- if (m_BuildId.length() != Oid::StringLength)
+std::vector<std::string>
+BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts)
+{
+ std::vector<std::string> BuildPartNames;
+ for (const std::string& BuildPartName : BuildPartNameStrs)
+ {
+ BuildPartNames.push_back(std::string(RemoveQuotes(BuildPartName)));
+ if (BuildPartNames.back().empty())
{
- throw OptionParseException(
- fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildId, Oid::StringLength),
- SubOption->help());
+ throw OptionParseException(fmt::format("'--build-part-names' ('{}') is invalid", BuildPartName), SubOpts.help());
}
- else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero)
+ }
+ return BuildPartNames;
+}
+
+CbObject
+BuildsCommand::ParseBuildMetadata(bool CreateBuild,
+ std::filesystem::path& BuildMetadataPath,
+ const std::string& BuildMetadata,
+ cxxopts::Options& SubOpts)
+{
+ if (CreateBuild)
+ {
+ if (BuildMetadataPath.empty() && BuildMetadata.empty())
{
- throw OptionParseException(fmt::format("'--build-id' ('{}') is invalid", m_BuildId), SubOption->help());
+ throw OptionParseException("'--metadata-path' or '--metadata' is required", SubOpts.help());
}
- else
+ if (!BuildMetadataPath.empty() && !BuildMetadata.empty())
{
- return BuildId;
+ throw OptionParseException(
+ fmt::format("'--metadata-path' ('{}') conflicts with '--metadata' ('{}')", BuildMetadataPath.string(), BuildMetadata),
+ SubOpts.help());
}
- };
- auto ParseBuildPartId = [&]() -> Oid {
- if (m_BuildPartId.length() != Oid::StringLength)
+ if (!BuildMetadataPath.empty())
{
- throw OptionParseException(
- fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildPartId, Oid::StringLength),
- SubOption->help());
+ MakeSafeAbsolutePathInPlace(BuildMetadataPath);
+ IoBuffer MetaDataJson = ReadFile(BuildMetadataPath).Flatten();
+ std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+ std::string JsonError;
+ CbObject MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+ if (!JsonError.empty())
+ {
+ throw std::runtime_error(
+ fmt::format("build metadata file '{}' is malformed. Reason: '{}'", BuildMetadataPath.string(), JsonError));
+ }
+ return MetaData;
}
- else if (Oid BuildPartId = Oid::FromHexString(m_BuildPartId); BuildPartId == Oid::Zero)
+ if (!BuildMetadata.empty())
{
- throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", m_BuildPartId), SubOption->help());
+ CbObjectWriter MetaDataWriter(1024);
+ ForEachStrTok(BuildMetadata, ';', [&](std::string_view Pair) {
+ size_t SplitPos = Pair.find('=');
+ if (SplitPos == std::string::npos || SplitPos == 0)
+ {
+ throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair));
+ }
+ MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1));
+ return true;
+ });
+ return MetaDataWriter.Save();
}
- else
+ }
+ else
+ {
+ if (!BuildMetadataPath.empty())
{
- return BuildPartId;
+ throw OptionParseException("'--metadata-path' requires '--create-build'", SubOpts.help());
}
- };
-
- auto ParseBuildPartIds = [&]() -> std::vector<Oid> {
- std::vector<Oid> BuildPartIds;
- for (const std::string& BuildPartId : m_BuildPartIds)
+ if (!BuildMetadata.empty())
{
- BuildPartIds.push_back(Oid::TryFromHexString(RemoveQuotes(BuildPartId)));
- if (BuildPartIds.back() == Oid::Zero)
- {
- throw OptionParseException(fmt::format("'--build-part-id' ('{}') is malformed", BuildPartId), SubOption->help());
- }
+ throw OptionParseException("'--metadata' requires '--create-build'", SubOpts.help());
}
- return BuildPartIds;
- };
+ }
+ return {};
+}
- auto ParseBuildPartNames = [&]() -> std::vector<std::string> {
- std::vector<std::string> BuildPartNames;
- for (const std::string& BuildPartName : m_BuildPartNames)
- {
- BuildPartNames.push_back(std::string(RemoveQuotes(BuildPartName)));
- if (BuildPartNames.back().empty())
+void
+BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts)
+{
+ if (Path.empty())
+ {
+ throw OptionParseException("'--local-path' is required", SubOpts.help());
+ }
+ MakeSafeAbsolutePathInPlace(Path);
+}
+
+IoHash
+BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts)
+{
+ if (BlobHashStr.empty())
+ {
+ throw OptionParseException("'--blob-hash' is required", SubOpts.help());
+ }
+
+ if (BlobHashStr.length() != IoHash::StringLength)
+ {
+ throw OptionParseException(
+ fmt::format("'--blob-hash' ('{}') is malformed, it must be {} characters long", BlobHashStr, IoHash::StringLength),
+ SubOpts.help());
+ }
+
+ IoHash BlobHash;
+ if (!IoHash::TryParse(BlobHashStr, BlobHash))
+ {
+ throw OptionParseException(fmt::format("'--blob-hash' ('{}') is malformed", BlobHashStr), SubOpts.help());
+ }
+
+ return BlobHash;
+}
+
+void
+BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards)
+{
+ auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) {
+ ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) {
+ if (!Wildcard.empty())
{
- throw OptionParseException(fmt::format("'--build-part-names' ('{}') is invalid", BuildPartName), SubOption->help());
+ std::string CleanWildcard(ToLower(Wildcard));
+ for (auto It = begin(CleanWildcard); It != end(CleanWildcard); It++)
+ {
+ if (*It == '\\')
+ {
+ *It = '/';
+ }
+ }
+
+ if (CleanWildcard.starts_with("./") || CleanWildcard.starts_with(".\\"))
+ {
+ CleanWildcard = CleanWildcard.substr(2);
+ }
+
+ Output.emplace_back(std::move(CleanWildcard));
}
- }
- return BuildPartNames;
+ return true;
+ });
};
- auto ParseBuildMetadata = [&]() -> CbObject {
- if (m_CreateBuild)
- {
- if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty())
- {
- throw OptionParseException("'--metadata-path' or '--metadata' is required", SubOption->help());
- }
- if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty())
- {
- throw OptionParseException(
- fmt::format("'--metadata-path' ('{}') conflicts with '--metadata' ('{}')", m_BuildMetadataPath, m_BuildMetadata),
- SubOption->help());
- }
+ SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards);
+ SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards);
+}
- if (!m_BuildMetadataPath.empty())
+void
+BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions)
+{
+ auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) {
+ ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) {
+ if (!Exclusion.empty())
{
- MakeSafeAbsolutePathInPlace(m_BuildMetadataPath);
- IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten();
- std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
- std::string JsonError;
- CbObject MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
- if (!JsonError.empty())
+ std::string CleanExclusion(ToLower(Exclusion));
+ if (CleanExclusion.length() > 2 && CleanExclusion.front() == '"' && CleanExclusion.back() == '"')
{
- throw std::runtime_error(
- fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError));
+ CleanExclusion = CleanExclusion.substr(1, CleanExclusion.length() - 2);
}
- return MetaData;
- }
- if (!m_BuildMetadata.empty())
- {
- CbObjectWriter MetaDataWriter(1024);
- ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) {
- size_t SplitPos = Pair.find('=');
- if (SplitPos == std::string::npos || SplitPos == 0)
- {
- throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair));
- }
- MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1));
- return true;
- });
- return MetaDataWriter.Save();
+ Output.emplace_back(std::move(CleanExclusion));
}
+ return true;
+ });
+ };
+
+ SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders);
+ SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions);
+}
+
+void
+BuildsCommand::ResolveZenFolderPath(const std::filesystem::path& DefaultPath)
+{
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = DefaultPath;
+ }
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+}
+
+EPartialBlockRequestMode
+BuildsCommand::ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts)
+{
+ if (PrimeCacheOnly)
+ {
+ return EPartialBlockRequestMode::Off;
+ }
+ EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests);
+ if (Mode == EPartialBlockRequestMode::Invalid)
+ {
+ throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests),
+ SubOpts.help());
+ }
+ return Mode;
+}
+
+void
+BuildsCommand::ParseZenProcessId(int& ZenProcessId)
+{
+ if (ZenProcessId == -1)
+ {
+ const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath();
+ ProcessHandle RunningProcess;
+ std::error_code Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false);
+ if (Ec)
+ {
+ throw std::runtime_error(fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message()));
}
- else
+ if (!RunningProcess.IsValid())
{
- if (!m_BuildMetadataPath.empty())
- {
- throw OptionParseException("'--metadata-path' requires '--create-build'", SubOption->help());
- }
- if (!m_BuildMetadata.empty())
- {
- throw OptionParseException("'--metadata' requires '--create-build'", SubOption->help());
- }
+ throw std::runtime_error(fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath));
}
- return {};
- };
+ ZenProcessId = RunningProcess.Pid();
+ }
+}
- UseSparseFiles = m_UseSparseFiles;
+//////////////////////////////////////////////////////////////////////////
+
+// ---------------------------------------------------------------------------
+// Subcommand implementations
+// ---------------------------------------------------------------------------
+
+BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent)
+: ZenSubCmdBase("list-namespaces", "List all namespaces and optionally their buckets")
+, m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddZenFolderOptions(Opts);
+ Opts.add_option("", "", "recursive", "Enable fetch of buckets within namespaces also", cxxopts::value(m_Recursive), "<recursive>");
+ Opts.add_option("",
+ "",
+ "result-path",
+ "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
+ cxxopts::value(m_ResultPath),
+ "<result-path>");
+ Opts.parse_positional({"result-path"});
+ Opts.positional_help("result-path");
+}
+
+void
+BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
- if (SubOption == &m_ListNamespacesOptions)
+ if (!m_ResultPath.empty())
{
- if (!m_ListResultPath.empty())
+ if (!IsQuiet)
{
- if (!IsQuiet)
- {
- LogExecutableVersionAndPid();
- }
+ ZenCmdBase::LogExecutableVersionAndPid();
}
+ }
+
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
+
+ m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
- ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
- : MakeSafeAbsolutePath(m_ZenFolderPath);
- CreateDirectories(ZenFolderPath);
- auto _ = MakeGuard([ZenFolderPath]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), ZenFolderPath); });
-
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(ZenFolderPath),
- /*RequriesNamespace*/ false,
- /*RequireBucket*/ false,
- /*BoostCacheBackgroundWorkerPool */ false);
-
- CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive);
- ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
- if (m_ListResultPath.empty())
+ CreateDirectories(m_Parent.GetZenFolderPath());
+ auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); });
+
+ std::unique_ptr<AuthMgr> Auth;
+ std::string DummyBuildId;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ DummyBuildId,
+ /*RequireNamespace*/ false,
+ /*RequireBucket*/ false,
+ /*BoostCacheBackgroundWorkerPool*/ false,
+ Auth,
+ Opts);
+
+ CbObject Response = Storage.BuildStorage->ListNamespaces(m_Recursive);
+ ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
+ if (m_ResultPath.empty())
+ {
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(Response.GetView(), SB);
+ ZEN_CONSOLE("{}", SB.ToView());
+ }
+ else
+ {
+ std::filesystem::path ResultPath = MakeSafeAbsolutePath(m_ResultPath);
+ if (ToLower(ResultPath.extension().string()) == ".cbo")
{
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- ZEN_CONSOLE("{}", SB.ToView());
+ MemoryView ResponseView = Response.GetView();
+ WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
}
else
{
- std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath);
- if (ToLower(ListResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(Response.GetView(), SB);
+ WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
}
- return;
}
+}
- if (SubOption == &m_ListOptions)
- {
- MakeSafeAbsolutePathInPlace(m_ListQueryPath);
- MakeSafeAbsolutePathInPlace(m_ListResultPath);
+BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list", "List builds matching a query"), m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddZenFolderOptions(Opts);
+ Opts.add_option("",
+ "",
+ "query-path",
+ "Path to json or compactbinary file containing list query",
+ cxxopts::value(m_QueryPath),
+ "<query-path>");
+ Opts.add_option("",
+ "",
+ "result-path",
+ "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
+ cxxopts::value(m_ResultPath),
+ "<result-path>");
+ Opts.parse_positional({"query-path", "result-path"});
+ Opts.positional_help("query-path result-path");
+}
+
+void
+BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
+
+ MakeSafeAbsolutePathInPlace(m_QueryPath);
+ MakeSafeAbsolutePathInPlace(m_ResultPath);
- if (!m_ListResultPath.empty())
+ if (!m_ResultPath.empty())
+ {
+ if (!IsQuiet)
{
- if (!IsQuiet)
- {
- LogExecutableVersionAndPid();
- }
+ ZenCmdBase::LogExecutableVersionAndPid();
}
- std::string JsonQuery;
- if (m_ListQueryPath.empty())
+ }
+ std::string JsonQuery;
+ if (m_QueryPath.empty())
+ {
+ CbObjectWriter QueryWriter;
+ QueryWriter.BeginObject("query");
+ QueryWriter.EndObject(); // query
+ CbObject QueryObject = QueryWriter.Save();
+ ExtendableStringBuilder<64> SB;
+ CompactBinaryToJson(QueryObject, SB);
+ JsonQuery = SB.ToString();
+ }
+ else
+ {
+ if (ToLower(m_QueryPath.extension().string()) == ".cbo")
{
- CbObjectWriter QueryWriter;
- QueryWriter.BeginObject("query");
- QueryWriter.EndObject(); // query
- CbObject QueryObject = QueryWriter.Save();
+ CbObject QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_QueryPath));
ExtendableStringBuilder<64> SB;
CompactBinaryToJson(QueryObject, SB);
JsonQuery = SB.ToString();
}
else
{
- if (ToLower(m_ListQueryPath.extension().string()) == ".cbo")
+ IoBuffer MetaDataJson = ReadFile(m_QueryPath).Flatten();
+ std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+ std::string JsonError;
+ CbObject QueryObject = LoadCompactBinaryFromJson(Json, JsonError)
+ .AsObject(); // We try to convert it so it is at least reaonably verified in format
+ if (!JsonError.empty())
{
- CbObject QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath));
- ExtendableStringBuilder<64> SB;
- CompactBinaryToJson(QueryObject, SB);
- JsonQuery = SB.ToString();
- }
- else
- {
- IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten();
- std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
- std::string JsonError;
- CbObject QueryObject = LoadCompactBinaryFromJson(Json, JsonError)
- .AsObject(); // We try to convert it so it is at least reaonably verified in format
- if (!JsonError.empty())
- {
- throw std::runtime_error(
- fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError));
- }
- JsonQuery = std::string(Json);
+ throw std::runtime_error(
+ fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_QueryPath.string(), JsonError));
}
+ JsonQuery = std::string(Json);
}
+ }
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
- }
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); });
+ m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ false,
- /*BoostCacheBackgroundWorkerPool */ false);
+ CreateDirectories(m_Parent.GetZenFolderPath());
+ auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); });
- CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery);
- ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
- if (m_ListResultPath.empty())
+ std::unique_ptr<AuthMgr> Auth;
+ std::string DummyBuildId;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ DummyBuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ false,
+ /*BoostCacheBackgroundWorkerPool*/ false,
+ Auth,
+ Opts);
+
+ CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery);
+ ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
+ if (m_ResultPath.empty())
+ {
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(Response.GetView(), SB);
+ ZEN_CONSOLE("{}", SB.ToView());
+ }
+ else
+ {
+ if (ToLower(m_ResultPath.extension().string()) == ".cbo")
{
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- ZEN_CONSOLE("{}", SB.ToView());
+ MemoryView ResponseView = Response.GetView();
+ WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
}
else
{
- if (ToLower(m_ListResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(Response.GetView(), SB);
+ WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
}
- return;
}
+}
- if (SubOption == &m_ListBlocksOptions)
- {
- MakeSafeAbsolutePathInPlace(m_ListResultPath);
-
- if (!m_ListResultPath.empty())
- {
- if (!IsQuiet)
- {
- LogExecutableVersionAndPid();
- }
- }
+BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent)
+: ZenSubCmdBase("list-blocks", "List blocks for a build")
+, m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddZenFolderOptions(Opts);
+ Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ Opts.add_option("",
+ "",
+ "result-path",
+ "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
+ cxxopts::value(m_ResultPath),
+ "<result-path>");
+ Opts.add_option("", "", "max-count", "Maximum number of blocks to list", cxxopts::value(m_MaxCount), "<maxcount>");
+ Opts.parse_positional({"build-id"});
+ Opts.positional_help("build-id");
+}
- if (m_ListBlocksMaxCount == 0)
- {
- throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_ListBlocksMaxCount), SubOption->help());
- }
+void
+BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ MakeSafeAbsolutePathInPlace(m_ResultPath);
- if (m_ZenFolderPath.empty())
+ if (!m_ResultPath.empty())
+ {
+ if (!IsQuiet)
{
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ ZenCmdBase::LogExecutableVersionAndPid();
}
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ }
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); });
+ if (m_MaxCount == 0)
+ {
+ throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_MaxCount), Opts.help());
+ }
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool */ false);
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- const Oid BuildId = ParseBuildId();
+ m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
- CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_ListBlocksMaxCount);
- ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
+ CreateDirectories(m_Parent.GetZenFolderPath());
+ auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); });
- std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response);
- if (!IsQuiet)
+ std::unique_ptr<AuthMgr> Auth;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ m_BuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ true,
+ /*BoostCacheBackgroundWorkerPool*/ false,
+ Auth,
+ Opts);
+
+ const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+
+ CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_MaxCount);
+ ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
+
+ std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size());
+ }
+ if (m_ResultPath.empty())
+ {
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(Response.GetView(), SB);
+ ForEachStrTok(SB.ToView(), '\n', [](std::string_view Row) {
+ ZEN_CONSOLE("{}", Row);
+ return true;
+ });
+ }
+ else
+ {
+ if (ToLower(m_ResultPath.extension().string()) == ".cbo")
{
- ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size());
+ MemoryView ResponseView = Response.GetView();
+ WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
}
- if (m_ListResultPath.empty())
+ else
{
ExtendableStringBuilder<1024> SB;
CompactBinaryToJson(Response.GetView(), SB);
- ForEachStrTok(SB.ToView(), '\n', [](std::string_view Row) {
- ZEN_CONSOLE("{}", Row);
- return true;
- });
- }
- else
- {
- if (ToLower(m_ListResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
}
- return;
}
- if (SubOption == &m_UploadOptions)
- {
- if (!IsQuiet)
- {
- LogExecutableVersionAndPid();
- }
+}
- TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent)
+: ZenSubCmdBase("upload", "Upload a folder to build storage")
+, m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddCacheOptions(Opts);
+ Parent.AddWorkerOptions(Opts);
+ Parent.AddZenFolderOptions(Opts);
+ Parent.AddExcludeFolderOption(Opts);
+ Parent.AddExcludeExtensionsOption(Opts);
+ Parent.AddChunkingCacheOptions(Opts);
+ Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
+ Opts.add_option("",
+ "",
+ "create-build",
+ "Set to true to create the containing build, if unset a builds-id must be given and the build already exist",
+ cxxopts::value(m_CreateBuild),
+ "<id>");
+ Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-id",
+ "Build part Id, if not given it will be auto generated",
+ cxxopts::value(m_BuildPartId),
+ "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-name",
+ "Name of the build part, if not given it will be be named after the directory name at end of local-path",
+ cxxopts::value(m_BuildPartName),
+ "<name>");
+ Opts.add_option("",
+ "",
+ "metadata-path",
+ "Path to json file that holds the metadata for the build. Requires the create-build option to be set",
+ cxxopts::value(m_BuildMetadataPath),
+ "<metadata-path>");
+ Opts.add_option(
+ "",
+ "",
+ "metadata",
+ "Key-value pairs separated by ';' with build meta data. (key1=value1;key2=value2). Requires the create-build option to be set",
+ cxxopts::value(m_BuildMetadata),
+ "<metadata>");
+ Opts.add_option("", "", "clean", "Ignore existing blocks", cxxopts::value(m_Clean), "<clean>");
+ Opts.add_option("",
+ "",
+ "block-min-reuse",
+ "Percent of an existing block that must be relevant for it to be resused. Defaults to 85.",
+ cxxopts::value(m_BlockReuseMinPercentLimit),
+ "<minreuse>");
+ Opts.add_option("cache",
+ "",
+ "zen-cache-upload",
+ "Upload data downloaded from remote host to zen cache",
+ cxxopts::value(m_UploadToZenCache),
+ "<uploadtozencache>");
+
+ Parent.AddMultipartOptions(Opts);
+
+ Opts.add_option("",
+ "",
+ "manifest-path",
+ "Path to a text file with one line of <local path>[TAB]<modification date> per file to include or a "
+ "structured .json file describing the parts",
+ cxxopts::value(m_ManifestPath),
+ "<manifestpath>");
+
+ Opts.add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), "<verify>");
+ Opts.add_option("",
+ "",
+ "find-max-block-count",
+ "The maximum number of blocks we search for in the build context",
+ cxxopts::value(m_FindBlockMaxCount),
+ "<maxblockcount>");
+
+ Opts.parse_positional({"local-path", "build-id"});
+ Opts.positional_help("local-path build-id");
+}
- ZenState InstanceState;
+void
+BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
- ParsePath();
+ if (!IsQuiet)
+ {
+ ZenCmdBase::LogExecutableVersionAndPid();
+ }
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
+ }
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
- }
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
- MakeSafeAbsolutePathInPlace(m_ChunkingCachePath);
+ ZenState InstanceState;
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });
+ m_Parent.ParsePath(m_Path, Opts);
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool */ false);
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- if (m_BuildPartName.empty() && m_ManifestPath.empty())
- {
- m_BuildPartName = m_Path.filename().string();
- }
+ m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
+ MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath);
- const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId();
- if (m_BuildId.empty())
- {
- m_BuildId = BuildId.ToString();
- }
+ CreateDirectories(m_Parent.GetZenFolderPath());
+ auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
- Oid BuildPartId;
- if (!m_BuildPartId.empty())
- {
- BuildPartId = ParseBuildPartId();
- }
+ std::unique_ptr<AuthMgr> Auth;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ m_BuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ true,
+ /*BoostCacheBackgroundWorkerPool*/ false,
+ Auth,
+ Opts);
+
+ if (m_BuildPartName.empty() && m_ManifestPath.empty())
+ {
+ m_BuildPartName = m_Path.filename().string();
+ }
- CbObject MetaData = ParseBuildMetadata();
+ const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : m_Parent.ParseBuildId(m_BuildId, Opts);
+ if (m_BuildId.empty())
+ {
+ m_BuildId = BuildId.ToString();
+ }
- const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath);
+ Oid BuildPartId;
+ if (!m_BuildPartId.empty())
+ {
+ BuildPartId = m_Parent.ParseBuildPartId(m_BuildPartId, Opts);
+ }
- std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
- std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
- ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+ CbObject MetaData = m_Parent.ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts);
- std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- std::unique_ptr<ChunkingCache> ChunkCache = m_ChunkingCachePath.empty()
- ? CreateNullChunkingCache()
- : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+ const std::filesystem::path TempDir = ZenTempFolderPath(m_Parent.GetZenFolderPath());
- std::vector<std::pair<Oid, std::string>> UploadedParts =
- UploadFolder(*Output,
- Workers,
- Storage,
- BuildId,
- BuildPartId,
- m_BuildPartName,
- m_Path,
- m_ManifestPath,
- MetaData,
- *ChunkController,
- *ChunkCache,
- UploadFolderOptions{.TempDir = TempDir,
- .FindBlockMaxCount = m_FindBlockMaxCount,
- .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_AllowMultiparts,
- .CreateBuild = m_CreateBuild,
- .IgnoreExistingBlocks = m_Clean,
- .UploadToZenCache = m_UploadToZenCache,
- .ExcludeFolders = ExcludeFolders,
- .ExcludeExtensions = ExcludeExtensions});
-
- if (!AbortFlag)
- {
- if (m_PostUploadVerify)
- {
- // TODO: Validate all parts
- for (const auto& Part : UploadedParts)
- {
- ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second);
- }
- }
- }
+ std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
+ std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
+ m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
- if (true)
+ std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
+ std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty()
+ ? CreateNullChunkingCache()
+ : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+
+ std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
+
+ std::vector<std::pair<Oid, std::string>> UploadedParts =
+ UploadFolder(*Output,
+ Workers,
+ Storage,
+ BuildId,
+ BuildPartId,
+ m_BuildPartName,
+ m_Path,
+ m_ManifestPath,
+ MetaData,
+ *ChunkController,
+ *ChunkCache,
+ UploadFolderOptions{.TempDir = TempDir,
+ .FindBlockMaxCount = m_FindBlockMaxCount,
+ .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .CreateBuild = m_CreateBuild,
+ .IgnoreExistingBlocks = m_Clean,
+ .UploadToZenCache = m_UploadToZenCache,
+ .ExcludeFolders = ExcludeFolders,
+ .ExcludeExtensions = ExcludeExtensions});
+
+ if (!AbortFlag)
+ {
+ if (m_PostUploadVerify)
{
- if (!IsQuiet)
+ for (const auto& Part : UploadedParts)
{
- ZEN_CONSOLE(
- "{}:\n"
- "Read: {}\n"
- "Write: {}\n"
- "Requests: {}\n"
- "Avg Request Time: {}\n"
- "Avg I/O Time: {}",
- Storage.BuildStorageHost.Name,
- NiceBytes(StorageStats.TotalBytesRead.load()),
- NiceBytes(StorageStats.TotalBytesWritten.load()),
- StorageStats.TotalRequestCount.load(),
- StorageStats.TotalExecutionTimeUs.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0,
- StorageStats.TotalRequestCount.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0);
+ ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second);
}
}
- if (AbortFlag)
- {
- throw std::runtime_error("Upload aborted");
- }
}
- auto ParseAllowPartialBlockRequests = [&]() -> EPartialBlockRequestMode {
- if (m_PrimeCacheOnly)
- {
- return EPartialBlockRequestMode::Off;
- }
- EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests);
- if (Mode == EPartialBlockRequestMode::Invalid)
- {
- throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests),
- SubOption->help());
- }
- return Mode;
- };
-
- if (SubOption == &m_DownloadOptions)
+ if (true)
{
if (!IsQuiet)
{
- LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
+ ZEN_CONSOLE(
+ "{}:\n"
+ "Read: {}\n"
+ "Write: {}\n"
+ "Requests: {}\n"
+ "Avg Request Time: {}\n"
+ "Avg I/O Time: {}",
+ Storage.BuildStorageHost.Name,
+ NiceBytes(StorageStats.TotalBytesRead.load()),
+ NiceBytes(StorageStats.TotalBytesWritten.load()),
+ StorageStats.TotalRequestCount.load(),
+ StorageStats.TotalExecutionTimeUs.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0,
+ StorageStats.TotalRequestCount.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0);
}
+ }
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Upload aborted");
+ }
+}
- ZenState InstanceState;
+BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent)
+: ZenSubCmdBase("download", "Download a build to a local folder")
+, m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddCacheOptions(Opts);
+ Parent.AddZenFolderOptions(Opts);
+ Parent.AddWorkerOptions(Opts);
+ Parent.AddWildcardOptions(Opts);
+ Parent.AddAppendNewContentOptions(Opts);
+ Parent.AddExcludeFolderOption(Opts);
+
+ Opts.add_option("cache",
+ "",
+ "cache-prime-only",
+ "Only download blobs missing in cache and upload to cache",
+ cxxopts::value(m_PrimeCacheOnly),
+ "<cacheprimeonly>");
+
+ Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
+ Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-id",
+ "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
+ cxxopts::value(m_BuildPartIds),
+ "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-name",
+ "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
+ "all parts will be downloaded",
+ cxxopts::value(m_BuildPartNames),
+ "<name>");
+ Opts.add_option("",
+ "",
+ "clean",
+ "Delete all data in target folder that is not part of the downloaded content",
+ cxxopts::value(m_Clean),
+ "<clean>");
+ Opts.add_option("",
+ "",
+ "force",
+ "Force download of all content by ignoring any existing local content",
+ cxxopts::value(m_Force),
+ "<force>");
+ Opts.add_option("cache",
+ "",
+ "zen-cache-upload",
+ "Upload data downloaded from remote host to zen cache",
+ cxxopts::value(m_UploadToZenCache),
+ "<uploadtozencache>");
+ Parent.AddMultipartOptions(Opts);
+
+ Parent.AddPartialBlockRequestOptions(Opts);
+
+ Opts.add_option(
+ "",
+ "",
+ "download-spec-path",
+ "Path to a text file with one line of <local path> per file to include or a structured .json file describing what to download.",
+ cxxopts::value(m_DownloadSpecPath),
+ "<downloadspecpath>");
- ParsePath();
+ Opts.add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), "<verify>");
+ Opts.add_option("",
+ "",
+ "enable-scavenge",
+ "Enable scavenging of data from previouse download locations",
+ cxxopts::value(m_EnableScavenging),
+ "<scavenge>");
+ Opts.add_option("",
+ "",
+ "allow-file-clone",
+ "Enable use of block reference counting when copying files",
+ cxxopts::value(m_AllowFileClone),
+ "<allowclone>");
+ Opts.parse_positional({"local-path", "build-id", "build-part-name"});
+ Opts.positional_help("local-path build-id build-part-name");
+}
- std::vector<std::string> IncludeWildcards;
- std::vector<std::string> ExcludeWildcards;
- ParseFileFilters(IncludeWildcards, ExcludeWildcards);
+void
+BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = m_Path / ZenFolderName;
- }
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ if (!IsQuiet)
+ {
+ ZenCmdBase::LogExecutableVersionAndPid();
+ }
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
+ }
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ m_PrimeCacheOnly);
+ ZenState InstanceState;
- const Oid BuildId = ParseBuildId();
+ m_Parent.ParsePath(m_Path, Opts);
- if (m_PostDownloadVerify && m_PrimeCacheOnly)
- {
- throw OptionParseException("'--cache-prime-only' conflicts with '--verify'", SubOption->help());
- }
+ std::vector<std::string> IncludeWildcards;
+ std::vector<std::string> ExcludeWildcards;
+ m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards);
- if (m_Clean && m_PrimeCacheOnly)
- {
- ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled");
- }
+ m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName);
- if (m_Force && m_PrimeCacheOnly)
- {
- ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled");
- }
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- if (m_AllowPartialBlockRequests != "false" && m_PrimeCacheOnly)
- {
- ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled");
- }
+ std::unique_ptr<AuthMgr> Auth;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ m_BuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ true,
+ /*BoostCacheBackgroundWorkerPool*/ m_PrimeCacheOnly,
+ Auth,
+ Opts);
+
+ const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+
+ if (m_PostDownloadVerify && m_PrimeCacheOnly)
+ {
+ throw OptionParseException("'--cache-prime-only' conflicts with '--verify'", Opts.help());
+ }
- std::vector<Oid> BuildPartIds = ParseBuildPartIds();
- std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+ if (m_Clean && m_PrimeCacheOnly)
+ {
+ ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled");
+ }
- EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests();
+ if (m_Force && m_PrimeCacheOnly)
+ {
+ ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled");
+ }
- if (m_AppendNewContent && m_Clean)
- {
- throw OptionParseException("'--append' conflicts with '--clean'", SubOption->help());
- }
+ if (m_Parent.m_AllowPartialBlockRequests != "false" && m_PrimeCacheOnly)
+ {
+ ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled");
+ }
- std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
- std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
- ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+ std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts);
+ std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts);
- DownloadFolder(*Output,
- Workers,
- Storage,
- StorageCacheStats,
- BuildId,
- BuildPartIds,
- BuildPartNames,
- m_DownloadSpecPath,
- m_Path,
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = m_ZenFolderPath,
- .AllowMultiparts = m_AllowMultiparts,
- .PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = m_Clean,
- .PostDownloadVerify = m_PostDownloadVerify,
- .PrimeCacheOnly = m_PrimeCacheOnly,
- .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force,
- .EnableTargetFolderScavenging = !m_Force,
- .AllowFileClone = m_AllowFileClone,
- .IncludeWildcards = IncludeWildcards,
- .ExcludeWildcards = ExcludeWildcards,
- .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory),
- .PopulateCache = m_UploadToZenCache,
- .AppendNewContent = m_AppendNewContent,
- .ExcludeFolders = ExcludeFolders});
+ EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(m_PrimeCacheOnly, Opts);
- if (AbortFlag)
- {
- throw std::runtime_error("Download aborted");
- }
+ if (m_Parent.m_AppendNewContent && m_Clean)
+ {
+ throw OptionParseException("'--append' conflicts with '--clean'", Opts.help());
}
- if (SubOption == &m_LsOptions)
+ std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
+ std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
+ m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+
+ std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
+
+ DownloadFolder(
+ *Output,
+ Workers,
+ Storage,
+ StorageCacheStats,
+ BuildId,
+ BuildPartIds,
+ BuildPartNames,
+ m_DownloadSpecPath,
+ m_Path,
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = m_Parent.GetZenFolderPath(),
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .CleanTargetFolder = m_Clean,
+ .PostDownloadVerify = m_PostDownloadVerify,
+ .PrimeCacheOnly = m_PrimeCacheOnly,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force,
+ .EnableTargetFolderScavenging = !m_Force,
+ .AllowFileClone = m_AllowFileClone,
+ .IncludeWildcards = IncludeWildcards,
+ .ExcludeWildcards = ExcludeWildcards,
+ .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Parent.m_BoostWorkerMemory),
+ .PopulateCache = m_UploadToZenCache,
+ .AppendNewContent = m_Parent.m_AppendNewContent,
+ .ExcludeFolders = ExcludeFolders});
+
+ if (AbortFlag)
{
- if (!m_LsResultPath.empty())
- {
- if (!IsQuiet)
- {
- LogExecutableVersionAndPid();
- }
- }
+ throw std::runtime_error("Download aborted");
+ }
+}
- ZenState InstanceState;
+BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "List files in a build"), m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddCacheOptions(Opts);
+ Parent.AddZenFolderOptions(Opts);
+ Parent.AddWorkerOptions(Opts);
+ Parent.AddWildcardOptions(Opts);
+
+ Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-id",
+ "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
+ cxxopts::value(m_BuildPartIds),
+ "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-name",
+ "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
+ "all parts will be downloaded",
+ cxxopts::value(m_BuildPartNames),
+ "<name>");
+
+ Opts.add_option("",
+ "",
+ "result-path",
+ "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console",
+ cxxopts::value(m_ResultPath),
+ "<result-path>");
+
+ Opts.add_option("",
+ "o",
+ "output-path",
+ "Path to output, extension .json or .cb (compac binary). Default is output to console",
+ cxxopts::value(m_ResultPath),
+ "<output-path>");
+
+ Opts.parse_positional({"build-id", "wildcard"});
+ Opts.positional_help("build-id wildcard");
+}
- std::vector<std::string> IncludeWildcards;
- std::vector<std::string> ExcludeWildcards;
- ParseFileFilters(IncludeWildcards, ExcludeWildcards);
+void
+BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
- if (m_ZenFolderPath.empty())
+ if (!m_ResultPath.empty())
+ {
+ if (!IsQuiet)
{
- m_ZenFolderPath = m_Path / ZenFolderName;
+ ZenCmdBase::LogExecutableVersionAndPid();
}
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ }
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ ZenState InstanceState;
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool */ false);
+ std::vector<std::string> IncludeWildcards;
+ std::vector<std::string> ExcludeWildcards;
+ m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards);
- const Oid BuildId = ParseBuildId();
+ m_Parent.ResolveZenFolderPath(m_Parent.m_StoragePath); // ls uses storage path context
- std::vector<Oid> BuildPartIds = ParseBuildPartIds();
- std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- std::unique_ptr<CbObjectWriter> StructuredOutput;
- if (!m_LsResultPath.empty())
- {
- MakeSafeAbsolutePathInPlace(m_LsResultPath);
- StructuredOutput = std::make_unique<CbObjectWriter>();
- }
+ std::unique_ptr<AuthMgr> Auth;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ m_BuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ true,
+ /*BoostCacheBackgroundWorkerPool*/ false,
+ Auth,
+ Opts);
+
+ const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+
+ std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts);
+ std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts);
+
+ std::unique_ptr<CbObjectWriter> StructuredOutput;
+ if (!m_ResultPath.empty())
+ {
+ MakeSafeAbsolutePathInPlace(m_ResultPath);
+ StructuredOutput = std::make_unique<CbObjectWriter>();
+ }
- ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get());
+ ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get());
- if (StructuredOutput)
+ if (StructuredOutput)
+ {
+ CbObject Response = StructuredOutput->Save();
+ if (ToLower(m_ResultPath.extension().string()) == ".cbo")
{
- CbObject Response = StructuredOutput->Save();
- if (ToLower(m_LsResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ MemoryView ResponseView = Response.GetView();
+ WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
}
-
- if (AbortFlag)
+ else
{
- throw std::runtime_error("List build aborted");
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(Response.GetView(), SB);
+ WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
}
}
- if (SubOption == &m_DiffOptions)
+ if (AbortFlag)
{
- if (!IsQuiet)
- {
- LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
-
- ParsePath();
- ParseDiffPath();
-
- MakeSafeAbsolutePathInPlace(m_ChunkingCachePath);
+ throw std::runtime_error("List build aborted");
+ }
+}
- std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
- std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
- ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff", "Diff two local folders"), m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddOutputOptions(Opts);
+ Parent.AddWorkerOptions(Opts);
+ Parent.AddExcludeFolderOption(Opts);
+ Parent.AddExcludeExtensionsOption(Opts);
+ Parent.AddChunkingCacheOptions(Opts);
+ Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
+ Opts.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>");
+ Opts.add_option("",
+ "",
+ "only-chunked",
+ "Skip files from diff summation that are not processed with chunking",
+ cxxopts::value(m_OnlyChunked),
+ "<only-chunked>");
+ Opts.parse_positional({"local-path", "compare-path"});
+ Opts.positional_help("local-path compare-path");
+}
- StandardChunkingControllerSettings ChunkingSettings;
- std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings);
- std::unique_ptr<ChunkingCache> ChunkCache = m_ChunkingCachePath.empty()
- ? CreateNullChunkingCache()
- : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+void
+BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
- if (m_OnlyChunked)
- {
- ExcludeExtensions.insert(ExcludeExtensions.end(),
- ChunkingSettings.SplitOnlyExtensions.begin(),
- ChunkingSettings.SplitOnlyExtensions.end());
- ExcludeExtensions.insert(ExcludeExtensions.end(),
- ChunkingSettings.SplitAndCompressExtensions.begin(),
- ChunkingSettings.SplitAndCompressExtensions.end());
- }
+ if (!IsQuiet)
+ {
+ ZenCmdBase::LogExecutableVersionAndPid();
+ }
- DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions);
- if (AbortFlag)
- {
- throw std::runtime_error("Diff folders aborted");
- }
+ TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
}
- if (SubOption == &m_PrimeCacheOptions)
+ m_Parent.ParsePath(m_Path, Opts);
+ if (m_DiffPath.empty())
{
- if (!IsQuiet)
- {
- LogExecutableVersionAndPid();
- }
+ throw OptionParseException("'--compare-path' is required", Opts.help());
+ }
+ MakeSafeAbsolutePathInPlace(m_DiffPath);
- TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath);
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
+ std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
+ m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
- }
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ StandardChunkingControllerSettings ChunkingSettings;
+ std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings);
+ std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty()
+ ? CreateNullChunkingCache()
+ : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u);
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });
+ if (m_OnlyChunked)
+ {
+ ExcludeExtensions.insert(ExcludeExtensions.end(),
+ ChunkingSettings.SplitOnlyExtensions.begin(),
+ ChunkingSettings.SplitOnlyExtensions.end());
+ ExcludeExtensions.insert(ExcludeExtensions.end(),
+ ChunkingSettings.SplitAndCompressExtensions.begin(),
+ ChunkingSettings.SplitAndCompressExtensions.end());
+ }
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool */ true);
+ DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions);
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Diff folders aborted");
+ }
+}
- const Oid BuildId = ParseBuildId();
+BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent)
+: ZenSubCmdBase("fetch-blob", "Fetch and validate a specific blob")
+, m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddCacheOptions(Opts);
+ Parent.AddZenFolderOptions(Opts);
+ Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ Opts.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>");
+ Opts.parse_positional({"build-id", "blob-hash"});
+ Opts.positional_help("build-id blob-hash");
+}
- std::vector<Oid> BuildPartIds = ParseBuildPartIds();
- std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+void
+BuildsFetchBlobSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
- std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
+ if (!IsQuiet)
+ {
+ ZenCmdBase::LogExecutableVersionAndPid();
+ }
- CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);
+ TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
+ }
- std::vector<std::pair<Oid, std::string>> AllBuildParts =
- ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- std::vector<Oid> AllBuildPartIds;
- AllBuildPartIds.reserve(AllBuildParts.size());
- for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts)
- {
- AllBuildPartIds.push_back(BuildPart.first);
- }
+ m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
- ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache");
-
- BuildsOperationPrimeCache PrimeOp(*Output,
- Storage,
- AbortFlag,
- PauseFlag,
- Workers.GetNetworkPool(),
- BuildId,
- AllBuildPartIds,
- BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet,
- .IsVerbose = IsVerbose,
- .ZenFolderPath = m_ZenFolderPath,
- .LargeAttachmentSize = PreferredMultipartChunkSize * 4u,
- .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
- .ForceUpload = m_Force},
- StorageCacheStats);
- PrimeOp.Execute();
+ CreateDirectories(m_Parent.GetZenFolderPath());
+ auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
- if (!IsQuiet)
- {
- if (Storage.CacheStorage)
- {
- ZEN_CONSOLE("Uploaded {} ({}) blobs to {}",
- StorageCacheStats.PutBlobCount.load(),
- NiceBytes(StorageCacheStats.PutBlobByteCount),
- Storage.CacheHost.Name);
- }
- }
+ std::unique_ptr<AuthMgr> Auth;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ m_BuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ true,
+ /*BoostCacheBackgroundWorkerPool*/ false,
+ Auth,
+ Opts);
+
+ IoHash BlobHash = m_Parent.ParseBlobHash(m_BlobHash, Opts);
+
+ const Oid BuildId = Oid::FromHexString(m_BuildId);
+
+ uint64_t CompressedSize;
+ uint64_t DecompressedSize;
+ ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Fetch blob aborted");
+ }
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize);
+ }
+}
- return;
+BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent)
+: ZenSubCmdBase("prime-cache", "Prime the zen cache with build data")
+, m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddCacheOptions(Opts);
+ Parent.AddWorkerOptions(Opts);
+ Parent.AddZenFolderOptions(Opts);
+ Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-id",
+ "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
+ cxxopts::value(m_BuildPartIds),
+ "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-name",
+ "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
+ "all parts will be downloaded",
+ cxxopts::value(m_BuildPartNames),
+ "<name>");
+ Opts.add_option("",
+ "",
+ "force",
+ "Force download of all blobs by ignoring any existing blobs in cache",
+ cxxopts::value(m_Force),
+ "<force>");
+ Opts.parse_positional({"build-id"});
+ Opts.positional_help("build-id");
+}
+
+void
+BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
+
+ if (!IsQuiet)
+ {
+ ZenCmdBase::LogExecutableVersionAndPid();
}
- if (SubOption == &m_FetchBlobOptions)
+ TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
+ if (!IsQuiet)
{
- if (!IsQuiet)
- {
- LogExecutableVersionAndPid();
- }
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
+ }
- TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
- }
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ CreateDirectories(m_Parent.GetZenFolderPath());
+ auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });
+ std::unique_ptr<AuthMgr> Auth;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ m_BuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ true,
+ /*BoostCacheBackgroundWorkerPool*/ true,
+ Auth,
+ Opts);
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool */ false);
+ const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
- IoHash BlobHash = ParseBlobHash();
+ std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts);
+ std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts);
- const Oid BuildId = Oid::FromHexString(m_BuildId);
+ std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
- uint64_t CompressedSize;
- uint64_t DecompressedSize;
- ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
- if (AbortFlag)
- {
- throw std::runtime_error("Fetch blob aborted");
- }
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
- BlobHash,
- CompressedSize,
- DecompressedSize);
- }
- return;
- }
+ CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);
- if (SubOption == &m_ValidateBuildPartOptions)
- {
- if (!IsQuiet)
- {
- LogExecutableVersionAndPid();
- }
+ std::vector<std::pair<Oid, std::string>> AllBuildParts =
+ ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
- TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ std::vector<Oid> AllBuildPartIds;
+ AllBuildPartIds.reserve(AllBuildParts.size());
+ for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts)
+ {
+ AllBuildPartIds.push_back(BuildPart.first);
+ }
- ZenState InstanceState;
+ ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache");
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
- if (m_ZenFolderPath.empty())
+ BuildsOperationPrimeCache PrimeOp(*Output,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ Workers.GetNetworkPool(),
+ BuildId,
+ AllBuildPartIds,
+ BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet,
+ .IsVerbose = IsVerbose,
+ .ZenFolderPath = m_Parent.GetZenFolderPath(),
+ .LargeAttachmentSize = PreferredMultipartChunkSize * 4u,
+ .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
+ .ForceUpload = m_Force},
+ StorageCacheStats);
+ PrimeOp.Execute();
+
+ if (!IsQuiet)
+ {
+ if (Storage.CacheStorage)
{
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ ZEN_CONSOLE("Uploaded {} ({}) blobs to {}",
+ StorageCacheStats.PutBlobCount.load(),
+ NiceBytes(StorageCacheStats.PutBlobByteCount),
+ Storage.CacheHost.Name);
}
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ }
+}
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });
+BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("pause", "Pause a running zen process"), m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
+ Opts.parse_positional({"process-id"});
+ Opts.positional_help("process-id");
+}
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool */ false);
+void
+BuildsPauseSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ using namespace builds_impl;
+ m_Parent.ParseZenProcessId(m_ZenProcessId);
+ ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Pause.store(true);
+}
- Oid BuildId = ParseBuildId();
+BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("resume", "Resume a paused zen process"), m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
+ Opts.parse_positional({"process-id"});
+ Opts.positional_help("process-id");
+}
- if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
- {
- throw OptionParseException(
- fmt::format("'--build-part-id' ('{}') conflicts with '--build-part-name' ('{}')", m_BuildPartId, m_BuildPartName),
- SubOption->help());
- }
+void
+BuildsResumeSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ using namespace builds_impl;
+ m_Parent.ParseZenProcessId(m_ZenProcessId);
+ ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Pause.store(false);
+}
- const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId();
+BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("abort", "Abort a running zen process"), m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
+ Opts.parse_positional({"process-id"});
+ Opts.positional_help("process-id");
+}
- ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+void
+BuildsAbortSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ using namespace builds_impl;
+ m_Parent.ParseZenProcessId(m_ZenProcessId);
+ ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Abort.store(true);
+}
- if (AbortFlag)
- {
- throw std::runtime_error("Validate build part failed");
- }
+BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent)
+: ZenSubCmdBase("validate-part", "Validate a build part")
+, m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddWorkerOptions(Opts);
+ Parent.AddZenFolderOptions(Opts);
+ Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-id",
+ "Build part Id, if not given it will be auto generated",
+ cxxopts::value(m_BuildPartId),
+ "<id>");
+ Opts.add_option("",
+ "",
+ "build-part-name",
+ "Name of the build part, if not given it will be be named after the directory name at end of local-path",
+ cxxopts::value(m_BuildPartName),
+ "<name>");
+ Opts.parse_positional({"build-id", "build-part-id"});
+ Opts.positional_help("build-id build-part-id");
+}
+
+void
+BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
+
+ if (!IsQuiet)
+ {
+ ZenCmdBase::LogExecutableVersionAndPid();
}
- if (SubOption == &m_MultiTestDownloadOptions)
+ TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
+ if (!IsQuiet)
{
- TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
+ }
- m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(m_SystemRootDir);
- CleanDirectory(m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
- auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
+ ZenState InstanceState;
- ParsePath();
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = m_Path / ZenFolderName;
- }
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
- EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests();
+ CreateDirectories(m_Parent.GetZenFolderPath());
+ auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ m_BuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ true,
+ /*BoostCacheBackgroundWorkerPool*/ false,
+ Auth,
+ Opts);
+
+ Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+
+ if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
+ {
+ throw OptionParseException(
+ fmt::format("'--build-part-id' ('{}') conflicts with '--build-part-name' ('{}')", m_BuildPartId, m_BuildPartName),
+ Opts.help());
+ }
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool */ false);
+ const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : m_Parent.ParseBuildPartId(m_BuildPartId, Opts);
- Stopwatch Timer;
- for (const std::string& BuildIdString : m_BuildIds)
- {
- Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString));
- if (BuildId == Oid::Zero)
- {
- throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), SubOption->help());
- }
- DownloadFolder(*Output,
- Workers,
- Storage,
- StorageCacheStats,
- BuildId,
- /*BuildPartIds,*/ {},
- /*BuildPartNames*/ {},
- /*ManifestPath*/ {},
- m_Path,
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = m_ZenFolderPath,
- .AllowMultiparts = m_AllowMultiparts,
- .PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = BuildIdString == m_BuildIds.front(),
- .PostDownloadVerify = true,
- .PrimeCacheOnly = false,
- .EnableOtherDownloadsScavenging = m_EnableScavenging,
- .EnableTargetFolderScavenging = false,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
- {
- throw std::runtime_error("Multitest aborted");
- }
- if (!IsQuiet)
- {
- ZEN_CONSOLE("\n");
- }
- }
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- }
- }
+ std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
- auto ParseZenProcessId = [&]() {
- if (m_ZenProcessId == -1)
- {
- const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath();
- ProcessHandle RunningProcess;
- std::error_code Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false);
- if (Ec)
- {
- throw std::runtime_error(
- fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message()));
- }
- if (!RunningProcess.IsValid())
- {
- throw std::runtime_error(
- fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath));
- }
- m_ZenProcessId = RunningProcess.Pid();
- }
- };
+ ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
- if (SubOption == &m_PauseOptions)
+ if (AbortFlag)
{
- ParseZenProcessId();
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Pause.store(true);
+ throw std::runtime_error("Validate build part failed");
}
+}
- if (SubOption == &m_ResumeOptions)
- {
- ParseZenProcessId();
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Pause.store(false);
- }
+BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test", "Run an upload/download test cycle"), m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddCacheOptions(Opts);
+ Parent.AddWorkerOptions(Opts);
+ Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
+ Parent.AddMultipartOptions(Opts);
+ Parent.AddPartialBlockRequestOptions(Opts);
+ Parent.AddWildcardOptions(Opts);
+ Parent.AddAppendNewContentOptions(Opts);
+ Parent.AddChunkingCacheOptions(Opts);
+ Opts.add_option("",
+ "",
+ "enable-scavenge",
+ "Enable scavenging of data from previouse download locations",
+ cxxopts::value(m_EnableScavenging),
+ "<scavenge>");
+ Opts.add_option("",
+ "",
+ "allow-file-clone",
+ "Enable use of block reference counting when copying files",
+ cxxopts::value(m_AllowFileClone),
+ "<allowclone>");
+ Opts.parse_positional({"local-path"});
+ Opts.positional_help("local-path");
+}
- if (SubOption == &m_AbortOptions)
+void
+BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
+
+ TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
+ if (!IsQuiet)
{
- ParseZenProcessId();
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Abort.store(true);
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
}
- if (SubOption == &m_TestOptions)
- {
- TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_Parent.m_SystemRootDir);
+ CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
+ auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); });
- m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(m_SystemRootDir);
- CleanDirectory(m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
- auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
+ m_Parent.ParsePath(m_Path, Opts);
- ParsePath();
+ if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty())
+ {
+ m_Parent.m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.m_StoragePath);
+ CreateDirectories(m_Parent.m_StoragePath);
+ m_Parent.m_StoragePath = m_Parent.m_StoragePath.generic_string();
+ }
- if (m_OverrideHost.empty() && m_StoragePath.empty())
+ auto StorageGuard = MakeGuard([this]() {
+ if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty())
{
- m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_StoragePath);
- CreateDirectories(m_StoragePath);
- m_StoragePath = m_StoragePath.generic_string();
+ DeleteDirectories(m_Parent.m_StoragePath);
}
+ });
- auto __ = MakeGuard([&]() {
- if (m_OverrideHost.empty() && m_StoragePath.empty())
- {
- DeleteDirectories(m_StoragePath);
- }
- });
+ EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts);
- EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests();
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ m_BuildPartName = m_Path.filename().string();
- const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test");
- const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2");
- const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3");
+ const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test");
+ const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2");
+ const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3");
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3);
+
+ auto DownloadGuard = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() {
CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath);
CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2);
CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3);
+ });
- auto ___ = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() {
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3);
- });
-
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = m_Path / ZenFolderName;
- }
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
- MakeSafeAbsolutePathInPlace(m_ChunkingCachePath);
+ m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName);
+ MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath);
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool */ false);
+ std::unique_ptr<AuthMgr> Auth;
+ std::string TestBuildId;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ TestBuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ true,
+ /*BoostCacheBackgroundWorkerPool*/ false,
+ Auth,
+ Opts);
+
+ m_BuildId = Oid::NewOid().ToString();
+ m_BuildPartId = Oid::NewOid().ToString();
+ m_CreateBuild = true;
+
+ const Oid BuildId = Oid::FromHexString(m_BuildId);
+ const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
+
+ auto MakeMetaData = [](const Oid& BuildId) -> CbObject {
+ CbObjectWriter BuildMetaDataWriter;
+ {
+ const uint32_t CL = BuildId.OidBits[2];
+ BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL));
+ BuildMetaDataWriter.AddString("branch", "ZenTestBuild");
+ BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild");
+ BuildMetaDataWriter.AddString("platform", "Windows");
+ BuildMetaDataWriter.AddString("project", "Test");
+ BuildMetaDataWriter.AddInteger("changelist", CL);
+ BuildMetaDataWriter.AddString("buildType", "test-folder");
+ }
+ return BuildMetaDataWriter.Save();
+ };
+ CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId));
+ {
+ ExtendableStringBuilder<256> SB;
+ CompactBinaryToJson(MetaData, SB);
+ ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView());
+ }
- m_BuildId = Oid::NewOid().ToString();
- m_BuildPartName = m_Path.filename().string();
- m_BuildPartId = Oid::NewOid().ToString();
- m_CreateBuild = true;
+ const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path);
- const Oid BuildId = Oid::FromHexString(m_BuildId);
- const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
+ std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
+ std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty()
+ ? CreateNullChunkingCache()
+ : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u);
- auto MakeMetaData = [](const Oid& BuildId) -> CbObject {
- CbObjectWriter BuildMetaDataWriter;
- {
- const uint32_t CL = BuildId.OidBits[2];
- BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL));
- BuildMetaDataWriter.AddString("branch", "ZenTestBuild");
- BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild");
- BuildMetaDataWriter.AddString("platform", "Windows");
- BuildMetaDataWriter.AddString("project", "Test");
- BuildMetaDataWriter.AddInteger("changelist", CL);
- BuildMetaDataWriter.AddString("buildType", "test-folder");
- }
- return BuildMetaDataWriter.Save();
- };
- CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId));
- {
- ExtendableStringBuilder<256> SB;
- CompactBinaryToJson(MetaData, SB);
- ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView());
- }
+ std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
- const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path);
+ UploadFolder(*Output,
+ Workers,
+ Storage,
+ BuildId,
+ BuildPartId,
+ m_BuildPartName,
+ m_Path,
+ {},
+ MetaData,
+ *ChunkController,
+ *ChunkCache,
+ UploadFolderOptions{.TempDir = UploadTempDir,
+ .FindBlockMaxCount = m_FindBlockMaxCount,
+ .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .CreateBuild = true,
+ .IgnoreExistingBlocks = false,
+ .UploadToZenCache = m_UploadToZenCache});
+
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Upload build)");
+ }
- std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- std::unique_ptr<ChunkingCache> ChunkCache = m_ChunkingCachePath.empty()
- ? CreateNullChunkingCache()
- : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+ {
+ ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path);
UploadFolder(*Output,
Workers,
Storage,
- BuildId,
- BuildPartId,
+ Oid::NewOid(),
+ Oid::NewOid(),
m_BuildPartName,
m_Path,
{},
@@ -4170,145 +4245,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
UploadFolderOptions{.TempDir = UploadTempDir,
.FindBlockMaxCount = m_FindBlockMaxCount,
.BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_AllowMultiparts,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
.CreateBuild = true,
.IgnoreExistingBlocks = false,
.UploadToZenCache = m_UploadToZenCache});
if (AbortFlag)
{
- throw std::runtime_error("Test aborted. (Upload build)");
+ throw std::runtime_error("Test aborted. (Upload again, chunking is cached)");
}
+ }
- {
- ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path);
-
- UploadFolder(*Output,
- Workers,
- Storage,
- Oid::NewOid(),
- Oid::NewOid(),
- m_BuildPartName,
- m_Path,
- {},
- MetaData,
- *ChunkController,
- *ChunkCache,
- UploadFolderOptions{.TempDir = UploadTempDir,
- .FindBlockMaxCount = m_FindBlockMaxCount,
- .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_AllowMultiparts,
- .CreateBuild = true,
- .IgnoreExistingBlocks = false,
- .UploadToZenCache = m_UploadToZenCache});
+ ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
- if (AbortFlag)
- {
- throw std::runtime_error("Test aborted. (Upload again, chunking is cached)");
- }
- }
+ if (!m_Parent.m_IncludeWildcard.empty() || !m_Parent.m_ExcludeWildcard.empty())
+ {
+ auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); });
- ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- if (!m_IncludeWildcard.empty() || !m_ExcludeWildcard.empty())
- {
- auto __ = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); });
-
- ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
-
- std::vector<std::string> IncludeWildcards;
- std::vector<std::string> ExcludeWildcards;
- ParseFileFilters(IncludeWildcards, ExcludeWildcards);
-
- DownloadFolder(*Output,
- Workers,
- Storage,
- StorageCacheStats,
- BuildId,
- {BuildPartId},
- /*BuildPartNames*/ {},
- /*ManifestPath*/ {},
- DownloadPath,
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
- .PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = true,
- .PostDownloadVerify = true,
- .PrimeCacheOnly = false,
- .EnableOtherDownloadsScavenging = m_EnableScavenging,
- .EnableTargetFolderScavenging = false,
- .AllowFileClone = m_AllowFileClone,
- .IncludeWildcards = IncludeWildcards,
- .ExcludeWildcards = ExcludeWildcards,
- .AppendNewContent = false});
- if (AbortFlag)
- {
- throw std::runtime_error("Test aborted. (Download build)");
- }
-
- ZEN_CONSOLE("\nDownload Filtered Out Remaining of Build {}, Part {} ({}) to '{}'",
- BuildId,
- BuildPartId,
- m_BuildPartName,
- DownloadPath);
- DownloadFolder(*Output,
- Workers,
- Storage,
- StorageCacheStats,
- BuildId,
- {BuildPartId},
- /*BuildPartNames*/ {},
- /*ManifestPath*/ {},
- DownloadPath,
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
- .PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = true,
- .PostDownloadVerify = true,
- .PrimeCacheOnly = false,
- .EnableOtherDownloadsScavenging = m_EnableScavenging,
- .EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone,
- .IncludeWildcards = ExcludeWildcards,
- .ExcludeWildcards = IncludeWildcards,
- .AppendNewContent = true});
- if (AbortFlag)
- {
- throw std::runtime_error("Test aborted. (Download build)");
- }
-
- ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
- Workers,
- Storage,
- StorageCacheStats,
- BuildId,
- {BuildPartId},
- /*BuildPartNames*/ {},
- /*ManifestPath*/ {},
- DownloadPath,
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
- .PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = false,
- .PostDownloadVerify = true,
- .PrimeCacheOnly = false,
- .EnableOtherDownloadsScavenging = m_EnableScavenging,
- .EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone,
- .IncludeWildcards = {},
- .ExcludeWildcards = {},
- .AppendNewContent = false});
- if (AbortFlag)
- {
- throw std::runtime_error("Test aborted. (Download build)");
- }
- }
+ std::vector<std::string> IncludeWildcards;
+ std::vector<std::string> ExcludeWildcards;
+ m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards);
- ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
DownloadFolder(*Output,
Workers,
Storage,
@@ -4318,23 +4277,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
-
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = true,
.PostDownloadVerify = true,
.PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = false,
- .AllowFileClone = m_AllowFileClone});
+ .AllowFileClone = m_AllowFileClone,
+ .IncludeWildcards = IncludeWildcards,
+ .ExcludeWildcards = ExcludeWildcards,
+ .AppendNewContent = false});
if (AbortFlag)
{
throw std::runtime_error("Test aborted. (Download build)");
}
- ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ ZEN_CONSOLE("\nDownload Filtered Out Remaining of Build {}, Part {} ({}) to '{}'",
+ BuildId,
+ BuildPartId,
+ m_BuildPartName,
+ DownloadPath);
DownloadFolder(*Output,
Workers,
Storage,
@@ -4344,115 +4309,25 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
-
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = false,
+ .CleanTargetFolder = true,
.PostDownloadVerify = true,
.PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
+ .AllowFileClone = m_AllowFileClone,
+ .IncludeWildcards = ExcludeWildcards,
+ .ExcludeWildcards = IncludeWildcards,
+ .AppendNewContent = true});
if (AbortFlag)
{
- throw std::runtime_error("Test aborted. (Re-download identical target)");
+ throw std::runtime_error("Test aborted. (Download build)");
}
- auto ScrambleDir = [&Workers](const std::filesystem::path& Path) {
- ZEN_CONSOLE("\nScrambling '{}'", Path);
- Stopwatch Timer;
- DirectoryContent DownloadContent;
- GetDirectoryContent(
- Path,
- DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
- DownloadContent);
- auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool {
- std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string();
- for (const std::string& ExcludeFolder : ExcludeFolders)
- {
- if (RelativePath.starts_with(ExcludeFolder))
- {
- if (RelativePath.length() == ExcludeFolder.length())
- {
- return false;
- }
- else if (RelativePath[ExcludeFolder.length()] == '/')
- {
- return false;
- }
- }
- }
- return true;
- };
-
- ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- uint32_t Randomizer = 0;
- auto FileSizeIt = DownloadContent.FileSizes.begin();
- for (const std::filesystem::path& FilePath : DownloadContent.Files)
- {
- if (IsAcceptedFolder(FilePath))
- {
- uint32_t Case = (Randomizer++) % 7;
- switch (Case)
- {
- case 0:
- {
- uint64_t SourceSize = *FileSizeIt;
- if (SourceSize > 256)
- {
- Work.ScheduleWork(
- Workers.GetIOWorkerPool(),
- [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
- if (!AbortFlag)
- {
- bool WasReadOnly = SetFileReadOnly(FilePath, false);
- {
- BasicFile Source(FilePath, BasicFile::Mode::kWrite);
- uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u);
- IoBuffer TempBuffer1(RangeSize);
- IoBuffer TempBuffer2(RangeSize);
- IoBuffer TempBuffer3(RangeSize);
- Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0);
- Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2);
- Source.Read(TempBuffer3.GetMutableView().GetData(), RangeSize, SourceSize - RangeSize);
- Source.Write(TempBuffer1, SourceSize / 2);
- Source.Write(TempBuffer2, SourceSize - RangeSize);
- Source.Write(TempBuffer3, SourceSize - 0);
- }
- if (WasReadOnly)
- {
- SetFileReadOnly(FilePath, true);
- }
- }
- });
- }
- }
- break;
- case 1:
- {
- (void)SetFileReadOnly(FilePath, false);
- (void)RemoveFile(FilePath);
- }
- break;
- default:
- break;
- }
- }
- FileSizeIt++;
- }
- Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(IsAborted, IsPaused);
- ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork);
- });
- ZEN_ASSERT(!AbortFlag.load());
- ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- };
-
- ScrambleDir(DownloadPath);
- ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
DownloadFolder(*Output,
Workers,
Storage,
@@ -4462,186 +4337,469 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
-
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
.PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
+ .AllowFileClone = m_AllowFileClone,
+ .IncludeWildcards = {},
+ .ExcludeWildcards = {},
+ .AppendNewContent = false});
if (AbortFlag)
{
- throw std::runtime_error("Test aborted. (Re-download scrambled target)");
+ throw std::runtime_error("Test aborted. (Download build)");
}
+ }
- ScrambleDir(DownloadPath);
+ ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ DownloadFolder(*Output,
+ Workers,
+ Storage,
+ StorageCacheStats,
+ BuildId,
+ {BuildPartId},
+ /*BuildPartNames*/ {},
+ /*ManifestPath*/ {},
+ DownloadPath,
+
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .CleanTargetFolder = true,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = false,
+ .AllowFileClone = m_AllowFileClone});
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Download build)");
+ }
- Oid BuildId2 = Oid::NewOid();
- Oid BuildPartId2 = Oid::NewOid();
+ ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ DownloadFolder(*Output,
+ Workers,
+ Storage,
+ StorageCacheStats,
+ BuildId,
+ {BuildPartId},
+ /*BuildPartNames*/ {},
+ /*ManifestPath*/ {},
+ DownloadPath,
+
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true,
+ .AllowFileClone = m_AllowFileClone});
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Re-download identical target)");
+ }
- CbObject MetaData2 = MakeMetaData(BuildId2);
- {
- ExtendableStringBuilder<256> SB;
- CompactBinaryToJson(MetaData, SB);
- ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView());
- }
+ auto ScrambleDir = [&Workers, this](const std::filesystem::path& Path) {
+ ZEN_CONSOLE("\nScrambling '{}'", Path);
+ Stopwatch Timer;
+ DirectoryContent DownloadContent;
+ GetDirectoryContent(
+ Path,
+ DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
+ DownloadContent);
+ auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool {
+ std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string();
+ for (const std::string& ExcludeFolder : ExcludeFolders)
+ {
+ if (RelativePath.starts_with(ExcludeFolder))
+ {
+ if (RelativePath.length() == ExcludeFolder.length())
+ {
+ return false;
+ }
+ else if (RelativePath[ExcludeFolder.length()] == '/')
+ {
+ return false;
+ }
+ }
+ }
+ return true;
+ };
- UploadFolder(*Output,
- Workers,
- Storage,
- BuildId2,
- BuildPartId2,
- m_BuildPartName,
- DownloadPath,
- {},
- MetaData2,
- *ChunkController,
- *ChunkCache,
- UploadFolderOptions{.TempDir = UploadTempDir,
- .FindBlockMaxCount = m_FindBlockMaxCount,
- .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_AllowMultiparts,
- .CreateBuild = true,
- .IgnoreExistingBlocks = false,
- .UploadToZenCache = m_UploadToZenCache});
+ ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
- if (AbortFlag)
+ uint32_t Randomizer = 0;
+ auto FileSizeIt = DownloadContent.FileSizes.begin();
+ for (const std::filesystem::path& FilePath : DownloadContent.Files)
{
- throw std::runtime_error("Test aborted. (Upload scrambled)");
+ if (IsAcceptedFolder(FilePath))
+ {
+ uint32_t Case = (Randomizer++) % 7;
+ switch (Case)
+ {
+ case 0:
+ {
+ uint64_t SourceSize = *FileSizeIt;
+ if (SourceSize > 256)
+ {
+ Work.ScheduleWork(
+ Workers.GetIOWorkerPool(),
+ [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
+ if (!AbortFlag)
+ {
+ bool WasReadOnly = SetFileReadOnly(FilePath, false);
+ {
+ BasicFile Source(FilePath, BasicFile::Mode::kWrite);
+ uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u);
+ IoBuffer TempBuffer1(RangeSize);
+ IoBuffer TempBuffer2(RangeSize);
+ IoBuffer TempBuffer3(RangeSize);
+ Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0);
+ Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2);
+ Source.Read(TempBuffer3.GetMutableView().GetData(), RangeSize, SourceSize - RangeSize);
+ Source.Write(TempBuffer1, SourceSize / 2);
+ Source.Write(TempBuffer2, SourceSize - RangeSize);
+ Source.Write(TempBuffer3, SourceSize - 0);
+ }
+ if (WasReadOnly)
+ {
+ SetFileReadOnly(FilePath, true);
+ }
+ }
+ });
+ }
+ }
+ break;
+ case 1:
+ {
+ (void)SetFileReadOnly(FilePath, false);
+ (void)RemoveFile(FilePath);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ FileSizeIt++;
}
+ Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(IsAborted, IsPaused);
+ ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork);
+ });
+ ZEN_ASSERT(!AbortFlag.load());
+ ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ };
- ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ScrambleDir(DownloadPath);
+ ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ DownloadFolder(*Output,
+ Workers,
+ Storage,
+ StorageCacheStats,
+ BuildId,
+ {BuildPartId},
+ /*BuildPartNames*/ {},
+ /*ManifestPath*/ {},
+ DownloadPath,
+
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true,
+ .AllowFileClone = m_AllowFileClone});
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Re-download scrambled target)");
+ }
- ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
- Workers,
- Storage,
- StorageCacheStats,
- BuildId,
- {BuildPartId},
- /*BuildPartNames*/ {},
- /*ManifestPath*/ {},
- DownloadPath,
+ ScrambleDir(DownloadPath);
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
- .PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = false,
- .PostDownloadVerify = true,
- .PrimeCacheOnly = false,
- .EnableOtherDownloadsScavenging = m_EnableScavenging,
- .EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
- {
- throw std::runtime_error("Test aborted. (Download original)");
- }
+ Oid BuildId2 = Oid::NewOid();
+ Oid BuildPartId2 = Oid::NewOid();
- ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
- Workers,
- Storage,
- StorageCacheStats,
- BuildId2,
- {BuildPartId2},
- /*BuildPartNames*/ {},
- /*ManifestPath*/ {},
- DownloadPath,
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
- .PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = false,
- .PostDownloadVerify = true,
- .PrimeCacheOnly = false,
- .EnableOtherDownloadsScavenging = m_EnableScavenging,
- .EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
- {
- throw std::runtime_error("Test aborted. (Download scrambled)");
- }
+ CbObject MetaData2 = MakeMetaData(BuildId2);
+ {
+ ExtendableStringBuilder<256> SB;
+ CompactBinaryToJson(MetaData, SB);
+ ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView());
+ }
- ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
- Workers,
- Storage,
- StorageCacheStats,
- BuildId2,
- {BuildPartId2},
- /*BuildPartNames*/ {},
- /*ManifestPath*/ {},
- DownloadPath,
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
- .PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = false,
- .PostDownloadVerify = true,
- .PrimeCacheOnly = false,
- .EnableOtherDownloadsScavenging = m_EnableScavenging,
- .EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ UploadFolder(*Output,
+ Workers,
+ Storage,
+ BuildId2,
+ BuildPartId2,
+ m_BuildPartName,
+ DownloadPath,
+ {},
+ MetaData2,
+ *ChunkController,
+ *ChunkCache,
+ UploadFolderOptions{.TempDir = UploadTempDir,
+ .FindBlockMaxCount = m_FindBlockMaxCount,
+ .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .CreateBuild = true,
+ .IgnoreExistingBlocks = false,
+ .UploadToZenCache = m_UploadToZenCache});
+
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Upload scrambled)");
+ }
+
+ ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+
+ ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ DownloadFolder(*Output,
+ Workers,
+ Storage,
+ StorageCacheStats,
+ BuildId,
+ {BuildPartId},
+ /*BuildPartNames*/ {},
+ /*ManifestPath*/ {},
+ DownloadPath,
+
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true,
+ .AllowFileClone = m_AllowFileClone});
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Download original)");
+ }
+
+ ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
+ DownloadFolder(*Output,
+ Workers,
+ Storage,
+ StorageCacheStats,
+ BuildId2,
+ {BuildPartId2},
+ /*BuildPartNames*/ {},
+ /*ManifestPath*/ {},
+ DownloadPath,
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true,
+ .AllowFileClone = m_AllowFileClone});
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Download scrambled)");
+ }
+
+ ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
+ DownloadFolder(*Output,
+ Workers,
+ Storage,
+ StorageCacheStats,
+ BuildId2,
+ {BuildPartId2},
+ /*BuildPartNames*/ {},
+ /*ManifestPath*/ {},
+ DownloadPath,
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true,
+ .AllowFileClone = m_AllowFileClone});
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Re-download scrambled)");
+ }
+
+ ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2);
+ DownloadFolder(*Output,
+ Workers,
+ Storage,
+ StorageCacheStats,
+ BuildId,
+ {BuildPartId},
+ /*BuildPartNames*/ {},
+ /*ManifestPath*/ {},
+ DownloadPath2,
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = DownloadPath2 / ZenFolderName,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true,
+ .AllowFileClone = m_AllowFileClone});
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Download original)");
+ }
+
+ ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3);
+ DownloadFolder(*Output,
+ Workers,
+ Storage,
+ StorageCacheStats,
+ BuildId,
+ {BuildPartId},
+ /*BuildPartNames*/ {},
+ /*ManifestPath*/ {},
+ DownloadPath3,
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = DownloadPath3 / ZenFolderName,
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .PartialBlockRequestMode = PartialBlockRequestMode,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true,
+ .AllowFileClone = m_AllowFileClone});
+ if (AbortFlag)
+ {
+ throw std::runtime_error("Test aborted. (Download original)");
+ }
+}
+
+BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent)
+: ZenSubCmdBase("multi-test-download", "Download multiple builds sequentially as a test")
+, m_Parent(Parent)
+{
+ auto& Opts = SubOptions();
+ Parent.AddSystemOptions(Opts);
+ Parent.AddCloudOptions(Opts);
+ Parent.AddFileOptions(Opts);
+ Parent.AddOutputOptions(Opts);
+ Parent.AddCacheOptions(Opts);
+ Parent.AddWorkerOptions(Opts);
+ Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
+ Opts.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>");
+ Opts.add_option("",
+ "",
+ "enable-scavenge",
+ "Enable scavenging of data from previouse download locations",
+ cxxopts::value(m_EnableScavenging),
+ "<scavenge>");
+ Opts.add_option("",
+ "",
+ "allow-file-clone",
+ "Enable use of block reference counting when copying files",
+ cxxopts::value(m_AllowFileClone),
+ "<allowclone>");
+ Opts.parse_positional({"local-path"});
+ Opts.positional_help("local-path");
+}
+
+void
+BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ auto& Opts = SubOptions();
+ using namespace builds_impl;
+
+ TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
+ }
+
+ m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_Parent.m_SystemRootDir);
+ CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
+ auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); });
+
+ m_Parent.ParsePath(m_Path, Opts);
+
+ m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName);
+
+ EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts);
+
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
+
+ std::unique_ptr<AuthMgr> Auth;
+ std::string DummyBuildId;
+ StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_Parent.GetZenFolderPath()),
+ DummyBuildId,
+ /*RequireNamespace*/ true,
+ /*RequireBucket*/ true,
+ /*BoostCacheBackgroundWorkerPool*/ false,
+ Auth,
+ Opts);
+
+ std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
+
+ Stopwatch Timer;
+ for (const std::string& BuildIdString : m_BuildIds)
+ {
+ Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString));
+ if (BuildId == Oid::Zero)
{
- throw std::runtime_error("Test aborted. (Re-download scrambled)");
+ throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), Opts.help());
}
-
- ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2);
DownloadFolder(*Output,
Workers,
Storage,
StorageCacheStats,
BuildId,
- {BuildPartId},
+ /*BuildPartIds,*/ {},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
- DownloadPath2,
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = DownloadPath2 / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
+ m_Path,
+ DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ .ZenFolderPath = m_Parent.GetZenFolderPath(),
+ .AllowMultiparts = m_Parent.m_AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = false,
+ .CleanTargetFolder = BuildIdString == m_BuildIds.front(),
.PostDownloadVerify = true,
.PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
- .EnableTargetFolderScavenging = true,
+ .EnableTargetFolderScavenging = false,
.AllowFileClone = m_AllowFileClone});
if (AbortFlag)
{
- throw std::runtime_error("Test aborted. (Download original)");
+ throw std::runtime_error("Multitest aborted");
}
-
- ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3);
- DownloadFolder(*Output,
- Workers,
- Storage,
- StorageCacheStats,
- BuildId,
- {BuildPartId},
- /*BuildPartNames*/ {},
- /*ManifestPath*/ {},
- DownloadPath3,
- DownloadOptions{.SystemRootDir = m_SystemRootDir,
- .ZenFolderPath = DownloadPath3 / ZenFolderName,
- .AllowMultiparts = m_AllowMultiparts,
- .PartialBlockRequestMode = PartialBlockRequestMode,
- .CleanTargetFolder = false,
- .PostDownloadVerify = true,
- .PrimeCacheOnly = false,
- .EnableOtherDownloadsScavenging = m_EnableScavenging,
- .EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ if (!IsQuiet)
{
- throw std::runtime_error("Test aborted. (Download original)");
+ ZEN_CONSOLE("\n");
}
}
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ }
}
} // namespace zen
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 5c80beed5..7ef71e176 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -7,11 +7,231 @@
#include <zenhttp/auth/authmgr.h>
#include <zenhttp/httpclientauth.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildstorageutil.h>
+#include <zenremotestore/partialblockrequestmode.h>
#include <filesystem>
namespace zen {
-class BuildsCommand : public CacheStoreCommand
+class BuildsCommand;
+
+class BuildsListNamespacesSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsListNamespacesSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ bool m_Recursive = false;
+ std::filesystem::path m_ResultPath;
+};
+
+class BuildsListSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsListSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::filesystem::path m_QueryPath;
+ std::filesystem::path m_ResultPath;
+};
+
+class BuildsListBlocksSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsListBlocksSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::string m_BuildId;
+ std::filesystem::path m_ResultPath;
+ uint32_t m_MaxCount = 16;
+};
+
+class BuildsUploadSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsUploadSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::filesystem::path m_Path;
+ std::string m_BuildId;
+ std::string m_BuildPartId;
+ std::string m_BuildPartName;
+ bool m_CreateBuild = false;
+ std::filesystem::path m_BuildMetadataPath;
+ std::string m_BuildMetadata;
+ bool m_Clean = false;
+ uint8_t m_BlockReuseMinPercentLimit = 85;
+ uint64_t m_FindBlockMaxCount = 10000;
+ bool m_PostUploadVerify = false;
+ std::filesystem::path m_ManifestPath;
+ bool m_UploadToZenCache = true;
+};
+
+class BuildsDownloadSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsDownloadSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::filesystem::path m_Path;
+ std::string m_BuildId;
+ std::vector<std::string> m_BuildPartIds;
+ std::vector<std::string> m_BuildPartNames;
+ bool m_Clean = false;
+ bool m_Force = false;
+ bool m_PostDownloadVerify = false;
+ bool m_EnableScavenging = true;
+ std::filesystem::path m_DownloadSpecPath;
+ bool m_UploadToZenCache = true;
+ bool m_PrimeCacheOnly = false;
+ bool m_AllowFileClone = true;
+};
+
+class BuildsLsSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsLsSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::string m_BuildId;
+ std::vector<std::string> m_BuildPartIds;
+ std::vector<std::string> m_BuildPartNames;
+ std::filesystem::path m_ResultPath;
+};
+
+class BuildsDiffSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsDiffSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::filesystem::path m_Path;
+ std::filesystem::path m_DiffPath;
+ bool m_OnlyChunked = false;
+};
+
+class BuildsFetchBlobSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsFetchBlobSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::string m_BuildId;
+ std::string m_BlobHash;
+};
+
+class BuildsPrimeCacheSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsPrimeCacheSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::string m_BuildId;
+ std::vector<std::string> m_BuildPartIds;
+ std::vector<std::string> m_BuildPartNames;
+ bool m_Force = false;
+};
+
+class BuildsPauseSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsPauseSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ int m_ZenProcessId = -1;
+};
+
+class BuildsResumeSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsResumeSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ int m_ZenProcessId = -1;
+};
+
+class BuildsAbortSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsAbortSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ int m_ZenProcessId = -1;
+};
+
+class BuildsValidatePartSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsValidatePartSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::string m_BuildId;
+ std::string m_BuildPartId;
+ std::string m_BuildPartName;
+};
+
+class BuildsTestSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsTestSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::filesystem::path m_Path;
+ std::string m_BuildPartName;
+ std::string m_BuildId;
+ std::string m_BuildPartId;
+ bool m_CreateBuild = false;
+ uint64_t m_FindBlockMaxCount = 10000;
+ uint8_t m_BlockReuseMinPercentLimit = 85;
+ bool m_UploadToZenCache = true;
+ bool m_EnableScavenging = true;
+ bool m_AllowFileClone = true;
+};
+
+class BuildsMultiTestDownloadSubCmd : public ZenSubCmdBase
+{
+public:
+ explicit BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent);
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ BuildsCommand& m_Parent;
+ std::filesystem::path m_Path;
+ std::vector<std::string> m_BuildIds;
+ bool m_EnableScavenging = true;
+ bool m_AllowFileClone = true;
+};
+
+class BuildsCommand : public CacheStoreCmdWithSubCommands
{
public:
static constexpr char Name[] = "builds";
@@ -21,25 +241,67 @@ public:
BuildsCommand();
~BuildsCommand();
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
+ cxxopts::Options& Options() override { return m_Options; }
+
+ // Option-adding helpers (called by subcommand constructors)
+ void AddSystemOptions(cxxopts::Options& Ops);
+ void AddCloudOptions(cxxopts::Options& Ops);
+ void AddFileOptions(cxxopts::Options& Ops);
+ void AddCacheOptions(cxxopts::Options& Ops);
+ void AddOutputOptions(cxxopts::Options& Ops);
+ void AddWorkerOptions(cxxopts::Options& Ops);
+ void AddZenFolderOptions(cxxopts::Options& Ops);
+ void AddChunkingCacheOptions(cxxopts::Options& Ops);
+ void AddWildcardOptions(cxxopts::Options& Ops);
+ void AddExcludeFolderOption(cxxopts::Options& Ops);
+ void AddExcludeExtensionsOption(cxxopts::Options& Ops);
+ void AddMultipartOptions(cxxopts::Options& Ops);
+ void AddPartialBlockRequestOptions(cxxopts::Options& Ops);
+ void AddAppendNewContentOptions(cxxopts::Options& Ops);
+
+ // Shared parsing/factory methods used by subcommand Run() implementations
+ void ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts);
+ StorageInstance CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
+ BuildStorageCache::Statistics& StorageCacheStats,
+ const std::filesystem::path& TempPath,
+ std::string& BuildId,
+ bool RequireNamespace,
+ bool RequireBucket,
+ bool BoostCacheBackgroundWorkerPool,
+ std::unique_ptr<AuthMgr>& Auth,
+ cxxopts::Options& SubOpts);
+ Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts);
+ Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts);
+ std::vector<Oid> ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts);
+ std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts);
+ CbObject ParseBuildMetadata(bool CreateBuild,
+ std::filesystem::path& BuildMetadataPath,
+ const std::string& BuildMetadata,
+ cxxopts::Options& SubOpts);
+ void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts);
+ IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts);
+ EPartialBlockRequestMode ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts);
+ void ParseZenProcessId(int& ZenProcessId);
+ void ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards);
+ void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions);
+
+ void ResolveZenFolderPath(const std::filesystem::path& DefaultPath);
+ const std::filesystem::path& GetZenFolderPath() const { return m_ZenFolderPath; }
-private:
cxxopts::Options m_Options{Name, Description};
+ std::string m_SubCommand;
+ // Shared state populated by AddXxxOptions helpers (bound via cxxopts::value references)
std::filesystem::path m_SystemRootDir;
+ bool m_UseSparseFiles = true;
bool m_PlainProgress = false;
bool m_LogProgress = false;
bool m_Verbose = false;
+ bool m_Quiet = false;
bool m_BoostWorkerCount = false;
bool m_BoostWorkerMemory = false;
bool m_BoostWorkers = false;
- bool m_UseSparseFiles = true;
- bool m_Quiet = false;
- bool m_AllowFileClone = true;
-
- std::filesystem::path m_ZenFolderPath;
// cloud builds
std::string m_OverrideHost;
@@ -51,103 +313,46 @@ private:
std::string m_Namespace;
std::string m_Bucket;
- // file storage
std::filesystem::path m_StoragePath;
bool m_WriteMetadataAsJson = false;
- // cache
std::string m_ZenCacheHost;
- bool m_UploadToZenCache = true;
- bool m_PrimeCacheOnly = false;
-
- std::string m_BuildId;
- bool m_CreateBuild = false;
- std::filesystem::path m_BuildMetadataPath;
- std::string m_BuildMetadata;
- std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path
- std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName
- bool m_Clean = false;
- bool m_Force = false;
- bool m_AppendNewContent = false;
- uint8_t m_BlockReuseMinPercentLimit = 85;
- bool m_AllowMultiparts = true;
- std::string m_AllowPartialBlockRequests = "true";
AuthCommandLineOptions m_AuthOptions;
- std::string m_Verb; // list, upload, download
-
- cxxopts::Options m_ListNamespacesOptions{"list-namespaces", "List available build namespaces"};
- bool m_ListNamespacesRecursive = false;
-
- cxxopts::Options m_ListOptions{"list", "List available builds"};
- std::filesystem::path m_ListQueryPath;
- std::filesystem::path m_ListResultPath;
-
- cxxopts::Options m_ListBlocksOptions{"list-blocks", "List recent blocks"};
- uint32_t m_ListBlocksMaxCount = 16;
-
- std::filesystem::path m_Path;
-
std::string m_IncludeWildcard;
std::string m_ExcludeWildcard;
-
std::string m_ExcludeFolders;
std::string m_ExcludeExtensions;
- cxxopts::Options m_UploadOptions{"upload", "Upload a folder"};
- uint64_t m_FindBlockMaxCount = 10000;
- bool m_PostUploadVerify = false;
std::filesystem::path m_ChunkingCachePath;
- std::filesystem::path m_ManifestPath;
-
- cxxopts::Options m_DownloadOptions{"download", "Download a folder"};
- std::vector<std::string> m_BuildPartNames;
- std::vector<std::string> m_BuildPartIds;
- bool m_PostDownloadVerify = false;
- bool m_EnableScavenging = true;
- std::filesystem::path m_DownloadSpecPath;
- cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"};
- std::filesystem::path m_LsResultPath;
+ bool m_AllowMultiparts = true;
+ std::string m_AllowPartialBlockRequests = "true";
- cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"};
- std::filesystem::path m_DiffPath;
- bool m_OnlyChunked = false;
+ bool m_AppendNewContent = false;
- cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"};
- std::string m_BlobHash;
-
- cxxopts::Options m_PrimeCacheOptions{"prime-cache", "Prime cache from a remote store"};
-
- cxxopts::Options m_PauseOptions{"pause", "Pause an ongoing zen builds process"};
- cxxopts::Options m_ResumeOptions{"resume", "Resume a paused zen builds process"};
- cxxopts::Options m_AbortOptions{"abort", "Abort an ongoing zen builds process"};
-
- int m_ZenProcessId = -1;
-
- cxxopts::Options m_ValidateBuildPartOptions{"validate-part", "Fetch a build part and validate all referenced attachments"};
-
- cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"};
-
- cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"};
- std::vector<std::string> m_BuildIds;
+private:
+ std::filesystem::path m_ZenFolderPath;
- cxxopts::Options* m_SubCommands[15] = {&m_ListNamespacesOptions,
- &m_ListOptions,
- &m_ListBlocksOptions,
- &m_UploadOptions,
- &m_DownloadOptions,
- &m_PauseOptions,
- &m_ResumeOptions,
- &m_AbortOptions,
- &m_DiffOptions,
- &m_LsOptions,
- &m_FetchBlobOptions,
- &m_PrimeCacheOptions,
- &m_ValidateBuildPartOptions,
- &m_TestOptions,
- &m_MultiTestDownloadOptions};
+protected:
+ BuildsListNamespacesSubCmd m_ListNamespacesSubCmd;
+ BuildsListSubCmd m_ListSubCmd;
+ BuildsListBlocksSubCmd m_ListBlocksSubCmd;
+ BuildsUploadSubCmd m_UploadSubCmd;
+ BuildsDownloadSubCmd m_DownloadSubCmd;
+ BuildsLsSubCmd m_LsSubCmd;
+ BuildsDiffSubCmd m_DiffSubCmd;
+ BuildsFetchBlobSubCmd m_FetchBlobSubCmd;
+ BuildsPrimeCacheSubCmd m_PrimeCacheSubCmd;
+ BuildsPauseSubCmd m_PauseSubCmd;
+ BuildsResumeSubCmd m_ResumeSubCmd;
+ BuildsAbortSubCmd m_AbortSubCmd;
+ BuildsValidatePartSubCmd m_ValidatePartSubCmd;
+ BuildsTestSubCmd m_TestSubCmd;
+ BuildsMultiTestDownloadSubCmd m_MultiTestDownloadSubCmd;
+
+ bool OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) override;
};
} // namespace zen