diff options
| author | Stefan Boberg <[email protected]> | 2026-03-16 12:03:00 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2026-03-16 12:03:00 +0100 |
| commit | dfdcdcc4645f6aa6c9c8ef6fd2a0b2f1e5b38070 (patch) | |
| tree | e058fcc16b9344f0809f8826f6cadd8a8ae3ee87 /src | |
| parent | Fix WsHttpSysConnection to use WinIoThreadPool abstraction instead of raw PTP_IO (diff) | |
| parent | Restructure zen builds using subcommands (#834) (diff) | |
| download | zen-dfdcdcc4645f6aa6c9c8ef6fd2a0b2f1e5b38070.tar.xz zen-dfdcdcc4645f6aa6c9c8ef6fd2a0b2f1e5b38070.zip | |
Merge branch 'main' into sb/threadpoolsb/threadpool
Diffstat (limited to 'src')
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 4548 | ||||
| -rw-r--r-- | src/zen/cmds/builds_cmd.h | 383 |
2 files changed, 2647 insertions, 2284 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index b4b4df7c9..93108dd47 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2011,2156 +2011,2231 @@ namespace builds_impl { } // namespace builds_impl ////////////////////////////////////////////////////////////////////////////////////////////////////// +// BuildsCommand — Option-adding helpers +// -BuildsCommand::BuildsCommand() +void +BuildsCommand::AddSystemOptions(cxxopts::Options& Ops) { - using namespace builds_impl; - m_Options.add_options()("h,help", "Print help"); - - auto AddSystemOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>"); - Ops.add_option("", - "", - "use-sparse-files", - "Enable use of sparse files when writing large files. Defaults to true.", - cxxopts::value(m_UseSparseFiles), - "<usesparsefiles>"); - }; - - auto AddCloudOptions = [this](cxxopts::Options& Ops) { - m_AuthOptions.AddOptions(Ops); - - Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>"); - Ops.add_option("cloud build", - "", - "url", - "Cloud Builds host url (legacy - use --override-host)", - cxxopts::value(m_OverrideHost), - "<url>"); - Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>"); - Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>"); - Ops.add_option("cloud build", - "", - "assume-http2", - "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", - cxxopts::value(m_AssumeHttp2), - "<assumehttp2>"); - Ops.add_option("cloud build", - "", - "verbose-http", - "Enable verbose option for http client", - cxxopts::value(m_VerboseHttp), - "<verbosehttp>"); - - Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>"); - Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>"); - Ops.add_option("cloud build", - "", - "allow-redirect", - "Allow redirect of requests", - cxxopts::value(m_AllowRedirect), - "<allow-redirect>"); - }; + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>"); + Ops.add_option("", + "", + "use-sparse-files", + "Enable use of sparse files when writing large files. Defaults to true.", + cxxopts::value(m_UseSparseFiles), + "<usesparsefiles>"); +} - auto AddFileOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>"); - Ops.add_option("filestorage", - "", - "json-metadata", - "Write build, part and block metadata as .json files in addition to .cb files", - cxxopts::value(m_WriteMetadataAsJson), - "<jsonmetadata>"); - }; +void +BuildsCommand::AddCloudOptions(cxxopts::Options& Ops) +{ + m_AuthOptions.AddOptions(Ops); + + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>"); + Ops.add_option("cloud build", + "", + "url", + "Cloud Builds host url (legacy - use --override-host)", + cxxopts::value(m_OverrideHost), + "<url>"); + Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>"); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>"); + Ops.add_option("cloud build", + "", + "assume-http2", + "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", + cxxopts::value(m_AssumeHttp2), + "<assumehttp2>"); + Ops.add_option("cloud build", + "", + "verbose-http", + "Enable verbose option for http client", + cxxopts::value(m_VerboseHttp), + "<verbosehttp>"); + + Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>"); + Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>"); + Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(m_AllowRedirect), "<allow-redirect>"); +} - auto AddCacheOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>"); - }; +void +BuildsCommand::AddFileOptions(cxxopts::Options& Ops) +{ + Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>"); + Ops.add_option("filestorage", + "", + "json-metadata", + "Write build, part and block metadata as .json files in addition to .cb files", + cxxopts::value(m_WriteMetadataAsJson), + "<jsonmetadata>"); +} - auto AddOutputOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("output", - "", - "plain-progress", - "Show progress using plain output", - cxxopts::value(m_PlainProgress), - "<plainprogress>"); - Ops.add_option("output", - "", - "log-progress", - "Write @progress style progress to output", - cxxopts::value(m_LogProgress), - "<logprogress>"); - Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); - Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>"); - }; +void +BuildsCommand::AddCacheOptions(cxxopts::Options& Ops) +{ + Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>"); +} - auto AddWorkerOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", - "", - "boost-worker-count", - "Increase the number of worker threads - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkerCount), - "<boostworkercount>"); - - Ops.add_option("", - "", - "boost-worker-memory", - "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to " - "be less responsive due to high memory usage", - cxxopts::value(m_BoostWorkerMemory), - "<boostworkermemory>"); - - Ops.add_option("", - "", - "boost-workers", - "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkers), - "<boostworkermemory>"); - }; +void +BuildsCommand::AddOutputOptions(cxxopts::Options& Ops) +{ + Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<plainprogress>"); + Ops.add_option("output", + "", + "log-progress", + "Write @progress style progress to output", + cxxopts::value(m_LogProgress), + "<logprogress>"); + Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); + Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>"); +} - auto AddZenFolderOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", - "", - "zen-folder-path", - fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName), - cxxopts::value(m_ZenFolderPath), - "<boostworkers>"); - }; +void +BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) +{ + Ops.add_option("", + "", + "boost-worker-count", + "Increase the number of worker threads - may cause computer to be less responsive", + cxxopts::value(m_BoostWorkerCount), + "<boostworkercount>"); + + Ops.add_option("", + "", + "boost-worker-memory", + "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to " + "be less responsive due to high memory usage", + cxxopts::value(m_BoostWorkerMemory), + "<boostworkermemory>"); + + Ops.add_option("", + "", + "boost-workers", + "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive", + cxxopts::value(m_BoostWorkers), + "<boostworkermemory>"); +} - auto AddChunkingCacheOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", - "", - "chunking-cache-path", - "Path to cache for chunking information of scanned files. Default is empty resulting in no caching", - cxxopts::value(m_ChunkingCachePath), - "<chunkingcachepath>"); - }; +void +BuildsCommand::AddZenFolderOptions(cxxopts::Options& Ops) +{ + Ops.add_option("", + "", + "zen-folder-path", + fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", builds_impl::ZenFolderName), + cxxopts::value(m_ZenFolderPath), + "<boostworkers>"); +} - auto AddWildcardOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", - "", - "wildcard", - "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;", - cxxopts::value(m_IncludeWildcard), - "<wildcard>"); - - Ops.add_option("", - "", - "exclude-wildcard", - "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard " - "include filter", - cxxopts::value(m_ExcludeWildcard), - "<excludewildcard>"); - }; +void +BuildsCommand::AddChunkingCacheOptions(cxxopts::Options& Ops) +{ + Ops.add_option("", + "", + "chunking-cache-path", + "Path to cache for chunking information of scanned files. Default is empty resulting in no caching", + cxxopts::value(m_ChunkingCachePath), + "<chunkingcachepath>"); +} - auto AddExcludeFolderOption = [this](cxxopts::Options& Ops) { - Ops.add_option("", - "", - "exclude-folders", - "Names of folders to exclude, separated by ;", - cxxopts::value(m_ExcludeFolders), - "<excludefolders>"); - }; +void +BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) +{ + Ops.add_option("", + "", + "wildcard", + "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;", + cxxopts::value(m_IncludeWildcard), + "<wildcard>"); + + Ops.add_option("", + "", + "exclude-wildcard", + "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard " + "include filter", + cxxopts::value(m_ExcludeWildcard), + "<excludewildcard>"); +} - auto AddExcludeExtensionsOption = [this](cxxopts::Options& Ops) { - Ops.add_option("", - "", - "exclude-extensions", - "Extensions to exclude, separated by ;" - "include filter", - cxxopts::value(m_ExcludeExtensions), - "<excludeextensions>"); - }; +void +BuildsCommand::AddExcludeFolderOption(cxxopts::Options& Ops) +{ + Ops.add_option("", + "", + "exclude-folders", + "Names of folders to exclude, separated by ;", + cxxopts::value(m_ExcludeFolders), + "<excludefolders>"); +} - auto AddMultipartOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", - "", - "allow-multipart", - "Allow large attachments to be transfered using multipart protocol. Defaults to true.", - cxxopts::value(m_AllowMultiparts), - "<allowmultipart>"); - }; +void +BuildsCommand::AddExcludeExtensionsOption(cxxopts::Options& Ops) +{ + Ops.add_option("", + "", + "exclude-extensions", + "Extensions to exclude, separated by ;" + "include filter", + cxxopts::value(m_ExcludeExtensions), + "<excludeextensions>"); +} - auto AddPartialBlockRequestOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", - "", - "allow-partial-block-requests", - "Allow request for partial chunk blocks.\n" - " false = only full block requests allowed\n" - " mixed = multiple partial block ranges requests per block allowed to zen cache, single partial block range " - "request per block to host\n" - " zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests " - "allowed to host\n" - " true = multiple partial block ranges requests per block allowed to zen cache and host\n" - "Defaults to 'mixed'.", - cxxopts::value(m_AllowPartialBlockRequests), - "<allowpartialblockrequests>"); - }; +void +BuildsCommand::AddMultipartOptions(cxxopts::Options& Ops) +{ + Ops.add_option("", + "", + "allow-multipart", + "Allow large attachments to be transfered using multipart protocol. Defaults to true.", + cxxopts::value(m_AllowMultiparts), + "<allowmultipart>"); +} - auto AddAppendNewContentOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", - "", - "append", - "Decides if the remote data should replace or append to the current local data.\n" - " false = the local content will be replaced by the remote content\n" - " true = the remote data will be overlayed on top of local data\n" - "Defaults to false.", - cxxopts::value(m_AppendNewContent), - "<append>"); - }; +void +BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) +{ + Ops.add_option("", + "", + "allow-partial-block-requests", + "Allow request for partial chunk blocks.\n" + " false = only full block requests allowed\n" + " mixed = multiple partial block ranges requests per block allowed to zen cache, single partial block range " + "request per block to host\n" + " zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests " + "allowed to host\n" + " true = multiple partial block ranges requests per block allowed to zen cache and host\n" + "Defaults to 'mixed'.", + cxxopts::value(m_AllowPartialBlockRequests), + "<allowpartialblockrequests>"); +} - m_Options.add_option("", - "v", - "verb", - "Verb for build - list-namespaces, list, upload, download, diff, fetch-blob, validate-part", - cxxopts::value(m_Verb), - "<verb>"); - m_Options.parse_positional({"verb"}); - m_Options.positional_help("verb"); - - // list-namespaces - AddSystemOptions(m_ListNamespacesOptions); - AddCloudOptions(m_ListNamespacesOptions); - AddFileOptions(m_ListNamespacesOptions); - AddOutputOptions(m_ListNamespacesOptions); - AddZenFolderOptions(m_ListNamespacesOptions); - m_ListNamespacesOptions.add_options()("h,help", "Print help"); - m_ListNamespacesOptions.add_option("", - "", - "recursive", - "Enable fetch of buckets within namespaces also", - cxxopts::value(m_ListNamespacesRecursive), - "<recursive>"); - m_ListNamespacesOptions.add_option( - "", - "", - "result-path", - "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", - cxxopts::value(m_ListResultPath), - "<result-path>"); - m_ListNamespacesOptions.parse_positional({"result-path"}); - m_ListNamespacesOptions.positional_help("result-path"); - - // list - AddSystemOptions(m_ListOptions); - AddCloudOptions(m_ListOptions); - AddFileOptions(m_ListOptions); - AddOutputOptions(m_ListOptions); - AddZenFolderOptions(m_ListOptions); - m_ListOptions.add_options()("h,help", "Print help"); - m_ListOptions.add_option("", - "", - "query-path", - "Path to json or compactbinary file containing list query", - cxxopts::value(m_ListQueryPath), - "<query-path>"); - m_ListOptions.add_option("", - "", - "result-path", - "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", - cxxopts::value(m_ListResultPath), - "<result-path>"); - m_ListOptions.parse_positional({"query-path", "result-path"}); - m_ListOptions.positional_help("query-path result-path"); - - // list-blocks - AddSystemOptions(m_ListBlocksOptions); - AddCloudOptions(m_ListBlocksOptions); - AddZenFolderOptions(m_ListBlocksOptions); - m_ListBlocksOptions.add_options()("h,help", "Print help"); - m_ListBlocksOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - m_ListBlocksOptions.add_option("", - "", - "result-path", - "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", - cxxopts::value(m_ListResultPath), - "<result-path>"); - - m_ListBlocksOptions - .add_option("", "", "max-count", "Maximum number of blocks to list", cxxopts::value(m_ListBlocksMaxCount), "<maxcount>"); - - m_ListBlocksOptions.parse_positional({"build-id"}); - m_ListBlocksOptions.positional_help("build-id"); - - // upload - AddSystemOptions(m_UploadOptions); - AddCloudOptions(m_UploadOptions); - AddFileOptions(m_UploadOptions); - AddOutputOptions(m_UploadOptions); - AddCacheOptions(m_UploadOptions); - AddWorkerOptions(m_UploadOptions); - AddZenFolderOptions(m_UploadOptions); - AddExcludeFolderOption(m_UploadOptions); - AddExcludeExtensionsOption(m_UploadOptions); - AddChunkingCacheOptions(m_UploadOptions); - m_UploadOptions.add_options()("h,help", "Print help"); - m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); - m_UploadOptions.add_option("", - "", - "create-build", - "Set to true to create the containing build, if unset a builds-id must be given and the build already exist", - cxxopts::value(m_CreateBuild), - "<id>"); - m_UploadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - m_UploadOptions.add_option("", - "", - "build-part-id", - "Build part Id, if not given it will be auto generated", - cxxopts::value(m_BuildPartId), - "<id>"); - m_UploadOptions.add_option("", - "", - "build-part-name", - "Name of the build part, if not given it will be be named after the directory name at end of local-path", - cxxopts::value(m_BuildPartName), - "<name>"); - m_UploadOptions.add_option("", - "", - "metadata-path", - "Path to json file that holds the metadata for the build. Requires the create-build option to be set", - cxxopts::value(m_BuildMetadataPath), - "<metadata-path>"); - m_UploadOptions.add_option( - "", - "", - "metadata", - "Key-value pairs separated by ';' with build meta data. (key1=value1;key2=value2). Requires the create-build option to be set", - cxxopts::value(m_BuildMetadata), - "<metadata>"); - m_UploadOptions.add_option("", "", "clean", "Ignore existing blocks", cxxopts::value(m_Clean), "<clean>"); - m_UploadOptions.add_option("", - "", - "block-min-reuse", - "Percent of an existing block that must be relevant for it to be resused. Defaults to 85.", - cxxopts::value(m_BlockReuseMinPercentLimit), - "<minreuse>"); - m_UploadOptions.add_option("cache", - "", - "zen-cache-upload", - "Upload data downloaded from remote host to zen cache", - cxxopts::value(m_UploadToZenCache), - "<uploadtozencache>"); - - AddMultipartOptions(m_UploadOptions); - - m_UploadOptions.add_option("", - "", - "manifest-path", - "Path to a text file with one line of <local path>[TAB]<modification date> per file to include or a " - "structured .json file describing the parts", - cxxopts::value(m_ManifestPath), - "<manifestpath>"); - - m_UploadOptions - .add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), "<verify>"); - m_UploadOptions.add_option("", - "", - "find-max-block-count", - "The maximum number of blocks we search for in the build context", - cxxopts::value(m_FindBlockMaxCount), - "<maxblockcount>"); - - m_UploadOptions.parse_positional({"local-path", "build-id"}); - m_UploadOptions.positional_help("local-path build-id"); - - // download - AddSystemOptions(m_DownloadOptions); - AddCloudOptions(m_DownloadOptions); - AddFileOptions(m_DownloadOptions); - AddOutputOptions(m_DownloadOptions); - AddCacheOptions(m_DownloadOptions); - AddZenFolderOptions(m_DownloadOptions); - AddWorkerOptions(m_DownloadOptions); - AddWildcardOptions(m_DownloadOptions); - AddAppendNewContentOptions(m_DownloadOptions); - AddExcludeFolderOption(m_DownloadOptions); - - m_DownloadOptions.add_option("cache", - "", - "cache-prime-only", - "Only download blobs missing in cache and upload to cache", - cxxopts::value(m_PrimeCacheOnly), - "<cacheprimeonly>"); - - m_DownloadOptions.add_options()("h,help", "Print help"); - m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); - m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - m_DownloadOptions.add_option( - "", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); - m_DownloadOptions.add_option("", - "", - "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", - cxxopts::value(m_BuildPartNames), - "<name>"); - m_DownloadOptions.add_option("", - "", - "clean", - "Delete all data in target folder that is not part of the downloaded content", - cxxopts::value(m_Clean), - "<clean>"); - m_DownloadOptions.add_option("", - "", - "force", - "Force download of all content by ignoring any existing local content", - cxxopts::value(m_Force), - "<force>"); - m_DownloadOptions.add_option("cache", - "", - "zen-cache-upload", - "Upload data downloaded from remote host to zen cache", - cxxopts::value(m_UploadToZenCache), - "<uploadtozencache>"); - AddMultipartOptions(m_DownloadOptions); - - AddPartialBlockRequestOptions(m_DownloadOptions); - - m_DownloadOptions.add_option( - "", - "", - "download-spec-path", - "Path to a text file with one line of <local path> per file to include or a structured .json file describing what to download.", - cxxopts::value(m_DownloadSpecPath), - "<downloadspecpath>"); +void +BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) +{ + Ops.add_option("", + "", + "append", + "Decides if the remote data should replace or append to the current local data.\n" + " false = the local content will be replaced by the remote content\n" + " true = the remote data will be overlayed on top of local data\n" + "Defaults to false.", + cxxopts::value(m_AppendNewContent), + "<append>"); +} - m_DownloadOptions - .add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), "<verify>"); - m_DownloadOptions.add_option("", - "", - "enable-scavenge", - "Enable scavenging of data from previouse download locations", - cxxopts::value(m_EnableScavenging), - "<scavenge>"); - m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); - m_DownloadOptions.positional_help("local-path build-id build-part-name"); - - m_DownloadOptions.add_option("", - "", - "allow-file-clone", - "Enable use of block reference counting when copying files", - cxxopts::value(m_AllowFileClone), - "<allowclone>"); - - // ls - AddSystemOptions(m_LsOptions); - AddCloudOptions(m_LsOptions); - AddFileOptions(m_LsOptions); - AddOutputOptions(m_LsOptions); - AddCacheOptions(m_LsOptions); - AddZenFolderOptions(m_LsOptions); - AddWorkerOptions(m_LsOptions); - AddWildcardOptions(m_LsOptions); - - m_LsOptions.add_options()("h,help", "Print help"); - m_LsOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - m_LsOptions.add_option( - "", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); - m_LsOptions.add_option("", - "", - "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", - cxxopts::value(m_BuildPartNames), - "<name>"); - - m_LsOptions.add_option("", - "", - "result-path", - "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", - cxxopts::value(m_LsResultPath), - "<result-path>"); - - m_LsOptions.add_option("", - "o", - "output-path", - "Path to output, extension .json or .cb (compac binary). Default is output to console", - cxxopts::value(m_LsResultPath), - "<output-path>"); - - m_LsOptions.parse_positional({"build-id", "wildcard"}); - m_LsOptions.positional_help("build-id wildcard"); - - // diff - AddOutputOptions(m_DiffOptions); - AddWorkerOptions(m_DiffOptions); - AddExcludeFolderOption(m_DiffOptions); - AddExcludeExtensionsOption(m_DiffOptions); - AddChunkingCacheOptions(m_DiffOptions); - m_DiffOptions.add_options()("h,help", "Print help"); - m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); - m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>"); - m_DiffOptions.add_option("", - "", - "only-chunked", - "Skip files from diff summation that are not processed with chunking", - cxxopts::value(m_OnlyChunked), - "<only-chunked>"); - m_DiffOptions.parse_positional({"local-path", "compare-path"}); - m_DiffOptions.positional_help("local-path compare-path"); - - // test - AddSystemOptions(m_TestOptions); - AddCloudOptions(m_TestOptions); - AddFileOptions(m_TestOptions); - AddOutputOptions(m_TestOptions); - AddCacheOptions(m_TestOptions); - AddWorkerOptions(m_TestOptions); - m_TestOptions.add_options()("h,help", "Print help"); - m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); - AddMultipartOptions(m_TestOptions); - AddPartialBlockRequestOptions(m_TestOptions); - AddWildcardOptions(m_TestOptions); - AddAppendNewContentOptions(m_TestOptions); - AddChunkingCacheOptions(m_TestOptions); - - m_TestOptions.add_option("", - "", - "enable-scavenge", - "Enable scavenging of data from previouse download locations", - cxxopts::value(m_EnableScavenging), - "<scavenge>"); - m_TestOptions.add_option("", - "", - "allow-file-clone", - "Enable use of block reference counting when copying files", - cxxopts::value(m_AllowFileClone), - "<allowclone>"); - m_TestOptions.parse_positional({"local-path"}); - m_TestOptions.positional_help("local-path"); - - // fetch-blob - AddSystemOptions(m_FetchBlobOptions); - AddCloudOptions(m_FetchBlobOptions); - AddFileOptions(m_FetchBlobOptions); - AddOutputOptions(m_FetchBlobOptions); - AddCacheOptions(m_FetchBlobOptions); - AddZenFolderOptions(m_FetchBlobOptions); - m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - m_FetchBlobOptions - .add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>"); - m_FetchBlobOptions.parse_positional({"build-id", "blob-hash"}); - m_FetchBlobOptions.positional_help("build-id blob-hash"); - - // prime-cache - AddSystemOptions(m_PrimeCacheOptions); - AddCloudOptions(m_PrimeCacheOptions); - AddFileOptions(m_PrimeCacheOptions); - AddOutputOptions(m_PrimeCacheOptions); - AddCacheOptions(m_PrimeCacheOptions); - AddWorkerOptions(m_PrimeCacheOptions); - AddZenFolderOptions(m_PrimeCacheOptions); - m_PrimeCacheOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - m_PrimeCacheOptions.add_option( - "", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); - m_PrimeCacheOptions.add_option("", - "", - "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", - cxxopts::value(m_BuildPartNames), - "<name>"); - - m_PrimeCacheOptions.add_option("", - "", - "force", - "Force download of all blobs by ignoring any existing blobs in cache", - cxxopts::value(m_Force), - "<force>"); - - m_PrimeCacheOptions.parse_positional({"build-id"}); - m_PrimeCacheOptions.positional_help("build-id"); - - auto AddZenProcessOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - }; - AddZenProcessOptions(m_PauseOptions); - m_PauseOptions.parse_positional({"process-id"}); - m_PauseOptions.positional_help("process-id"); - - AddZenProcessOptions(m_ResumeOptions); - m_ResumeOptions.parse_positional({"process-id"}); - m_ResumeOptions.positional_help("process-id"); - - AddZenProcessOptions(m_AbortOptions); - m_AbortOptions.parse_positional({"process-id"}); - m_AbortOptions.positional_help("process-id"); - - // validate-part - AddSystemOptions(m_ValidateBuildPartOptions); - AddCloudOptions(m_ValidateBuildPartOptions); - AddFileOptions(m_ValidateBuildPartOptions); - AddOutputOptions(m_ValidateBuildPartOptions); - AddWorkerOptions(m_ValidateBuildPartOptions); - AddZenFolderOptions(m_ValidateBuildPartOptions); - m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - m_ValidateBuildPartOptions.add_option("", - "", - "build-part-id", - "Build part Id, if not given it will be auto generated", - cxxopts::value(m_BuildPartId), - "<id>"); - m_ValidateBuildPartOptions.add_option( - "", - "", - "build-part-name", - "Name of the build part, if not given it will be be named after the directory name at end of local-path", - cxxopts::value(m_BuildPartName), - "<name>"); - m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"}); - m_ValidateBuildPartOptions.positional_help("build-id build-part-id"); - - // multi-test-download - AddSystemOptions(m_MultiTestDownloadOptions); - AddCloudOptions(m_MultiTestDownloadOptions); - AddFileOptions(m_MultiTestDownloadOptions); - AddOutputOptions(m_MultiTestDownloadOptions); - AddCacheOptions(m_MultiTestDownloadOptions); - AddWorkerOptions(m_MultiTestDownloadOptions); - m_MultiTestDownloadOptions - .add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); - m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>"); - m_MultiTestDownloadOptions.add_option("", - "", - "enable-scavenge", - "Enable scavenging of data from previouse download locations", - cxxopts::value(m_EnableScavenging), - "<scavenge>"); - m_MultiTestDownloadOptions.add_option("", - "", - "allow-file-clone", - "Enable use of block reference counting when copying files", - cxxopts::value(m_AllowFileClone), - "<allowclone>"); - m_MultiTestDownloadOptions.parse_positional({"local-path"}); - m_MultiTestDownloadOptions.positional_help("local-path"); +BuildsCommand::BuildsCommand() +: m_ListNamespacesSubCmd(*this) +, m_ListSubCmd(*this) +, m_ListBlocksSubCmd(*this) +, m_UploadSubCmd(*this) +, m_DownloadSubCmd(*this) +, m_LsSubCmd(*this) +, m_DiffSubCmd(*this) +, m_FetchBlobSubCmd(*this) +, m_PrimeCacheSubCmd(*this) +, m_PauseSubCmd(*this) +, m_ResumeSubCmd(*this) +, m_AbortSubCmd(*this) +, m_ValidatePartSubCmd(*this) +, m_TestSubCmd(*this) +, m_MultiTestDownloadSubCmd(*this) +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), ""); + m_Options.parse_positional({"subcommand"}); + + AddSubCommand(m_ListNamespacesSubCmd); + AddSubCommand(m_ListSubCmd); + AddSubCommand(m_ListBlocksSubCmd); + AddSubCommand(m_UploadSubCmd); + AddSubCommand(m_DownloadSubCmd); + AddSubCommand(m_LsSubCmd); + AddSubCommand(m_DiffSubCmd); + AddSubCommand(m_FetchBlobSubCmd); + AddSubCommand(m_PrimeCacheSubCmd); + AddSubCommand(m_PauseSubCmd); + AddSubCommand(m_ResumeSubCmd); + AddSubCommand(m_AbortSubCmd); + AddSubCommand(m_ValidatePartSubCmd); + AddSubCommand(m_TestSubCmd); + AddSubCommand(m_MultiTestDownloadSubCmd); } BuildsCommand::~BuildsCommand() = default; -void -BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +bool +BuildsCommand::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/) { using namespace builds_impl; - ZEN_UNUSED(GlobalOptions); signal(SIGINT, SignalCallbackHandler); #if ZEN_PLATFORM_WINDOWS signal(SIGBREAK, SignalCallbackHandler); #endif // ZEN_PLATFORM_WINDOWS - using namespace std::literals; - - std::vector<char*> SubCommandArguments; - cxxopts::Options* SubOption = nullptr; - int ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments); - if (!ParseOptions(ParentCommandArgCount, argv)) + // Validate output options + if (m_Verbose && m_Quiet) { - return; + throw OptionParseException("'--verbose' conflicts with '--quiet'", {}); } - - if (SubOption == nullptr) + if (m_LogProgress && m_PlainProgress) { - throw OptionParseException("'verb' option is required", m_Options.help()); + throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", {}); + } + if (m_LogProgress && m_Quiet) + { + throw OptionParseException("'--quiet' conflicts with '--log-progress'", {}); + } + if (m_PlainProgress && m_Quiet) + { + throw OptionParseException("'--quiet' conflicts with '--plain-progress'", {}); + } + IsVerbose = m_Verbose; + IsQuiet = m_Quiet; + if (m_LogProgress) + { + ProgressMode = ProgressBar::Mode::Log; + } + else if (m_PlainProgress) + { + ProgressMode = ProgressBar::Mode::Plain; + } + else if (m_Verbose) + { + ProgressMode = ProgressBar::Mode::Plain; + } + else if (IsQuiet) + { + ProgressMode = ProgressBar::Mode::Quiet; + } + else + { + ProgressMode = ProgressBar::Mode::Pretty; } - if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data())) + if (m_BoostWorkers) { - return; + m_BoostWorkerCount = true; + m_BoostWorkerMemory = true; } - auto ParseSystemOptions = [&]() { - if (m_SystemRootDir.empty()) - { - m_SystemRootDir = PickDefaultSystemRootDirectory(); - } - MakeSafeAbsolutePathInPlace(m_SystemRootDir); - }; - ParseSystemOptions(); + // Parse system options + if (m_SystemRootDir.empty()) + { + m_SystemRootDir = PickDefaultSystemRootDirectory(); + } + MakeSafeAbsolutePathInPlace(m_SystemRootDir); - auto ParseStorageOptions = [&](bool RequireNamespace, bool RequireBucket) { - if (!m_Url.empty()) - { - if (!m_Host.empty()) - { - throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOption->help()); - } - if (!m_Bucket.empty()) - { - throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), - SubOption->help()); - } - if (!m_BuildId.empty()) - { - throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", m_BuildId, m_Url), - SubOption->help()); - } - if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, m_BuildId)) - { - throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", - SubOption->help()); - } - } + UseSparseFiles = m_UseSparseFiles; - if (!m_OverrideHost.empty() || !m_Host.empty()) - { - if (!m_StoragePath.empty()) - { - throw OptionParseException( - fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath), - SubOption->help()); - } - if (RequireNamespace && m_Namespace.empty()) - { - throw OptionParseException("'--namespace' is required", SubOption->help()); - } - if (RequireBucket && m_Bucket.empty()) - { - throw OptionParseException("'--bucket' is required", SubOption->help()); - } - } - else if (m_StoragePath.empty()) - { - throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help()); - } - MakeSafeAbsolutePathInPlace(m_StoragePath); - }; + return true; +} - auto ParseOutputOptions = [&]() { - if (m_Verbose && m_Quiet) - { - throw OptionParseException("'--verbose' conflicts with '--quiet'", SubOption->help()); - } - if (m_LogProgress && m_PlainProgress) - { - throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", SubOption->help()); - } - if (m_LogProgress && m_Quiet) +void +BuildsCommand::ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts) +{ + if (!m_Url.empty()) + { + if (!m_Host.empty()) { - throw OptionParseException("'--quiet' conflicts with '--log-progress'", SubOption->help()); + throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOpts.help()); } - if (m_PlainProgress && m_Quiet) + if (!m_Bucket.empty()) { - throw OptionParseException("'--quiet' conflicts with '--plain-progress'", SubOption->help()); + throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), SubOpts.help()); } - IsVerbose = m_Verbose; - IsQuiet = m_Quiet; - if (m_LogProgress) + if (!BuildId.empty()) { - ProgressMode = ProgressBar::Mode::Log; + throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Url), SubOpts.help()); } - else if (m_PlainProgress) + if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, BuildId)) { - ProgressMode = ProgressBar::Mode::Plain; + throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOpts.help()); } - else if (m_Verbose) + } + + if (!m_OverrideHost.empty() || !m_Host.empty()) + { + if (!m_StoragePath.empty()) { - ProgressMode = ProgressBar::Mode::Plain; + throw OptionParseException( + fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath), + SubOpts.help()); } - else if (IsQuiet) + if (RequireNamespace && m_Namespace.empty()) { - ProgressMode = ProgressBar::Mode::Quiet; + throw OptionParseException("'--namespace' is required", SubOpts.help()); } - else + if (RequireBucket && m_Bucket.empty()) { - ProgressMode = ProgressBar::Mode::Pretty; + throw OptionParseException("'--bucket' is required", SubOpts.help()); } - }; - ParseOutputOptions(); - - if (m_BoostWorkers) + } + else if (m_StoragePath.empty()) { - m_BoostWorkerCount = true; - m_BoostWorkerMemory = true; + throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } + MakeSafeAbsolutePathInPlace(m_StoragePath); +} - std::unique_ptr<AuthMgr> Auth; - - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - - auto CreateBuildStorage = [&](BuildStorageBase::Statistics& StorageStats, +StorageInstance +BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, BuildStorageCache::Statistics& StorageCacheStats, const std::filesystem::path& TempPath, + std::string& BuildId, bool RequireNamespace, bool RequireBucket, - bool BoostCacheBackgroundWorkerPool) -> StorageInstance { - ParseStorageOptions(RequireNamespace, RequireBucket); + bool BoostCacheBackgroundWorkerPool, + std::unique_ptr<AuthMgr>& Auth, + cxxopts::Options& SubOpts) +{ + using namespace builds_impl; - HttpClientSettings ClientSettings{ - .LogCategory = "httpbuildsclient", - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 2, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; + ParseStorageOptions(BuildId, RequireNamespace, RequireBucket, SubOpts); - std::string StorageDescription; - std::string CacheDescription; + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 2, + .Verbose = m_VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; - StorageInstance Result; + std::string StorageDescription; + std::string CacheDescription; - if (!m_Host.empty() || !m_OverrideHost.empty()) - { - m_AuthOptions.ParseOptions(*SubOption, - m_SystemRootDir, - ClientSettings, - m_Host.empty() ? m_OverrideHost : m_Host, - Auth, - IsQuiet, - /*Hidden*/ false, - m_Verbose); - - BuildStorageResolveResult ResolveRes = - ResolveBuildStorage(*Output, ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose); - if (!ResolveRes.Cloud.Address.empty()) + StorageInstance Result; + + if (!m_Host.empty() || !m_OverrideHost.empty()) + { + m_AuthOptions.ParseOptions(SubOpts, + m_SystemRootDir, + ClientSettings, + m_Host.empty() ? m_OverrideHost : m_Host, + Auth, + IsQuiet, + /*Hidden*/ false, + m_Verbose); + + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*CreateConsoleLogOutput(ProgressMode), + ClientSettings, + m_Host, + m_OverrideHost, + m_ZenCacheHost, + ZenCacheResolveMode::All, + m_Verbose); + if (!ResolveRes.Cloud.Address.empty()) + { + ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; + Result.BuildStorageHttp = + std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); + + Result.BuildStorage = CreateJupiterBuildStorage(Log(), + *Result.BuildStorageHttp, + StorageStats, + m_Namespace, + m_Bucket, + m_AllowRedirect, + TempPath / "storage"); + Result.BuildStorageHost = ResolveRes.Cloud; + + uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0; + + StorageDescription = + fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}", + ResolveRes.Cloud.Name, + (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket, + NiceLatencyNs(HostLatencyNs)); + + if (!ResolveRes.Cache.Address.empty()) { - ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; - Result.BuildStorageHttp = - std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); - - Result.BuildStorage = CreateJupiterBuildStorage(Log(), - *Result.BuildStorageHttp, - StorageStats, - m_Namespace, - m_Bucket, - m_AllowRedirect, - TempPath / "storage"); - Result.BuildStorageHost = ResolveRes.Cloud; - - uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0; - - StorageDescription = - fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}", - ResolveRes.Cloud.Name, - (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), - Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket, - NiceLatencyNs(HostLatencyNs)); - - if (!ResolveRes.Cache.Address.empty()) + Result.CacheHttp = std::make_unique<HttpClient>( + ResolveRes.Cache.Address, + HttpClientSettings{ + .LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, + .AllowResume = true, + .RetryCount = 0, + .Verbose = m_VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, + []() { return AbortFlag.load(); }); + Result.CacheStorage = + CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = ResolveRes.Cache; + + uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; + + CacheDescription = + fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}", + ResolveRes.Cache.Name, + (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address), + Result.CacheHttp->GetSessionId(), + NiceLatencyNs(CacheLatencyNs)); + + if (!m_Namespace.empty()) { - Result.CacheHttp = std::make_unique<HttpClient>( - ResolveRes.Cache.Address, - HttpClientSettings{ - .LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{3000}, - .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, - .AllowResume = true, - .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); - Result.CacheStorage = - CreateZenBuildStorageCache(*Result.CacheHttp, - StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheHost = ResolveRes.Cache; - - uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; - - CacheDescription = - fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}", - ResolveRes.Cache.Name, - (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address), - Result.CacheHttp->GetSessionId(), - NiceLatencyNs(CacheLatencyNs)); - - if (!m_Namespace.empty()) - { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); - } - if (!m_Bucket.empty()) - { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); - } + CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); } - } - } - else if (!m_StoragePath.empty()) - { - StorageDescription = fmt::format("folder {}", m_StoragePath); - Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - - Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), - .Name = "Disk", - .LatencySec = 1.0 / 100000, // 1 us - .Caps = {.MaxRangeCountPerRequest = 2048u}}; - - if (!m_ZenCacheHost.empty()) - { - ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); - - if (TestResult.Success) + if (!m_Bucket.empty()) { - Result.CacheHttp = std::make_unique<HttpClient>( - m_ZenCacheHost, - HttpClientSettings{ - .LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{3000}, - .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); - - Result.CacheStorage = - CreateZenBuildStorageCache(*Result.CacheHttp, - StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheHost = - BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, - .Name = m_ZenCacheHost, - .AssumeHttp2 = m_AssumeHttp2, - .LatencySec = TestResult.LatencySeconds, - .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; - - CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); - - if (!m_Namespace.empty()) - { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); - } - if (!m_Bucket.empty()) - { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); - } + CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); } } } - else - { - throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help()); - } + } + else if (!m_StoragePath.empty()) + { + StorageDescription = fmt::format("folder {}", m_StoragePath); + Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - if (!IsQuiet) - { - ZEN_CONSOLE("Remote: {}", StorageDescription); - if (!Result.CacheHost.Name.empty()) - { - ZEN_CONSOLE("Cache : {}", CacheDescription); - } - } - return Result; - }; + Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), + .Name = "Disk", + .LatencySec = 1.0 / 100000, // 1 us + .Caps = {.MaxRangeCountPerRequest = 2048u}}; - auto ParsePath = [&]() { - if (m_Path.empty()) + if (!m_ZenCacheHost.empty()) { - throw OptionParseException("'--local-path' is required", SubOption->help()); - } - MakeSafeAbsolutePathInPlace(m_Path); - }; + ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); - auto ParseFileFilters = [&](std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) { - auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) { - ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) { - if (!Wildcard.empty()) + if (TestResult.Success) + { + Result.CacheHttp = std::make_unique<HttpClient>( + m_ZenCacheHost, + HttpClientSettings{ + .LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 0, + .Verbose = m_VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, + []() { return AbortFlag.load(); }); + + Result.CacheStorage = + CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, + .Name = m_ZenCacheHost, + .AssumeHttp2 = m_AssumeHttp2, + .LatencySec = TestResult.LatencySeconds, + .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; + + CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); + + if (!m_Namespace.empty()) { - std::string CleanWildcard(ToLower(Wildcard)); - for (auto It = begin(CleanWildcard); It != end(CleanWildcard); It++) - { - if (*It == '\\') - { - *It = '/'; - } - } - - if (CleanWildcard.starts_with("./") || CleanWildcard.starts_with(".\\")) - { - CleanWildcard = CleanWildcard.substr(2); - } - - Output.emplace_back(std::move(CleanWildcard)); + CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); } - return true; - }); - }; - - SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards); - SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards); - }; - - auto ParseExcludeFolderAndExtension = [&](std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) { - auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) { - ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) { - if (!Exclusion.empty()) + if (!m_Bucket.empty()) { - std::string CleanExclusion(ToLower(Exclusion)); - if (CleanExclusion.length() > 2 && CleanExclusion.front() == '"' && CleanExclusion.back() == '"') - { - CleanExclusion = CleanExclusion.substr(1, CleanExclusion.length() - 2); - } - Output.emplace_back(std::move(CleanExclusion)); + CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); } - return true; - }); - }; - - SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders); - SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions); - }; - - auto ParseDiffPath = [&]() { - if (m_DiffPath.empty()) - { - throw OptionParseException("'--compare-path' is required", SubOption->help()); + } } - MakeSafeAbsolutePathInPlace(m_DiffPath); - }; + } + else + { + throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); + } - auto ParseBlobHash = [&]() -> IoHash { - if (m_BlobHash.empty()) + if (!IsQuiet) + { + ZEN_CONSOLE("Remote: {}", StorageDescription); + if (!Result.CacheHost.Name.empty()) { - throw OptionParseException("'--blob-hash' is required", SubOption->help()); + ZEN_CONSOLE("Cache : {}", CacheDescription); } + } + return Result; +} - if (m_BlobHash.length() != IoHash::StringLength) - { - throw OptionParseException( - fmt::format("'--blob-hash' ('{}') is malformed, it must be {} characters long", m_BlobHash, IoHash::StringLength), - SubOption->help()); - } +Oid +BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) +{ + if (BuildIdStr.length() != Oid::StringLength) + { + throw OptionParseException( + fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildIdStr, Oid::StringLength), + SubOpts.help()); + } + else if (Oid BuildId = Oid::FromHexString(BuildIdStr); BuildId == Oid::Zero) + { + throw OptionParseException(fmt::format("'--build-id' ('{}') is invalid", BuildIdStr), SubOpts.help()); + } + else + { + return BuildId; + } +} + +Oid +BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) +{ + if (BuildPartIdStr.length() != Oid::StringLength) + { + throw OptionParseException( + fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildPartIdStr, Oid::StringLength), + SubOpts.help()); + } + else if (Oid BuildPartId = Oid::FromHexString(BuildPartIdStr); BuildPartId == Oid::Zero) + { + throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildPartIdStr), SubOpts.help()); + } + else + { + return BuildPartId; + } +} - IoHash BlobHash; - if (!IoHash::TryParse(m_BlobHash, BlobHash)) +std::vector<Oid> +BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts) +{ + std::vector<Oid> BuildPartIds; + for (const std::string& BuildPartId : BuildPartIdStrs) + { + BuildPartIds.push_back(Oid::TryFromHexString(RemoveQuotes(BuildPartId))); + if (BuildPartIds.back() == Oid::Zero) { - throw OptionParseException(fmt::format("'--blob-hash' ('{}') is malformed", m_BlobHash), SubOption->help()); + throw OptionParseException(fmt::format("'--build-part-id' ('{}') is malformed", BuildPartId), SubOpts.help()); } + } + return BuildPartIds; +} - return BlobHash; - }; - - auto ParseBuildId = [&]() -> Oid { - if (m_BuildId.length() != Oid::StringLength) +std::vector<std::string> +BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts) +{ + std::vector<std::string> BuildPartNames; + for (const std::string& BuildPartName : BuildPartNameStrs) + { + BuildPartNames.push_back(std::string(RemoveQuotes(BuildPartName))); + if (BuildPartNames.back().empty()) { - throw OptionParseException( - fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildId, Oid::StringLength), - SubOption->help()); + throw OptionParseException(fmt::format("'--build-part-names' ('{}') is invalid", BuildPartName), SubOpts.help()); } - else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero) + } + return BuildPartNames; +} + +CbObject +BuildsCommand::ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts) +{ + if (CreateBuild) + { + if (BuildMetadataPath.empty() && BuildMetadata.empty()) { - throw OptionParseException(fmt::format("'--build-id' ('{}') is invalid", m_BuildId), SubOption->help()); + throw OptionParseException("'--metadata-path' or '--metadata' is required", SubOpts.help()); } - else + if (!BuildMetadataPath.empty() && !BuildMetadata.empty()) { - return BuildId; + throw OptionParseException( + fmt::format("'--metadata-path' ('{}') conflicts with '--metadata' ('{}')", BuildMetadataPath.string(), BuildMetadata), + SubOpts.help()); } - }; - auto ParseBuildPartId = [&]() -> Oid { - if (m_BuildPartId.length() != Oid::StringLength) + if (!BuildMetadataPath.empty()) { - throw OptionParseException( - fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildPartId, Oid::StringLength), - SubOption->help()); + MakeSafeAbsolutePathInPlace(BuildMetadataPath); + IoBuffer MetaDataJson = ReadFile(BuildMetadataPath).Flatten(); + std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + CbObject MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (!JsonError.empty()) + { + throw std::runtime_error( + fmt::format("build metadata file '{}' is malformed. Reason: '{}'", BuildMetadataPath.string(), JsonError)); + } + return MetaData; } - else if (Oid BuildPartId = Oid::FromHexString(m_BuildPartId); BuildPartId == Oid::Zero) + if (!BuildMetadata.empty()) { - throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", m_BuildPartId), SubOption->help()); + CbObjectWriter MetaDataWriter(1024); + ForEachStrTok(BuildMetadata, ';', [&](std::string_view Pair) { + size_t SplitPos = Pair.find('='); + if (SplitPos == std::string::npos || SplitPos == 0) + { + throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair)); + } + MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); + return true; + }); + return MetaDataWriter.Save(); } - else + } + else + { + if (!BuildMetadataPath.empty()) { - return BuildPartId; + throw OptionParseException("'--metadata-path' requires '--create-build'", SubOpts.help()); } - }; - - auto ParseBuildPartIds = [&]() -> std::vector<Oid> { - std::vector<Oid> BuildPartIds; - for (const std::string& BuildPartId : m_BuildPartIds) + if (!BuildMetadata.empty()) { - BuildPartIds.push_back(Oid::TryFromHexString(RemoveQuotes(BuildPartId))); - if (BuildPartIds.back() == Oid::Zero) - { - throw OptionParseException(fmt::format("'--build-part-id' ('{}') is malformed", BuildPartId), SubOption->help()); - } + throw OptionParseException("'--metadata' requires '--create-build'", SubOpts.help()); } - return BuildPartIds; - }; + } + return {}; +} - auto ParseBuildPartNames = [&]() -> std::vector<std::string> { - std::vector<std::string> BuildPartNames; - for (const std::string& BuildPartName : m_BuildPartNames) - { - BuildPartNames.push_back(std::string(RemoveQuotes(BuildPartName))); - if (BuildPartNames.back().empty()) +void +BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) +{ + if (Path.empty()) + { + throw OptionParseException("'--local-path' is required", SubOpts.help()); + } + MakeSafeAbsolutePathInPlace(Path); +} + +IoHash +BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) +{ + if (BlobHashStr.empty()) + { + throw OptionParseException("'--blob-hash' is required", SubOpts.help()); + } + + if (BlobHashStr.length() != IoHash::StringLength) + { + throw OptionParseException( + fmt::format("'--blob-hash' ('{}') is malformed, it must be {} characters long", BlobHashStr, IoHash::StringLength), + SubOpts.help()); + } + + IoHash BlobHash; + if (!IoHash::TryParse(BlobHashStr, BlobHash)) + { + throw OptionParseException(fmt::format("'--blob-hash' ('{}') is malformed", BlobHashStr), SubOpts.help()); + } + + return BlobHash; +} + +void +BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) +{ + auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) { + ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) { + if (!Wildcard.empty()) { - throw OptionParseException(fmt::format("'--build-part-names' ('{}') is invalid", BuildPartName), SubOption->help()); + std::string CleanWildcard(ToLower(Wildcard)); + for (auto It = begin(CleanWildcard); It != end(CleanWildcard); It++) + { + if (*It == '\\') + { + *It = '/'; + } + } + + if (CleanWildcard.starts_with("./") || CleanWildcard.starts_with(".\\")) + { + CleanWildcard = CleanWildcard.substr(2); + } + + Output.emplace_back(std::move(CleanWildcard)); } - } - return BuildPartNames; + return true; + }); }; - auto ParseBuildMetadata = [&]() -> CbObject { - if (m_CreateBuild) - { - if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty()) - { - throw OptionParseException("'--metadata-path' or '--metadata' is required", SubOption->help()); - } - if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty()) - { - throw OptionParseException( - fmt::format("'--metadata-path' ('{}') conflicts with '--metadata' ('{}')", m_BuildMetadataPath, m_BuildMetadata), - SubOption->help()); - } + SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards); + SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards); +} - if (!m_BuildMetadataPath.empty()) +void +BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) +{ + auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) { + ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) { + if (!Exclusion.empty()) { - MakeSafeAbsolutePathInPlace(m_BuildMetadataPath); - IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten(); - std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); - std::string JsonError; - CbObject MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); - if (!JsonError.empty()) + std::string CleanExclusion(ToLower(Exclusion)); + if (CleanExclusion.length() > 2 && CleanExclusion.front() == '"' && CleanExclusion.back() == '"') { - throw std::runtime_error( - fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError)); + CleanExclusion = CleanExclusion.substr(1, CleanExclusion.length() - 2); } - return MetaData; - } - if (!m_BuildMetadata.empty()) - { - CbObjectWriter MetaDataWriter(1024); - ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) { - size_t SplitPos = Pair.find('='); - if (SplitPos == std::string::npos || SplitPos == 0) - { - throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair)); - } - MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); - return true; - }); - return MetaDataWriter.Save(); + Output.emplace_back(std::move(CleanExclusion)); } + return true; + }); + }; + + SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders); + SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions); +} + +void +BuildsCommand::ResolveZenFolderPath(const std::filesystem::path& DefaultPath) +{ + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = DefaultPath; + } + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); +} + +EPartialBlockRequestMode +BuildsCommand::ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts) +{ + if (PrimeCacheOnly) + { + return EPartialBlockRequestMode::Off; + } + EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests); + if (Mode == EPartialBlockRequestMode::Invalid) + { + throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests), + SubOpts.help()); + } + return Mode; +} + +void +BuildsCommand::ParseZenProcessId(int& ZenProcessId) +{ + if (ZenProcessId == -1) + { + const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath(); + ProcessHandle RunningProcess; + std::error_code Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false); + if (Ec) + { + throw std::runtime_error(fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message())); } - else + if (!RunningProcess.IsValid()) { - if (!m_BuildMetadataPath.empty()) - { - throw OptionParseException("'--metadata-path' requires '--create-build'", SubOption->help()); - } - if (!m_BuildMetadata.empty()) - { - throw OptionParseException("'--metadata' requires '--create-build'", SubOption->help()); - } + throw std::runtime_error(fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath)); } - return {}; - }; + ZenProcessId = RunningProcess.Pid(); + } +} - UseSparseFiles = m_UseSparseFiles; +////////////////////////////////////////////////////////////////////////// + +// --------------------------------------------------------------------------- +// Subcommand implementations +// --------------------------------------------------------------------------- + +BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) +: ZenSubCmdBase("list-namespaces", "List all namespaces and optionally their buckets") +, m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddZenFolderOptions(Opts); + Opts.add_option("", "", "recursive", "Enable fetch of buckets within namespaces also", cxxopts::value(m_Recursive), "<recursive>"); + Opts.add_option("", + "", + "result-path", + "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", + cxxopts::value(m_ResultPath), + "<result-path>"); + Opts.parse_positional({"result-path"}); + Opts.positional_help("result-path"); +} + +void +BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; - if (SubOption == &m_ListNamespacesOptions) + if (!m_ResultPath.empty()) { - if (!m_ListResultPath.empty()) + if (!IsQuiet) { - if (!IsQuiet) - { - LogExecutableVersionAndPid(); - } + ZenCmdBase::LogExecutableVersionAndPid(); } + } + + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), ZenFolderPath); }); - - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(ZenFolderPath), - /*RequriesNamespace*/ false, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool */ false); - - CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive); - ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); - if (m_ListResultPath.empty()) + CreateDirectories(m_Parent.GetZenFolderPath()); + auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); + + std::unique_ptr<AuthMgr> Auth; + std::string DummyBuildId; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + DummyBuildId, + /*RequireNamespace*/ false, + /*RequireBucket*/ false, + /*BoostCacheBackgroundWorkerPool*/ false, + Auth, + Opts); + + CbObject Response = Storage.BuildStorage->ListNamespaces(m_Recursive); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); + if (m_ResultPath.empty()) + { + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + ZEN_CONSOLE("{}", SB.ToView()); + } + else + { + std::filesystem::path ResultPath = MakeSafeAbsolutePath(m_ResultPath); + if (ToLower(ResultPath.extension().string()) == ".cbo") { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - ZEN_CONSOLE("{}", SB.ToView()); + MemoryView ResponseView = Response.GetView(); + WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } else { - std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath); - if (ToLower(ListResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - return; } +} - if (SubOption == &m_ListOptions) - { - MakeSafeAbsolutePathInPlace(m_ListQueryPath); - MakeSafeAbsolutePathInPlace(m_ListResultPath); +BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list", "List builds matching a query"), m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddZenFolderOptions(Opts); + Opts.add_option("", + "", + "query-path", + "Path to json or compactbinary file containing list query", + cxxopts::value(m_QueryPath), + "<query-path>"); + Opts.add_option("", + "", + "result-path", + "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", + cxxopts::value(m_ResultPath), + "<result-path>"); + Opts.parse_positional({"query-path", "result-path"}); + Opts.positional_help("query-path result-path"); +} + +void +BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; + + MakeSafeAbsolutePathInPlace(m_QueryPath); + MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ListResultPath.empty()) + if (!m_ResultPath.empty()) + { + if (!IsQuiet) { - if (!IsQuiet) - { - LogExecutableVersionAndPid(); - } + ZenCmdBase::LogExecutableVersionAndPid(); } - std::string JsonQuery; - if (m_ListQueryPath.empty()) + } + std::string JsonQuery; + if (m_QueryPath.empty()) + { + CbObjectWriter QueryWriter; + QueryWriter.BeginObject("query"); + QueryWriter.EndObject(); // query + CbObject QueryObject = QueryWriter.Save(); + ExtendableStringBuilder<64> SB; + CompactBinaryToJson(QueryObject, SB); + JsonQuery = SB.ToString(); + } + else + { + if (ToLower(m_QueryPath.extension().string()) == ".cbo") { - CbObjectWriter QueryWriter; - QueryWriter.BeginObject("query"); - QueryWriter.EndObject(); // query - CbObject QueryObject = QueryWriter.Save(); + CbObject QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_QueryPath)); ExtendableStringBuilder<64> SB; CompactBinaryToJson(QueryObject, SB); JsonQuery = SB.ToString(); } else { - if (ToLower(m_ListQueryPath.extension().string()) == ".cbo") + IoBuffer MetaDataJson = ReadFile(m_QueryPath).Flatten(); + std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + CbObject QueryObject = LoadCompactBinaryFromJson(Json, JsonError) + .AsObject(); // We try to convert it so it is at least reaonably verified in format + if (!JsonError.empty()) { - CbObject QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath)); - ExtendableStringBuilder<64> SB; - CompactBinaryToJson(QueryObject, SB); - JsonQuery = SB.ToString(); - } - else - { - IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten(); - std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); - std::string JsonError; - CbObject QueryObject = LoadCompactBinaryFromJson(Json, JsonError) - .AsObject(); // We try to convert it so it is at least reaonably verified in format - if (!JsonError.empty()) - { - throw std::runtime_error( - fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError)); - } - JsonQuery = std::string(Json); + throw std::runtime_error( + fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_QueryPath.string(), JsonError)); } + JsonQuery = std::string(Json); } + } - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; - } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); }); + m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool */ false); + CreateDirectories(m_Parent.GetZenFolderPath()); + auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery); - ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); - if (m_ListResultPath.empty()) + std::unique_ptr<AuthMgr> Auth; + std::string DummyBuildId; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + DummyBuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ false, + /*BoostCacheBackgroundWorkerPool*/ false, + Auth, + Opts); + + CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); + if (m_ResultPath.empty()) + { + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + ZEN_CONSOLE("{}", SB.ToView()); + } + else + { + if (ToLower(m_ResultPath.extension().string()) == ".cbo") { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - ZEN_CONSOLE("{}", SB.ToView()); + MemoryView ResponseView = Response.GetView(); + WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } else { - if (ToLower(m_ListResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - return; } +} - if (SubOption == &m_ListBlocksOptions) - { - MakeSafeAbsolutePathInPlace(m_ListResultPath); - - if (!m_ListResultPath.empty()) - { - if (!IsQuiet) - { - LogExecutableVersionAndPid(); - } - } +BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) +: ZenSubCmdBase("list-blocks", "List blocks for a build") +, m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddZenFolderOptions(Opts); + Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + Opts.add_option("", + "", + "result-path", + "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", + cxxopts::value(m_ResultPath), + "<result-path>"); + Opts.add_option("", "", "max-count", "Maximum number of blocks to list", cxxopts::value(m_MaxCount), "<maxcount>"); + Opts.parse_positional({"build-id"}); + Opts.positional_help("build-id"); +} - if (m_ListBlocksMaxCount == 0) - { - throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_ListBlocksMaxCount), SubOption->help()); - } +void +BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + MakeSafeAbsolutePathInPlace(m_ResultPath); - if (m_ZenFolderPath.empty()) + if (!m_ResultPath.empty()) + { + if (!IsQuiet) { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + ZenCmdBase::LogExecutableVersionAndPid(); } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + } - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); }); + if (m_MaxCount == 0) + { + throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_MaxCount), Opts.help()); + } - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool */ false); + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - const Oid BuildId = ParseBuildId(); + m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_ListBlocksMaxCount); - ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); + CreateDirectories(m_Parent.GetZenFolderPath()); + auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response); - if (!IsQuiet) + std::unique_ptr<AuthMgr> Auth; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + m_BuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ true, + /*BoostCacheBackgroundWorkerPool*/ false, + Auth, + Opts); + + const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + + CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_MaxCount); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); + + std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response); + if (!IsQuiet) + { + ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size()); + } + if (m_ResultPath.empty()) + { + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + ForEachStrTok(SB.ToView(), '\n', [](std::string_view Row) { + ZEN_CONSOLE("{}", Row); + return true; + }); + } + else + { + if (ToLower(m_ResultPath.extension().string()) == ".cbo") { - ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size()); + MemoryView ResponseView = Response.GetView(); + WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } - if (m_ListResultPath.empty()) + else { ExtendableStringBuilder<1024> SB; CompactBinaryToJson(Response.GetView(), SB); - ForEachStrTok(SB.ToView(), '\n', [](std::string_view Row) { - ZEN_CONSOLE("{}", Row); - return true; - }); - } - else - { - if (ToLower(m_ListResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - return; } - if (SubOption == &m_UploadOptions) - { - if (!IsQuiet) - { - LogExecutableVersionAndPid(); - } +} - TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } +BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) +: ZenSubCmdBase("upload", "Upload a folder to build storage") +, m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddCacheOptions(Opts); + Parent.AddWorkerOptions(Opts); + Parent.AddZenFolderOptions(Opts); + Parent.AddExcludeFolderOption(Opts); + Parent.AddExcludeExtensionsOption(Opts); + Parent.AddChunkingCacheOptions(Opts); + Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); + Opts.add_option("", + "", + "create-build", + "Set to true to create the containing build, if unset a builds-id must be given and the build already exist", + cxxopts::value(m_CreateBuild), + "<id>"); + Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + Opts.add_option("", + "", + "build-part-id", + "Build part Id, if not given it will be auto generated", + cxxopts::value(m_BuildPartId), + "<id>"); + Opts.add_option("", + "", + "build-part-name", + "Name of the build part, if not given it will be be named after the directory name at end of local-path", + cxxopts::value(m_BuildPartName), + "<name>"); + Opts.add_option("", + "", + "metadata-path", + "Path to json file that holds the metadata for the build. Requires the create-build option to be set", + cxxopts::value(m_BuildMetadataPath), + "<metadata-path>"); + Opts.add_option( + "", + "", + "metadata", + "Key-value pairs separated by ';' with build meta data. (key1=value1;key2=value2). Requires the create-build option to be set", + cxxopts::value(m_BuildMetadata), + "<metadata>"); + Opts.add_option("", "", "clean", "Ignore existing blocks", cxxopts::value(m_Clean), "<clean>"); + Opts.add_option("", + "", + "block-min-reuse", + "Percent of an existing block that must be relevant for it to be resused. Defaults to 85.", + cxxopts::value(m_BlockReuseMinPercentLimit), + "<minreuse>"); + Opts.add_option("cache", + "", + "zen-cache-upload", + "Upload data downloaded from remote host to zen cache", + cxxopts::value(m_UploadToZenCache), + "<uploadtozencache>"); + + Parent.AddMultipartOptions(Opts); + + Opts.add_option("", + "", + "manifest-path", + "Path to a text file with one line of <local path>[TAB]<modification date> per file to include or a " + "structured .json file describing the parts", + cxxopts::value(m_ManifestPath), + "<manifestpath>"); + + Opts.add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), "<verify>"); + Opts.add_option("", + "", + "find-max-block-count", + "The maximum number of blocks we search for in the build context", + cxxopts::value(m_FindBlockMaxCount), + "<maxblockcount>"); + + Opts.parse_positional({"local-path", "build-id"}); + Opts.positional_help("local-path build-id"); +} - ZenState InstanceState; +void +BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; - ParsePath(); + if (!IsQuiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); + } - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); + if (!IsQuiet) + { + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; - } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); - MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); + ZenState InstanceState; - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); + m_Parent.ParsePath(m_Path, Opts); - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool */ false); + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - if (m_BuildPartName.empty() && m_ManifestPath.empty()) - { - m_BuildPartName = m_Path.filename().string(); - } + m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(); - if (m_BuildId.empty()) - { - m_BuildId = BuildId.ToString(); - } + CreateDirectories(m_Parent.GetZenFolderPath()); + auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - Oid BuildPartId; - if (!m_BuildPartId.empty()) - { - BuildPartId = ParseBuildPartId(); - } + std::unique_ptr<AuthMgr> Auth; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + m_BuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ true, + /*BoostCacheBackgroundWorkerPool*/ false, + Auth, + Opts); + + if (m_BuildPartName.empty() && m_ManifestPath.empty()) + { + m_BuildPartName = m_Path.filename().string(); + } - CbObject MetaData = ParseBuildMetadata(); + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : m_Parent.ParseBuildId(m_BuildId, Opts); + if (m_BuildId.empty()) + { + m_BuildId = BuildId.ToString(); + } - const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath); + Oid BuildPartId; + if (!m_BuildPartId.empty()) + { + BuildPartId = m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + } - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; - std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + CbObject MetaData = m_Parent.ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); - std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_ChunkingCachePath.empty() - ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u); + const std::filesystem::path TempDir = ZenTempFolderPath(m_Parent.GetZenFolderPath()); - std::vector<std::pair<Oid, std::string>> UploadedParts = - UploadFolder(*Output, - Workers, - Storage, - BuildId, - BuildPartId, - m_BuildPartName, - m_Path, - m_ManifestPath, - MetaData, - *ChunkController, - *ChunkCache, - UploadFolderOptions{.TempDir = TempDir, - .FindBlockMaxCount = m_FindBlockMaxCount, - .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_AllowMultiparts, - .CreateBuild = m_CreateBuild, - .IgnoreExistingBlocks = m_Clean, - .UploadToZenCache = m_UploadToZenCache, - .ExcludeFolders = ExcludeFolders, - .ExcludeExtensions = ExcludeExtensions}); - - if (!AbortFlag) - { - if (m_PostUploadVerify) - { - // TODO: Validate all parts - for (const auto& Part : UploadedParts) - { - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second); - } - } - } + std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; + std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; + m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); - if (true) + std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); + std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + ? CreateNullChunkingCache() + : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + + std::vector<std::pair<Oid, std::string>> UploadedParts = + UploadFolder(*Output, + Workers, + Storage, + BuildId, + BuildPartId, + m_BuildPartName, + m_Path, + m_ManifestPath, + MetaData, + *ChunkController, + *ChunkCache, + UploadFolderOptions{.TempDir = TempDir, + .FindBlockMaxCount = m_FindBlockMaxCount, + .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .CreateBuild = m_CreateBuild, + .IgnoreExistingBlocks = m_Clean, + .UploadToZenCache = m_UploadToZenCache, + .ExcludeFolders = ExcludeFolders, + .ExcludeExtensions = ExcludeExtensions}); + + if (!AbortFlag) + { + if (m_PostUploadVerify) { - if (!IsQuiet) + for (const auto& Part : UploadedParts) { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.BuildStorageHost.Name, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second); } } - if (AbortFlag) - { - throw std::runtime_error("Upload aborted"); - } } - auto ParseAllowPartialBlockRequests = [&]() -> EPartialBlockRequestMode { - if (m_PrimeCacheOnly) - { - return EPartialBlockRequestMode::Off; - } - EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests); - if (Mode == EPartialBlockRequestMode::Invalid) - { - throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests), - SubOption->help()); - } - return Mode; - }; - - if (SubOption == &m_DownloadOptions) + if (true) { if (!IsQuiet) { - LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + Storage.BuildStorageHost.Name, + NiceBytes(StorageStats.TotalBytesRead.load()), + NiceBytes(StorageStats.TotalBytesWritten.load()), + StorageStats.TotalRequestCount.load(), + StorageStats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0, + StorageStats.TotalRequestCount.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0); } + } + if (AbortFlag) + { + throw std::runtime_error("Upload aborted"); + } +} - ZenState InstanceState; +BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) +: ZenSubCmdBase("download", "Download a build to a local folder") +, m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddCacheOptions(Opts); + Parent.AddZenFolderOptions(Opts); + Parent.AddWorkerOptions(Opts); + Parent.AddWildcardOptions(Opts); + Parent.AddAppendNewContentOptions(Opts); + Parent.AddExcludeFolderOption(Opts); + + Opts.add_option("cache", + "", + "cache-prime-only", + "Only download blobs missing in cache and upload to cache", + cxxopts::value(m_PrimeCacheOnly), + "<cacheprimeonly>"); + + Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); + Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + Opts.add_option("", + "", + "build-part-id", + "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", + cxxopts::value(m_BuildPartIds), + "<id>"); + Opts.add_option("", + "", + "build-part-name", + "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " + "all parts will be downloaded", + cxxopts::value(m_BuildPartNames), + "<name>"); + Opts.add_option("", + "", + "clean", + "Delete all data in target folder that is not part of the downloaded content", + cxxopts::value(m_Clean), + "<clean>"); + Opts.add_option("", + "", + "force", + "Force download of all content by ignoring any existing local content", + cxxopts::value(m_Force), + "<force>"); + Opts.add_option("cache", + "", + "zen-cache-upload", + "Upload data downloaded from remote host to zen cache", + cxxopts::value(m_UploadToZenCache), + "<uploadtozencache>"); + Parent.AddMultipartOptions(Opts); + + Parent.AddPartialBlockRequestOptions(Opts); + + Opts.add_option( + "", + "", + "download-spec-path", + "Path to a text file with one line of <local path> per file to include or a structured .json file describing what to download.", + cxxopts::value(m_DownloadSpecPath), + "<downloadspecpath>"); - ParsePath(); + Opts.add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), "<verify>"); + Opts.add_option("", + "", + "enable-scavenge", + "Enable scavenging of data from previouse download locations", + cxxopts::value(m_EnableScavenging), + "<scavenge>"); + Opts.add_option("", + "", + "allow-file-clone", + "Enable use of block reference counting when copying files", + cxxopts::value(m_AllowFileClone), + "<allowclone>"); + Opts.parse_positional({"local-path", "build-id", "build-part-name"}); + Opts.positional_help("local-path build-id build-part-name"); +} - std::vector<std::string> IncludeWildcards; - std::vector<std::string> ExcludeWildcards; - ParseFileFilters(IncludeWildcards, ExcludeWildcards); +void +BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = m_Path / ZenFolderName; - } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + if (!IsQuiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); + } - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); + if (!IsQuiet) + { + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ m_PrimeCacheOnly); + ZenState InstanceState; - const Oid BuildId = ParseBuildId(); + m_Parent.ParsePath(m_Path, Opts); - if (m_PostDownloadVerify && m_PrimeCacheOnly) - { - throw OptionParseException("'--cache-prime-only' conflicts with '--verify'", SubOption->help()); - } + std::vector<std::string> IncludeWildcards; + std::vector<std::string> ExcludeWildcards; + m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); - if (m_Clean && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled"); - } + m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - if (m_Force && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled"); - } + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - if (m_AllowPartialBlockRequests != "false" && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled"); - } + std::unique_ptr<AuthMgr> Auth; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + m_BuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ true, + /*BoostCacheBackgroundWorkerPool*/ m_PrimeCacheOnly, + Auth, + Opts); + + const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + + if (m_PostDownloadVerify && m_PrimeCacheOnly) + { + throw OptionParseException("'--cache-prime-only' conflicts with '--verify'", Opts.help()); + } - std::vector<Oid> BuildPartIds = ParseBuildPartIds(); - std::vector<std::string> BuildPartNames = ParseBuildPartNames(); + if (m_Clean && m_PrimeCacheOnly) + { + ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled"); + } - EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(); + if (m_Force && m_PrimeCacheOnly) + { + ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled"); + } - if (m_AppendNewContent && m_Clean) - { - throw OptionParseException("'--append' conflicts with '--clean'", SubOption->help()); - } + if (m_Parent.m_AllowPartialBlockRequests != "false" && m_PrimeCacheOnly) + { + ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled"); + } - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; - std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); - DownloadFolder(*Output, - Workers, - Storage, - StorageCacheStats, - BuildId, - BuildPartIds, - BuildPartNames, - m_DownloadSpecPath, - m_Path, - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = m_ZenFolderPath, - .AllowMultiparts = m_AllowMultiparts, - .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = m_Clean, - .PostDownloadVerify = m_PostDownloadVerify, - .PrimeCacheOnly = m_PrimeCacheOnly, - .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force, - .EnableTargetFolderScavenging = !m_Force, - .AllowFileClone = m_AllowFileClone, - .IncludeWildcards = IncludeWildcards, - .ExcludeWildcards = ExcludeWildcards, - .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory), - .PopulateCache = m_UploadToZenCache, - .AppendNewContent = m_AppendNewContent, - .ExcludeFolders = ExcludeFolders}); + EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(m_PrimeCacheOnly, Opts); - if (AbortFlag) - { - throw std::runtime_error("Download aborted"); - } + if (m_Parent.m_AppendNewContent && m_Clean) + { + throw OptionParseException("'--append' conflicts with '--clean'", Opts.help()); } - if (SubOption == &m_LsOptions) + std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; + std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; + m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + + DownloadFolder( + *Output, + Workers, + Storage, + StorageCacheStats, + BuildId, + BuildPartIds, + BuildPartNames, + m_DownloadSpecPath, + m_Path, + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = m_Parent.GetZenFolderPath(), + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .PartialBlockRequestMode = PartialBlockRequestMode, + .CleanTargetFolder = m_Clean, + .PostDownloadVerify = m_PostDownloadVerify, + .PrimeCacheOnly = m_PrimeCacheOnly, + .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force, + .EnableTargetFolderScavenging = !m_Force, + .AllowFileClone = m_AllowFileClone, + .IncludeWildcards = IncludeWildcards, + .ExcludeWildcards = ExcludeWildcards, + .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Parent.m_BoostWorkerMemory), + .PopulateCache = m_UploadToZenCache, + .AppendNewContent = m_Parent.m_AppendNewContent, + .ExcludeFolders = ExcludeFolders}); + + if (AbortFlag) { - if (!m_LsResultPath.empty()) - { - if (!IsQuiet) - { - LogExecutableVersionAndPid(); - } - } + throw std::runtime_error("Download aborted"); + } +} - ZenState InstanceState; +BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "List files in a build"), m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddCacheOptions(Opts); + Parent.AddZenFolderOptions(Opts); + Parent.AddWorkerOptions(Opts); + Parent.AddWildcardOptions(Opts); + + Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + Opts.add_option("", + "", + "build-part-id", + "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", + cxxopts::value(m_BuildPartIds), + "<id>"); + Opts.add_option("", + "", + "build-part-name", + "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " + "all parts will be downloaded", + cxxopts::value(m_BuildPartNames), + "<name>"); + + Opts.add_option("", + "", + "result-path", + "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", + cxxopts::value(m_ResultPath), + "<result-path>"); + + Opts.add_option("", + "o", + "output-path", + "Path to output, extension .json or .cb (compac binary). Default is output to console", + cxxopts::value(m_ResultPath), + "<output-path>"); + + Opts.parse_positional({"build-id", "wildcard"}); + Opts.positional_help("build-id wildcard"); +} - std::vector<std::string> IncludeWildcards; - std::vector<std::string> ExcludeWildcards; - ParseFileFilters(IncludeWildcards, ExcludeWildcards); +void +BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; - if (m_ZenFolderPath.empty()) + if (!m_ResultPath.empty()) + { + if (!IsQuiet) { - m_ZenFolderPath = m_Path / ZenFolderName; + ZenCmdBase::LogExecutableVersionAndPid(); } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + } - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + ZenState InstanceState; - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool */ false); + std::vector<std::string> IncludeWildcards; + std::vector<std::string> ExcludeWildcards; + m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); - const Oid BuildId = ParseBuildId(); + m_Parent.ResolveZenFolderPath(m_Parent.m_StoragePath); // ls uses storage path context - std::vector<Oid> BuildPartIds = ParseBuildPartIds(); - std::vector<std::string> BuildPartNames = ParseBuildPartNames(); + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - std::unique_ptr<CbObjectWriter> StructuredOutput; - if (!m_LsResultPath.empty()) - { - MakeSafeAbsolutePathInPlace(m_LsResultPath); - StructuredOutput = std::make_unique<CbObjectWriter>(); - } + std::unique_ptr<AuthMgr> Auth; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + m_BuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ true, + /*BoostCacheBackgroundWorkerPool*/ false, + Auth, + Opts); + + const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + + std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + + std::unique_ptr<CbObjectWriter> StructuredOutput; + if (!m_ResultPath.empty()) + { + MakeSafeAbsolutePathInPlace(m_ResultPath); + StructuredOutput = std::make_unique<CbObjectWriter>(); + } - ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); + ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); - if (StructuredOutput) + if (StructuredOutput) + { + CbObject Response = StructuredOutput->Save(); + if (ToLower(m_ResultPath.extension().string()) == ".cbo") { - CbObject Response = StructuredOutput->Save(); - if (ToLower(m_LsResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + MemoryView ResponseView = Response.GetView(); + WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } - - if (AbortFlag) + else { - throw std::runtime_error("List build aborted"); + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } } - if (SubOption == &m_DiffOptions) + if (AbortFlag) { - if (!IsQuiet) - { - LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } - - ParsePath(); - ParseDiffPath(); - - MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); + throw std::runtime_error("List build aborted"); + } +} - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; - std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); +BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff", "Diff two local folders"), m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddOutputOptions(Opts); + Parent.AddWorkerOptions(Opts); + Parent.AddExcludeFolderOption(Opts); + Parent.AddExcludeExtensionsOption(Opts); + Parent.AddChunkingCacheOptions(Opts); + Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); + Opts.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>"); + Opts.add_option("", + "", + "only-chunked", + "Skip files from diff summation that are not processed with chunking", + cxxopts::value(m_OnlyChunked), + "<only-chunked>"); + Opts.parse_positional({"local-path", "compare-path"}); + Opts.positional_help("local-path compare-path"); +} - StandardChunkingControllerSettings ChunkingSettings; - std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings); - std::unique_ptr<ChunkingCache> ChunkCache = m_ChunkingCachePath.empty() - ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u); +void +BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; - if (m_OnlyChunked) - { - ExcludeExtensions.insert(ExcludeExtensions.end(), - ChunkingSettings.SplitOnlyExtensions.begin(), - ChunkingSettings.SplitOnlyExtensions.end()); - ExcludeExtensions.insert(ExcludeExtensions.end(), - ChunkingSettings.SplitAndCompressExtensions.begin(), - ChunkingSettings.SplitAndCompressExtensions.end()); - } + if (!IsQuiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); + } - DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions); - if (AbortFlag) - { - throw std::runtime_error("Diff folders aborted"); - } + TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); + if (!IsQuiet) + { + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); } - if (SubOption == &m_PrimeCacheOptions) + m_Parent.ParsePath(m_Path, Opts); + if (m_DiffPath.empty()) { - if (!IsQuiet) - { - LogExecutableVersionAndPid(); - } + throw OptionParseException("'--compare-path' is required", Opts.help()); + } + MakeSafeAbsolutePathInPlace(m_DiffPath); - TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; + std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; + m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; - } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + StandardChunkingControllerSettings ChunkingSettings; + std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings); + std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + ? CreateNullChunkingCache() + : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); + if (m_OnlyChunked) + { + ExcludeExtensions.insert(ExcludeExtensions.end(), + ChunkingSettings.SplitOnlyExtensions.begin(), + ChunkingSettings.SplitOnlyExtensions.end()); + ExcludeExtensions.insert(ExcludeExtensions.end(), + ChunkingSettings.SplitAndCompressExtensions.begin(), + ChunkingSettings.SplitAndCompressExtensions.end()); + } - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool */ true); + DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions); + if (AbortFlag) + { + throw std::runtime_error("Diff folders aborted"); + } +} - const Oid BuildId = ParseBuildId(); +BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) +: ZenSubCmdBase("fetch-blob", "Fetch and validate a specific blob") +, m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddCacheOptions(Opts); + Parent.AddZenFolderOptions(Opts); + Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + Opts.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>"); + Opts.parse_positional({"build-id", "blob-hash"}); + Opts.positional_help("build-id blob-hash"); +} - std::vector<Oid> BuildPartIds = ParseBuildPartIds(); - std::vector<std::string> BuildPartNames = ParseBuildPartNames(); +void +BuildsFetchBlobSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + if (!IsQuiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); + } - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); + TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); + if (!IsQuiet) + { + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - std::vector<Oid> AllBuildPartIds; - AllBuildPartIds.reserve(AllBuildParts.size()); - for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts) - { - AllBuildPartIds.push_back(BuildPart.first); - } + m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache"); - - BuildsOperationPrimeCache PrimeOp(*Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetNetworkPool(), - BuildId, - AllBuildPartIds, - BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .ZenFolderPath = m_ZenFolderPath, - .LargeAttachmentSize = PreferredMultipartChunkSize * 4u, - .PreferredMultipartChunkSize = PreferredMultipartChunkSize, - .ForceUpload = m_Force}, - StorageCacheStats); - PrimeOp.Execute(); + CreateDirectories(m_Parent.GetZenFolderPath()); + auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - if (!IsQuiet) - { - if (Storage.CacheStorage) - { - ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", - StorageCacheStats.PutBlobCount.load(), - NiceBytes(StorageCacheStats.PutBlobByteCount), - Storage.CacheHost.Name); - } - } + std::unique_ptr<AuthMgr> Auth; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + m_BuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ true, + /*BoostCacheBackgroundWorkerPool*/ false, + Auth, + Opts); + + IoHash BlobHash = m_Parent.ParseBlobHash(m_BlobHash, Opts); + + const Oid BuildId = Oid::FromHexString(m_BuildId); + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); + if (AbortFlag) + { + throw std::runtime_error("Fetch blob aborted"); + } + if (!IsQuiet) + { + ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize); + } +} - return; +BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) +: ZenSubCmdBase("prime-cache", "Prime the zen cache with build data") +, m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddCacheOptions(Opts); + Parent.AddWorkerOptions(Opts); + Parent.AddZenFolderOptions(Opts); + Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + Opts.add_option("", + "", + "build-part-id", + "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", + cxxopts::value(m_BuildPartIds), + "<id>"); + Opts.add_option("", + "", + "build-part-name", + "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " + "all parts will be downloaded", + cxxopts::value(m_BuildPartNames), + "<name>"); + Opts.add_option("", + "", + "force", + "Force download of all blobs by ignoring any existing blobs in cache", + cxxopts::value(m_Force), + "<force>"); + Opts.parse_positional({"build-id"}); + Opts.positional_help("build-id"); +} + +void +BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; + + if (!IsQuiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); } - if (SubOption == &m_FetchBlobOptions) + TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); + if (!IsQuiet) { - if (!IsQuiet) - { - LogExecutableVersionAndPid(); - } + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } - TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; - } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + CreateDirectories(m_Parent.GetZenFolderPath()); + auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); + std::unique_ptr<AuthMgr> Auth; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + m_BuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ true, + /*BoostCacheBackgroundWorkerPool*/ true, + Auth, + Opts); - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool */ false); + const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); - IoHash BlobHash = ParseBlobHash(); + std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); - const Oid BuildId = Oid::FromHexString(m_BuildId); + std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - uint64_t CompressedSize; - uint64_t DecompressedSize; - ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); - if (AbortFlag) - { - throw std::runtime_error("Fetch blob aborted"); - } - if (!IsQuiet) - { - ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", - BlobHash, - CompressedSize, - DecompressedSize); - } - return; - } + CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - if (SubOption == &m_ValidateBuildPartOptions) - { - if (!IsQuiet) - { - LogExecutableVersionAndPid(); - } + std::vector<std::pair<Oid, std::string>> AllBuildParts = + ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + std::vector<Oid> AllBuildPartIds; + AllBuildPartIds.reserve(AllBuildParts.size()); + for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts) + { + AllBuildPartIds.push_back(BuildPart.first); + } - ZenState InstanceState; + ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache"); - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - if (m_ZenFolderPath.empty()) + BuildsOperationPrimeCache PrimeOp(*Output, + Storage, + AbortFlag, + PauseFlag, + Workers.GetNetworkPool(), + BuildId, + AllBuildPartIds, + BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet, + .IsVerbose = IsVerbose, + .ZenFolderPath = m_Parent.GetZenFolderPath(), + .LargeAttachmentSize = PreferredMultipartChunkSize * 4u, + .PreferredMultipartChunkSize = PreferredMultipartChunkSize, + .ForceUpload = m_Force}, + StorageCacheStats); + PrimeOp.Execute(); + + if (!IsQuiet) + { + if (Storage.CacheStorage) { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", + StorageCacheStats.PutBlobCount.load(), + NiceBytes(StorageCacheStats.PutBlobByteCount), + Storage.CacheHost.Name); } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + } +} - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); +BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("pause", "Pause a running zen process"), m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); + Opts.parse_positional({"process-id"}); + Opts.positional_help("process-id"); +} - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool */ false); +void +BuildsPauseSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + using namespace builds_impl; + m_Parent.ParseZenProcessId(m_ZenProcessId); + ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(true); +} - Oid BuildId = ParseBuildId(); +BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("resume", "Resume a paused zen process"), m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); + Opts.parse_positional({"process-id"}); + Opts.positional_help("process-id"); +} - if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) - { - throw OptionParseException( - fmt::format("'--build-part-id' ('{}') conflicts with '--build-part-name' ('{}')", m_BuildPartId, m_BuildPartName), - SubOption->help()); - } +void +BuildsResumeSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + using namespace builds_impl; + m_Parent.ParseZenProcessId(m_ZenProcessId); + ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(false); +} - const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(); +BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("abort", "Abort a running zen process"), m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); + Opts.parse_positional({"process-id"}); + Opts.positional_help("process-id"); +} - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); +void +BuildsAbortSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + using namespace builds_impl; + m_Parent.ParseZenProcessId(m_ZenProcessId); + ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Abort.store(true); +} - if (AbortFlag) - { - throw std::runtime_error("Validate build part failed"); - } +BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) +: ZenSubCmdBase("validate-part", "Validate a build part") +, m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddWorkerOptions(Opts); + Parent.AddZenFolderOptions(Opts); + Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); + Opts.add_option("", + "", + "build-part-id", + "Build part Id, if not given it will be auto generated", + cxxopts::value(m_BuildPartId), + "<id>"); + Opts.add_option("", + "", + "build-part-name", + "Name of the build part, if not given it will be be named after the directory name at end of local-path", + cxxopts::value(m_BuildPartName), + "<name>"); + Opts.parse_positional({"build-id", "build-part-id"}); + Opts.positional_help("build-id build-part-id"); +} + +void +BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; + + if (!IsQuiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); } - if (SubOption == &m_MultiTestDownloadOptions) + TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); + if (!IsQuiet) { - TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } - m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_SystemRootDir); - CleanDirectory(m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); + ZenState InstanceState; - ParsePath(); + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = m_Path / ZenFolderName; - } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(); + CreateDirectories(m_Parent.GetZenFolderPath()); + auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + std::unique_ptr<AuthMgr> Auth; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + m_BuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ true, + /*BoostCacheBackgroundWorkerPool*/ false, + Auth, + Opts); + + Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + + if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) + { + throw OptionParseException( + fmt::format("'--build-part-id' ('{}') conflicts with '--build-part-name' ('{}')", m_BuildPartId, m_BuildPartName), + Opts.help()); + } - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool */ false); + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : m_Parent.ParseBuildPartId(m_BuildPartId, Opts); - Stopwatch Timer; - for (const std::string& BuildIdString : m_BuildIds) - { - Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString)); - if (BuildId == Oid::Zero) - { - throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), SubOption->help()); - } - DownloadFolder(*Output, - Workers, - Storage, - StorageCacheStats, - BuildId, - /*BuildPartIds,*/ {}, - /*BuildPartNames*/ {}, - /*ManifestPath*/ {}, - m_Path, - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = m_ZenFolderPath, - .AllowMultiparts = m_AllowMultiparts, - .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = BuildIdString == m_BuildIds.front(), - .PostDownloadVerify = true, - .PrimeCacheOnly = false, - .EnableOtherDownloadsScavenging = m_EnableScavenging, - .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) - { - throw std::runtime_error("Multitest aborted"); - } - if (!IsQuiet) - { - ZEN_CONSOLE("\n"); - } - } - if (!IsQuiet) - { - ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - } - } + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - auto ParseZenProcessId = [&]() { - if (m_ZenProcessId == -1) - { - const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath(); - ProcessHandle RunningProcess; - std::error_code Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false); - if (Ec) - { - throw std::runtime_error( - fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message())); - } - if (!RunningProcess.IsValid()) - { - throw std::runtime_error( - fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath)); - } - m_ZenProcessId = RunningProcess.Pid(); - } - }; + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); - if (SubOption == &m_PauseOptions) + if (AbortFlag) { - ParseZenProcessId(); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(true); + throw std::runtime_error("Validate build part failed"); } +} - if (SubOption == &m_ResumeOptions) - { - ParseZenProcessId(); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(false); - } +BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test", "Run an upload/download test cycle"), m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddCacheOptions(Opts); + Parent.AddWorkerOptions(Opts); + Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); + Parent.AddMultipartOptions(Opts); + Parent.AddPartialBlockRequestOptions(Opts); + Parent.AddWildcardOptions(Opts); + Parent.AddAppendNewContentOptions(Opts); + Parent.AddChunkingCacheOptions(Opts); + Opts.add_option("", + "", + "enable-scavenge", + "Enable scavenging of data from previouse download locations", + cxxopts::value(m_EnableScavenging), + "<scavenge>"); + Opts.add_option("", + "", + "allow-file-clone", + "Enable use of block reference counting when copying files", + cxxopts::value(m_AllowFileClone), + "<allowclone>"); + Opts.parse_positional({"local-path"}); + Opts.positional_help("local-path"); +} - if (SubOption == &m_AbortOptions) +void +BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; + + TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); + if (!IsQuiet) { - ParseZenProcessId(); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Abort.store(true); + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); } - if (SubOption == &m_TestOptions) - { - TransferThreadWorkers Workers(m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_Parent.m_SystemRootDir); + CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); - m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_SystemRootDir); - CleanDirectory(m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); + m_Parent.ParsePath(m_Path, Opts); - ParsePath(); + if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + { + m_Parent.m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.m_StoragePath); + CreateDirectories(m_Parent.m_StoragePath); + m_Parent.m_StoragePath = m_Parent.m_StoragePath.generic_string(); + } - if (m_OverrideHost.empty() && m_StoragePath.empty()) + auto StorageGuard = MakeGuard([this]() { + if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) { - m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_StoragePath); - CreateDirectories(m_StoragePath); - m_StoragePath = m_StoragePath.generic_string(); + DeleteDirectories(m_Parent.m_StoragePath); } + }); - auto __ = MakeGuard([&]() { - if (m_OverrideHost.empty() && m_StoragePath.empty()) - { - DeleteDirectories(m_StoragePath); - } - }); + EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts); - EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(); + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + m_BuildPartName = m_Path.filename().string(); - const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test"); - const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); - const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3"); + const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test"); + const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); + const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3"); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + + auto DownloadGuard = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + }); - auto ___ = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() { - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); - }); - - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = m_Path / ZenFolderName; - } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); - MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); + m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); + MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool */ false); + std::unique_ptr<AuthMgr> Auth; + std::string TestBuildId; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + TestBuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ true, + /*BoostCacheBackgroundWorkerPool*/ false, + Auth, + Opts); + + m_BuildId = Oid::NewOid().ToString(); + m_BuildPartId = Oid::NewOid().ToString(); + m_CreateBuild = true; + + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + + auto MakeMetaData = [](const Oid& BuildId) -> CbObject { + CbObjectWriter BuildMetaDataWriter; + { + const uint32_t CL = BuildId.OidBits[2]; + BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL)); + BuildMetaDataWriter.AddString("branch", "ZenTestBuild"); + BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild"); + BuildMetaDataWriter.AddString("platform", "Windows"); + BuildMetaDataWriter.AddString("project", "Test"); + BuildMetaDataWriter.AddInteger("changelist", CL); + BuildMetaDataWriter.AddString("buildType", "test-folder"); + } + return BuildMetaDataWriter.Save(); + }; + CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId)); + { + ExtendableStringBuilder<256> SB; + CompactBinaryToJson(MetaData, SB); + ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); + } - m_BuildId = Oid::NewOid().ToString(); - m_BuildPartName = m_Path.filename().string(); - m_BuildPartId = Oid::NewOid().ToString(); - m_CreateBuild = true; + const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); + std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + ? CreateNullChunkingCache() + : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); - auto MakeMetaData = [](const Oid& BuildId) -> CbObject { - CbObjectWriter BuildMetaDataWriter; - { - const uint32_t CL = BuildId.OidBits[2]; - BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL)); - BuildMetaDataWriter.AddString("branch", "ZenTestBuild"); - BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild"); - BuildMetaDataWriter.AddString("platform", "Windows"); - BuildMetaDataWriter.AddString("project", "Test"); - BuildMetaDataWriter.AddInteger("changelist", CL); - BuildMetaDataWriter.AddString("buildType", "test-folder"); - } - return BuildMetaDataWriter.Save(); - }; - CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId)); - { - ExtendableStringBuilder<256> SB; - CompactBinaryToJson(MetaData, SB); - ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); - } + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); + UploadFolder(*Output, + Workers, + Storage, + BuildId, + BuildPartId, + m_BuildPartName, + m_Path, + {}, + MetaData, + *ChunkController, + *ChunkCache, + UploadFolderOptions{.TempDir = UploadTempDir, + .FindBlockMaxCount = m_FindBlockMaxCount, + .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .CreateBuild = true, + .IgnoreExistingBlocks = false, + .UploadToZenCache = m_UploadToZenCache}); + + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Upload build)"); + } - std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_ChunkingCachePath.empty() - ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u); + { + ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path); UploadFolder(*Output, Workers, Storage, - BuildId, - BuildPartId, + Oid::NewOid(), + Oid::NewOid(), m_BuildPartName, m_Path, {}, @@ -4170,145 +4245,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_AllowMultiparts, + .AllowMultiparts = m_Parent.m_AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, .UploadToZenCache = m_UploadToZenCache}); if (AbortFlag) { - throw std::runtime_error("Test aborted. (Upload build)"); + throw std::runtime_error("Test aborted. (Upload again, chunking is cached)"); } + } - { - ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path); - - UploadFolder(*Output, - Workers, - Storage, - Oid::NewOid(), - Oid::NewOid(), - m_BuildPartName, - m_Path, - {}, - MetaData, - *ChunkController, - *ChunkCache, - UploadFolderOptions{.TempDir = UploadTempDir, - .FindBlockMaxCount = m_FindBlockMaxCount, - .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_AllowMultiparts, - .CreateBuild = true, - .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); - if (AbortFlag) - { - throw std::runtime_error("Test aborted. (Upload again, chunking is cached)"); - } - } + if (!m_Parent.m_IncludeWildcard.empty() || !m_Parent.m_ExcludeWildcard.empty()) + { + auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); }); - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - if (!m_IncludeWildcard.empty() || !m_ExcludeWildcard.empty()) - { - auto __ = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); }); - - ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - - std::vector<std::string> IncludeWildcards; - std::vector<std::string> ExcludeWildcards; - ParseFileFilters(IncludeWildcards, ExcludeWildcards); - - DownloadFolder(*Output, - Workers, - Storage, - StorageCacheStats, - BuildId, - {BuildPartId}, - /*BuildPartNames*/ {}, - /*ManifestPath*/ {}, - DownloadPath, - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, - .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = true, - .PostDownloadVerify = true, - .PrimeCacheOnly = false, - .EnableOtherDownloadsScavenging = m_EnableScavenging, - .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone, - .IncludeWildcards = IncludeWildcards, - .ExcludeWildcards = ExcludeWildcards, - .AppendNewContent = false}); - if (AbortFlag) - { - throw std::runtime_error("Test aborted. (Download build)"); - } - - ZEN_CONSOLE("\nDownload Filtered Out Remaining of Build {}, Part {} ({}) to '{}'", - BuildId, - BuildPartId, - m_BuildPartName, - DownloadPath); - DownloadFolder(*Output, - Workers, - Storage, - StorageCacheStats, - BuildId, - {BuildPartId}, - /*BuildPartNames*/ {}, - /*ManifestPath*/ {}, - DownloadPath, - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, - .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = true, - .PostDownloadVerify = true, - .PrimeCacheOnly = false, - .EnableOtherDownloadsScavenging = m_EnableScavenging, - .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone, - .IncludeWildcards = ExcludeWildcards, - .ExcludeWildcards = IncludeWildcards, - .AppendNewContent = true}); - if (AbortFlag) - { - throw std::runtime_error("Test aborted. (Download build)"); - } - - ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, - Workers, - Storage, - StorageCacheStats, - BuildId, - {BuildPartId}, - /*BuildPartNames*/ {}, - /*ManifestPath*/ {}, - DownloadPath, - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, - .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = false, - .PostDownloadVerify = true, - .PrimeCacheOnly = false, - .EnableOtherDownloadsScavenging = m_EnableScavenging, - .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone, - .IncludeWildcards = {}, - .ExcludeWildcards = {}, - .AppendNewContent = false}); - if (AbortFlag) - { - throw std::runtime_error("Test aborted. (Download build)"); - } - } + std::vector<std::string> IncludeWildcards; + std::vector<std::string> ExcludeWildcards; + m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); DownloadFolder(*Output, Workers, Storage, @@ -4318,23 +4277,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - - DownloadOptions{.SystemRootDir = m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, + .AllowMultiparts = m_Parent.m_AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); + .AllowFileClone = m_AllowFileClone, + .IncludeWildcards = IncludeWildcards, + .ExcludeWildcards = ExcludeWildcards, + .AppendNewContent = false}); if (AbortFlag) { throw std::runtime_error("Test aborted. (Download build)"); } - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + ZEN_CONSOLE("\nDownload Filtered Out Remaining of Build {}, Part {} ({}) to '{}'", + BuildId, + BuildPartId, + m_BuildPartName, + DownloadPath); DownloadFolder(*Output, Workers, Storage, @@ -4344,115 +4309,25 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - - DownloadOptions{.SystemRootDir = m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, + .AllowMultiparts = m_Parent.m_AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = false, + .CleanTargetFolder = true, .PostDownloadVerify = true, .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); + .AllowFileClone = m_AllowFileClone, + .IncludeWildcards = ExcludeWildcards, + .ExcludeWildcards = IncludeWildcards, + .AppendNewContent = true}); if (AbortFlag) { - throw std::runtime_error("Test aborted. (Re-download identical target)"); + throw std::runtime_error("Test aborted. (Download build)"); } - auto ScrambleDir = [&Workers](const std::filesystem::path& Path) { - ZEN_CONSOLE("\nScrambling '{}'", Path); - Stopwatch Timer; - DirectoryContent DownloadContent; - GetDirectoryContent( - Path, - DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, - DownloadContent); - auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool { - std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string(); - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - uint32_t Randomizer = 0; - auto FileSizeIt = DownloadContent.FileSizes.begin(); - for (const std::filesystem::path& FilePath : DownloadContent.Files) - { - if (IsAcceptedFolder(FilePath)) - { - uint32_t Case = (Randomizer++) % 7; - switch (Case) - { - case 0: - { - uint64_t SourceSize = *FileSizeIt; - if (SourceSize > 256) - { - Work.ScheduleWork( - Workers.GetIOWorkerPool(), - [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { - if (!AbortFlag) - { - bool WasReadOnly = SetFileReadOnly(FilePath, false); - { - BasicFile Source(FilePath, BasicFile::Mode::kWrite); - uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u); - IoBuffer TempBuffer1(RangeSize); - IoBuffer TempBuffer2(RangeSize); - IoBuffer TempBuffer3(RangeSize); - Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0); - Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2); - Source.Read(TempBuffer3.GetMutableView().GetData(), RangeSize, SourceSize - RangeSize); - Source.Write(TempBuffer1, SourceSize / 2); - Source.Write(TempBuffer2, SourceSize - RangeSize); - Source.Write(TempBuffer3, SourceSize - 0); - } - if (WasReadOnly) - { - SetFileReadOnly(FilePath, true); - } - } - }); - } - } - break; - case 1: - { - (void)SetFileReadOnly(FilePath, false); - (void)RemoveFile(FilePath); - } - break; - default: - break; - } - } - FileSizeIt++; - } - Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, IsPaused); - ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); - }); - ZEN_ASSERT(!AbortFlag.load()); - ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }; - - ScrambleDir(DownloadPath); - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); DownloadFolder(*Output, Workers, Storage, @@ -4462,186 +4337,469 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - - DownloadOptions{.SystemRootDir = m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, + .AllowMultiparts = m_Parent.m_AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); + .AllowFileClone = m_AllowFileClone, + .IncludeWildcards = {}, + .ExcludeWildcards = {}, + .AppendNewContent = false}); if (AbortFlag) { - throw std::runtime_error("Test aborted. (Re-download scrambled target)"); + throw std::runtime_error("Test aborted. (Download build)"); } + } - ScrambleDir(DownloadPath); + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(*Output, + Workers, + Storage, + StorageCacheStats, + BuildId, + {BuildPartId}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, + DownloadPath, + + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .PartialBlockRequestMode = PartialBlockRequestMode, + .CleanTargetFolder = true, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = false, + .AllowFileClone = m_AllowFileClone}); + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Download build)"); + } - Oid BuildId2 = Oid::NewOid(); - Oid BuildPartId2 = Oid::NewOid(); + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(*Output, + Workers, + Storage, + StorageCacheStats, + BuildId, + {BuildPartId}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, + DownloadPath, + + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .PartialBlockRequestMode = PartialBlockRequestMode, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true, + .AllowFileClone = m_AllowFileClone}); + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Re-download identical target)"); + } - CbObject MetaData2 = MakeMetaData(BuildId2); - { - ExtendableStringBuilder<256> SB; - CompactBinaryToJson(MetaData, SB); - ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); - } + auto ScrambleDir = [&Workers, this](const std::filesystem::path& Path) { + ZEN_CONSOLE("\nScrambling '{}'", Path); + Stopwatch Timer; + DirectoryContent DownloadContent; + GetDirectoryContent( + Path, + DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, + DownloadContent); + auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool { + std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string(); + for (const std::string& ExcludeFolder : ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; + }; - UploadFolder(*Output, - Workers, - Storage, - BuildId2, - BuildPartId2, - m_BuildPartName, - DownloadPath, - {}, - MetaData2, - *ChunkController, - *ChunkCache, - UploadFolderOptions{.TempDir = UploadTempDir, - .FindBlockMaxCount = m_FindBlockMaxCount, - .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_AllowMultiparts, - .CreateBuild = true, - .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - if (AbortFlag) + uint32_t Randomizer = 0; + auto FileSizeIt = DownloadContent.FileSizes.begin(); + for (const std::filesystem::path& FilePath : DownloadContent.Files) { - throw std::runtime_error("Test aborted. (Upload scrambled)"); + if (IsAcceptedFolder(FilePath)) + { + uint32_t Case = (Randomizer++) % 7; + switch (Case) + { + case 0: + { + uint64_t SourceSize = *FileSizeIt; + if (SourceSize > 256) + { + Work.ScheduleWork( + Workers.GetIOWorkerPool(), + [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { + if (!AbortFlag) + { + bool WasReadOnly = SetFileReadOnly(FilePath, false); + { + BasicFile Source(FilePath, BasicFile::Mode::kWrite); + uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u); + IoBuffer TempBuffer1(RangeSize); + IoBuffer TempBuffer2(RangeSize); + IoBuffer TempBuffer3(RangeSize); + Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0); + Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2); + Source.Read(TempBuffer3.GetMutableView().GetData(), RangeSize, SourceSize - RangeSize); + Source.Write(TempBuffer1, SourceSize / 2); + Source.Write(TempBuffer2, SourceSize - RangeSize); + Source.Write(TempBuffer3, SourceSize - 0); + } + if (WasReadOnly) + { + SetFileReadOnly(FilePath, true); + } + } + }); + } + } + break; + case 1: + { + (void)SetFileReadOnly(FilePath, false); + (void)RemoveFile(FilePath); + } + break; + default: + break; + } + } + FileSizeIt++; } + Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, IsPaused); + ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); + }); + ZEN_ASSERT(!AbortFlag.load()); + ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }; - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ScrambleDir(DownloadPath); + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(*Output, + Workers, + Storage, + StorageCacheStats, + BuildId, + {BuildPartId}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, + DownloadPath, + + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .PartialBlockRequestMode = PartialBlockRequestMode, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true, + .AllowFileClone = m_AllowFileClone}); + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Re-download scrambled target)"); + } - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, - Workers, - Storage, - StorageCacheStats, - BuildId, - {BuildPartId}, - /*BuildPartNames*/ {}, - /*ManifestPath*/ {}, - DownloadPath, + ScrambleDir(DownloadPath); - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, - .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = false, - .PostDownloadVerify = true, - .PrimeCacheOnly = false, - .EnableOtherDownloadsScavenging = m_EnableScavenging, - .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) - { - throw std::runtime_error("Test aborted. (Download original)"); - } + Oid BuildId2 = Oid::NewOid(); + Oid BuildPartId2 = Oid::NewOid(); - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, - Workers, - Storage, - StorageCacheStats, - BuildId2, - {BuildPartId2}, - /*BuildPartNames*/ {}, - /*ManifestPath*/ {}, - DownloadPath, - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, - .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = false, - .PostDownloadVerify = true, - .PrimeCacheOnly = false, - .EnableOtherDownloadsScavenging = m_EnableScavenging, - .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) - { - throw std::runtime_error("Test aborted. (Download scrambled)"); - } + CbObject MetaData2 = MakeMetaData(BuildId2); + { + ExtendableStringBuilder<256> SB; + CompactBinaryToJson(MetaData, SB); + ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); + } - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, - Workers, - Storage, - StorageCacheStats, - BuildId2, - {BuildPartId2}, - /*BuildPartNames*/ {}, - /*ManifestPath*/ {}, - DownloadPath, - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, - .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = false, - .PostDownloadVerify = true, - .PrimeCacheOnly = false, - .EnableOtherDownloadsScavenging = m_EnableScavenging, - .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + UploadFolder(*Output, + Workers, + Storage, + BuildId2, + BuildPartId2, + m_BuildPartName, + DownloadPath, + {}, + MetaData2, + *ChunkController, + *ChunkCache, + UploadFolderOptions{.TempDir = UploadTempDir, + .FindBlockMaxCount = m_FindBlockMaxCount, + .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .CreateBuild = true, + .IgnoreExistingBlocks = false, + .UploadToZenCache = m_UploadToZenCache}); + + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Upload scrambled)"); + } + + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(*Output, + Workers, + Storage, + StorageCacheStats, + BuildId, + {BuildPartId}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, + DownloadPath, + + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .PartialBlockRequestMode = PartialBlockRequestMode, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true, + .AllowFileClone = m_AllowFileClone}); + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Download original)"); + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); + DownloadFolder(*Output, + Workers, + Storage, + StorageCacheStats, + BuildId2, + {BuildPartId2}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, + DownloadPath, + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .PartialBlockRequestMode = PartialBlockRequestMode, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true, + .AllowFileClone = m_AllowFileClone}); + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Download scrambled)"); + } + + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); + DownloadFolder(*Output, + Workers, + Storage, + StorageCacheStats, + BuildId2, + {BuildPartId2}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, + DownloadPath, + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .PartialBlockRequestMode = PartialBlockRequestMode, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true, + .AllowFileClone = m_AllowFileClone}); + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Re-download scrambled)"); + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); + DownloadFolder(*Output, + Workers, + Storage, + StorageCacheStats, + BuildId, + {BuildPartId}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, + DownloadPath2, + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = DownloadPath2 / ZenFolderName, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .PartialBlockRequestMode = PartialBlockRequestMode, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true, + .AllowFileClone = m_AllowFileClone}); + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Download original)"); + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3); + DownloadFolder(*Output, + Workers, + Storage, + StorageCacheStats, + BuildId, + {BuildPartId}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, + DownloadPath3, + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = DownloadPath3 / ZenFolderName, + .AllowMultiparts = m_Parent.m_AllowMultiparts, + .PartialBlockRequestMode = PartialBlockRequestMode, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true, + .AllowFileClone = m_AllowFileClone}); + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Download original)"); + } +} + +BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent) +: ZenSubCmdBase("multi-test-download", "Download multiple builds sequentially as a test") +, m_Parent(Parent) +{ + auto& Opts = SubOptions(); + Parent.AddSystemOptions(Opts); + Parent.AddCloudOptions(Opts); + Parent.AddFileOptions(Opts); + Parent.AddOutputOptions(Opts); + Parent.AddCacheOptions(Opts); + Parent.AddWorkerOptions(Opts); + Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); + Opts.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>"); + Opts.add_option("", + "", + "enable-scavenge", + "Enable scavenging of data from previouse download locations", + cxxopts::value(m_EnableScavenging), + "<scavenge>"); + Opts.add_option("", + "", + "allow-file-clone", + "Enable use of block reference counting when copying files", + cxxopts::value(m_AllowFileClone), + "<allowclone>"); + Opts.parse_positional({"local-path"}); + Opts.positional_help("local-path"); +} + +void +BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + auto& Opts = SubOptions(); + using namespace builds_impl; + + TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); + if (!IsQuiet) + { + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } + + m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_Parent.m_SystemRootDir); + CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + + m_Parent.ParsePath(m_Path, Opts); + + m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); + + EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts); + + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + std::unique_ptr<AuthMgr> Auth; + std::string DummyBuildId; + StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_Parent.GetZenFolderPath()), + DummyBuildId, + /*RequireNamespace*/ true, + /*RequireBucket*/ true, + /*BoostCacheBackgroundWorkerPool*/ false, + Auth, + Opts); + + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + + Stopwatch Timer; + for (const std::string& BuildIdString : m_BuildIds) + { + Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString)); + if (BuildId == Oid::Zero) { - throw std::runtime_error("Test aborted. (Re-download scrambled)"); + throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), Opts.help()); } - - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); DownloadFolder(*Output, Workers, Storage, StorageCacheStats, BuildId, - {BuildPartId}, + /*BuildPartIds,*/ {}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, - DownloadPath2, - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = DownloadPath2 / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, + m_Path, + DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + .ZenFolderPath = m_Parent.GetZenFolderPath(), + .AllowMultiparts = m_Parent.m_AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = false, + .CleanTargetFolder = BuildIdString == m_BuildIds.front(), .PostDownloadVerify = true, .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, - .EnableTargetFolderScavenging = true, + .EnableTargetFolderScavenging = false, .AllowFileClone = m_AllowFileClone}); if (AbortFlag) { - throw std::runtime_error("Test aborted. (Download original)"); + throw std::runtime_error("Multitest aborted"); } - - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3); - DownloadFolder(*Output, - Workers, - Storage, - StorageCacheStats, - BuildId, - {BuildPartId}, - /*BuildPartNames*/ {}, - /*ManifestPath*/ {}, - DownloadPath3, - DownloadOptions{.SystemRootDir = m_SystemRootDir, - .ZenFolderPath = DownloadPath3 / ZenFolderName, - .AllowMultiparts = m_AllowMultiparts, - .PartialBlockRequestMode = PartialBlockRequestMode, - .CleanTargetFolder = false, - .PostDownloadVerify = true, - .PrimeCacheOnly = false, - .EnableOtherDownloadsScavenging = m_EnableScavenging, - .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + if (!IsQuiet) { - throw std::runtime_error("Test aborted. (Download original)"); + ZEN_CONSOLE("\n"); } } + if (!IsQuiet) + { + ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } } } // namespace zen diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 5c80beed5..7ef71e176 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -7,11 +7,231 @@ #include <zenhttp/auth/authmgr.h> #include <zenhttp/httpclientauth.h> +#include <zenremotestore/builds/buildstoragecache.h> +#include <zenremotestore/builds/buildstorageutil.h> +#include <zenremotestore/partialblockrequestmode.h> #include <filesystem> namespace zen { -class BuildsCommand : public CacheStoreCommand +class BuildsCommand; + +class BuildsListNamespacesSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsListNamespacesSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + bool m_Recursive = false; + std::filesystem::path m_ResultPath; +}; + +class BuildsListSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsListSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_QueryPath; + std::filesystem::path m_ResultPath; +}; + +class BuildsListBlocksSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsListBlocksSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::filesystem::path m_ResultPath; + uint32_t m_MaxCount = 16; +}; + +class BuildsUploadSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsUploadSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::string m_BuildId; + std::string m_BuildPartId; + std::string m_BuildPartName; + bool m_CreateBuild = false; + std::filesystem::path m_BuildMetadataPath; + std::string m_BuildMetadata; + bool m_Clean = false; + uint8_t m_BlockReuseMinPercentLimit = 85; + uint64_t m_FindBlockMaxCount = 10000; + bool m_PostUploadVerify = false; + std::filesystem::path m_ManifestPath; + bool m_UploadToZenCache = true; +}; + +class BuildsDownloadSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsDownloadSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::string m_BuildId; + std::vector<std::string> m_BuildPartIds; + std::vector<std::string> m_BuildPartNames; + bool m_Clean = false; + bool m_Force = false; + bool m_PostDownloadVerify = false; + bool m_EnableScavenging = true; + std::filesystem::path m_DownloadSpecPath; + bool m_UploadToZenCache = true; + bool m_PrimeCacheOnly = false; + bool m_AllowFileClone = true; +}; + +class BuildsLsSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsLsSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::vector<std::string> m_BuildPartIds; + std::vector<std::string> m_BuildPartNames; + std::filesystem::path m_ResultPath; +}; + +class BuildsDiffSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsDiffSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::filesystem::path m_DiffPath; + bool m_OnlyChunked = false; +}; + +class BuildsFetchBlobSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsFetchBlobSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::string m_BlobHash; +}; + +class BuildsPrimeCacheSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsPrimeCacheSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::vector<std::string> m_BuildPartIds; + std::vector<std::string> m_BuildPartNames; + bool m_Force = false; +}; + +class BuildsPauseSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsPauseSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + int m_ZenProcessId = -1; +}; + +class BuildsResumeSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsResumeSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + int m_ZenProcessId = -1; +}; + +class BuildsAbortSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsAbortSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + int m_ZenProcessId = -1; +}; + +class BuildsValidatePartSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsValidatePartSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::string m_BuildPartId; + std::string m_BuildPartName; +}; + +class BuildsTestSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsTestSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::string m_BuildPartName; + std::string m_BuildId; + std::string m_BuildPartId; + bool m_CreateBuild = false; + uint64_t m_FindBlockMaxCount = 10000; + uint8_t m_BlockReuseMinPercentLimit = 85; + bool m_UploadToZenCache = true; + bool m_EnableScavenging = true; + bool m_AllowFileClone = true; +}; + +class BuildsMultiTestDownloadSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::vector<std::string> m_BuildIds; + bool m_EnableScavenging = true; + bool m_AllowFileClone = true; +}; + +class BuildsCommand : public CacheStoreCmdWithSubCommands { public: static constexpr char Name[] = "builds"; @@ -21,25 +241,67 @@ public: BuildsCommand(); ~BuildsCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + cxxopts::Options& Options() override { return m_Options; } + + // Option-adding helpers (called by subcommand constructors) + void AddSystemOptions(cxxopts::Options& Ops); + void AddCloudOptions(cxxopts::Options& Ops); + void AddFileOptions(cxxopts::Options& Ops); + void AddCacheOptions(cxxopts::Options& Ops); + void AddOutputOptions(cxxopts::Options& Ops); + void AddWorkerOptions(cxxopts::Options& Ops); + void AddZenFolderOptions(cxxopts::Options& Ops); + void AddChunkingCacheOptions(cxxopts::Options& Ops); + void AddWildcardOptions(cxxopts::Options& Ops); + void AddExcludeFolderOption(cxxopts::Options& Ops); + void AddExcludeExtensionsOption(cxxopts::Options& Ops); + void AddMultipartOptions(cxxopts::Options& Ops); + void AddPartialBlockRequestOptions(cxxopts::Options& Ops); + void AddAppendNewContentOptions(cxxopts::Options& Ops); + + // Shared parsing/factory methods used by subcommand Run() implementations + void ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts); + StorageInstance CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + const std::filesystem::path& TempPath, + std::string& BuildId, + bool RequireNamespace, + bool RequireBucket, + bool BoostCacheBackgroundWorkerPool, + std::unique_ptr<AuthMgr>& Auth, + cxxopts::Options& SubOpts); + Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts); + Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts); + std::vector<Oid> ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts); + std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts); + CbObject ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts); + void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts); + IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts); + EPartialBlockRequestMode ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts); + void ParseZenProcessId(int& ZenProcessId); + void ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards); + void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions); + + void ResolveZenFolderPath(const std::filesystem::path& DefaultPath); + const std::filesystem::path& GetZenFolderPath() const { return m_ZenFolderPath; } -private: cxxopts::Options m_Options{Name, Description}; + std::string m_SubCommand; + // Shared state populated by AddXxxOptions helpers (bound via cxxopts::value references) std::filesystem::path m_SystemRootDir; + bool m_UseSparseFiles = true; bool m_PlainProgress = false; bool m_LogProgress = false; bool m_Verbose = false; + bool m_Quiet = false; bool m_BoostWorkerCount = false; bool m_BoostWorkerMemory = false; bool m_BoostWorkers = false; - bool m_UseSparseFiles = true; - bool m_Quiet = false; - bool m_AllowFileClone = true; - - std::filesystem::path m_ZenFolderPath; // cloud builds std::string m_OverrideHost; @@ -51,103 +313,46 @@ private: std::string m_Namespace; std::string m_Bucket; - // file storage std::filesystem::path m_StoragePath; bool m_WriteMetadataAsJson = false; - // cache std::string m_ZenCacheHost; - bool m_UploadToZenCache = true; - bool m_PrimeCacheOnly = false; - - std::string m_BuildId; - bool m_CreateBuild = false; - std::filesystem::path m_BuildMetadataPath; - std::string m_BuildMetadata; - std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path - std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName - bool m_Clean = false; - bool m_Force = false; - bool m_AppendNewContent = false; - uint8_t m_BlockReuseMinPercentLimit = 85; - bool m_AllowMultiparts = true; - std::string m_AllowPartialBlockRequests = "true"; AuthCommandLineOptions m_AuthOptions; - std::string m_Verb; // list, upload, download - - cxxopts::Options m_ListNamespacesOptions{"list-namespaces", "List available build namespaces"}; - bool m_ListNamespacesRecursive = false; - - cxxopts::Options m_ListOptions{"list", "List available builds"}; - std::filesystem::path m_ListQueryPath; - std::filesystem::path m_ListResultPath; - - cxxopts::Options m_ListBlocksOptions{"list-blocks", "List recent blocks"}; - uint32_t m_ListBlocksMaxCount = 16; - - std::filesystem::path m_Path; - std::string m_IncludeWildcard; std::string m_ExcludeWildcard; - std::string m_ExcludeFolders; std::string m_ExcludeExtensions; - cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; - uint64_t m_FindBlockMaxCount = 10000; - bool m_PostUploadVerify = false; std::filesystem::path m_ChunkingCachePath; - std::filesystem::path m_ManifestPath; - - cxxopts::Options m_DownloadOptions{"download", "Download a folder"}; - std::vector<std::string> m_BuildPartNames; - std::vector<std::string> m_BuildPartIds; - bool m_PostDownloadVerify = false; - bool m_EnableScavenging = true; - std::filesystem::path m_DownloadSpecPath; - cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"}; - std::filesystem::path m_LsResultPath; + bool m_AllowMultiparts = true; + std::string m_AllowPartialBlockRequests = "true"; - cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; - std::filesystem::path m_DiffPath; - bool m_OnlyChunked = false; + bool m_AppendNewContent = false; - cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"}; - std::string m_BlobHash; - - cxxopts::Options m_PrimeCacheOptions{"prime-cache", "Prime cache from a remote store"}; - - cxxopts::Options m_PauseOptions{"pause", "Pause an ongoing zen builds process"}; - cxxopts::Options m_ResumeOptions{"resume", "Resume a paused zen builds process"}; - cxxopts::Options m_AbortOptions{"abort", "Abort an ongoing zen builds process"}; - - int m_ZenProcessId = -1; - - cxxopts::Options m_ValidateBuildPartOptions{"validate-part", "Fetch a build part and validate all referenced attachments"}; - - cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"}; - - cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; - std::vector<std::string> m_BuildIds; +private: + std::filesystem::path m_ZenFolderPath; - cxxopts::Options* m_SubCommands[15] = {&m_ListNamespacesOptions, - &m_ListOptions, - &m_ListBlocksOptions, - &m_UploadOptions, - &m_DownloadOptions, - &m_PauseOptions, - &m_ResumeOptions, - &m_AbortOptions, - &m_DiffOptions, - &m_LsOptions, - &m_FetchBlobOptions, - &m_PrimeCacheOptions, - &m_ValidateBuildPartOptions, - &m_TestOptions, - &m_MultiTestDownloadOptions}; +protected: + BuildsListNamespacesSubCmd m_ListNamespacesSubCmd; + BuildsListSubCmd m_ListSubCmd; + BuildsListBlocksSubCmd m_ListBlocksSubCmd; + BuildsUploadSubCmd m_UploadSubCmd; + BuildsDownloadSubCmd m_DownloadSubCmd; + BuildsLsSubCmd m_LsSubCmd; + BuildsDiffSubCmd m_DiffSubCmd; + BuildsFetchBlobSubCmd m_FetchBlobSubCmd; + BuildsPrimeCacheSubCmd m_PrimeCacheSubCmd; + BuildsPauseSubCmd m_PauseSubCmd; + BuildsResumeSubCmd m_ResumeSubCmd; + BuildsAbortSubCmd m_AbortSubCmd; + BuildsValidatePartSubCmd m_ValidatePartSubCmd; + BuildsTestSubCmd m_TestSubCmd; + BuildsMultiTestDownloadSubCmd m_MultiTestDownloadSubCmd; + + bool OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) override; }; } // namespace zen |