diff options
Diffstat (limited to 'src/zen/cmds/builds_cmd.h')
| -rw-r--r-- | src/zen/cmds/builds_cmd.h | 383 |
1 files changed, 294 insertions, 89 deletions
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 5c80beed5..7ef71e176 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -7,11 +7,231 @@ #include <zenhttp/auth/authmgr.h> #include <zenhttp/httpclientauth.h> +#include <zenremotestore/builds/buildstoragecache.h> +#include <zenremotestore/builds/buildstorageutil.h> +#include <zenremotestore/partialblockrequestmode.h> #include <filesystem> namespace zen { -class BuildsCommand : public CacheStoreCommand +class BuildsCommand; + +class BuildsListNamespacesSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsListNamespacesSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + bool m_Recursive = false; + std::filesystem::path m_ResultPath; +}; + +class BuildsListSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsListSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_QueryPath; + std::filesystem::path m_ResultPath; +}; + +class BuildsListBlocksSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsListBlocksSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::filesystem::path m_ResultPath; + uint32_t m_MaxCount = 16; +}; + +class BuildsUploadSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsUploadSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::string m_BuildId; + std::string m_BuildPartId; + std::string m_BuildPartName; + bool m_CreateBuild = false; + std::filesystem::path m_BuildMetadataPath; + std::string m_BuildMetadata; + bool m_Clean = false; + uint8_t m_BlockReuseMinPercentLimit = 85; + uint64_t m_FindBlockMaxCount = 10000; + bool m_PostUploadVerify = false; + std::filesystem::path m_ManifestPath; + bool m_UploadToZenCache = true; +}; + +class BuildsDownloadSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsDownloadSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::string m_BuildId; + std::vector<std::string> m_BuildPartIds; + std::vector<std::string> m_BuildPartNames; + bool m_Clean = false; + bool m_Force = false; + bool m_PostDownloadVerify = false; + bool m_EnableScavenging = true; + std::filesystem::path m_DownloadSpecPath; + bool m_UploadToZenCache = true; + bool m_PrimeCacheOnly = false; + bool m_AllowFileClone = true; +}; + +class BuildsLsSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsLsSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::vector<std::string> m_BuildPartIds; + std::vector<std::string> m_BuildPartNames; + std::filesystem::path m_ResultPath; +}; + +class BuildsDiffSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsDiffSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::filesystem::path m_DiffPath; + bool m_OnlyChunked = false; +}; + +class BuildsFetchBlobSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsFetchBlobSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::string m_BlobHash; +}; + +class BuildsPrimeCacheSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsPrimeCacheSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::vector<std::string> m_BuildPartIds; + std::vector<std::string> m_BuildPartNames; + bool m_Force = false; +}; + +class BuildsPauseSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsPauseSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + int m_ZenProcessId = -1; +}; + +class BuildsResumeSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsResumeSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + int m_ZenProcessId = -1; +}; + +class BuildsAbortSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsAbortSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + int m_ZenProcessId = -1; +}; + +class BuildsValidatePartSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsValidatePartSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::string m_BuildId; + std::string m_BuildPartId; + std::string m_BuildPartName; +}; + +class BuildsTestSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsTestSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::string m_BuildPartName; + std::string m_BuildId; + std::string m_BuildPartId; + bool m_CreateBuild = false; + uint64_t m_FindBlockMaxCount = 10000; + uint8_t m_BlockReuseMinPercentLimit = 85; + bool m_UploadToZenCache = true; + bool m_EnableScavenging = true; + bool m_AllowFileClone = true; +}; + +class BuildsMultiTestDownloadSubCmd : public ZenSubCmdBase +{ +public: + explicit BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + BuildsCommand& m_Parent; + std::filesystem::path m_Path; + std::vector<std::string> m_BuildIds; + bool m_EnableScavenging = true; + bool m_AllowFileClone = true; +}; + +class BuildsCommand : public CacheStoreCmdWithSubCommands { public: static constexpr char Name[] = "builds"; @@ -21,25 +241,67 @@ public: BuildsCommand(); ~BuildsCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + cxxopts::Options& Options() override { return m_Options; } + + // Option-adding helpers (called by subcommand constructors) + void AddSystemOptions(cxxopts::Options& Ops); + void AddCloudOptions(cxxopts::Options& Ops); + void AddFileOptions(cxxopts::Options& Ops); + void AddCacheOptions(cxxopts::Options& Ops); + void AddOutputOptions(cxxopts::Options& Ops); + void AddWorkerOptions(cxxopts::Options& Ops); + void AddZenFolderOptions(cxxopts::Options& Ops); + void AddChunkingCacheOptions(cxxopts::Options& Ops); + void AddWildcardOptions(cxxopts::Options& Ops); + void AddExcludeFolderOption(cxxopts::Options& Ops); + void AddExcludeExtensionsOption(cxxopts::Options& Ops); + void AddMultipartOptions(cxxopts::Options& Ops); + void AddPartialBlockRequestOptions(cxxopts::Options& Ops); + void AddAppendNewContentOptions(cxxopts::Options& Ops); + + // Shared parsing/factory methods used by subcommand Run() implementations + void ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts); + StorageInstance CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + const std::filesystem::path& TempPath, + std::string& BuildId, + bool RequireNamespace, + bool RequireBucket, + bool BoostCacheBackgroundWorkerPool, + std::unique_ptr<AuthMgr>& Auth, + cxxopts::Options& SubOpts); + Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts); + Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts); + std::vector<Oid> ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts); + std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts); + CbObject ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts); + void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts); + IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts); + EPartialBlockRequestMode ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts); + void ParseZenProcessId(int& ZenProcessId); + void ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards); + void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions); + + void ResolveZenFolderPath(const std::filesystem::path& DefaultPath); + const std::filesystem::path& GetZenFolderPath() const { return m_ZenFolderPath; } -private: cxxopts::Options m_Options{Name, Description}; + std::string m_SubCommand; + // Shared state populated by AddXxxOptions helpers (bound via cxxopts::value references) std::filesystem::path m_SystemRootDir; + bool m_UseSparseFiles = true; bool m_PlainProgress = false; bool m_LogProgress = false; bool m_Verbose = false; + bool m_Quiet = false; bool m_BoostWorkerCount = false; bool m_BoostWorkerMemory = false; bool m_BoostWorkers = false; - bool m_UseSparseFiles = true; - bool m_Quiet = false; - bool m_AllowFileClone = true; - - std::filesystem::path m_ZenFolderPath; // cloud builds std::string m_OverrideHost; @@ -51,103 +313,46 @@ private: std::string m_Namespace; std::string m_Bucket; - // file storage std::filesystem::path m_StoragePath; bool m_WriteMetadataAsJson = false; - // cache std::string m_ZenCacheHost; - bool m_UploadToZenCache = true; - bool m_PrimeCacheOnly = false; - - std::string m_BuildId; - bool m_CreateBuild = false; - std::filesystem::path m_BuildMetadataPath; - std::string m_BuildMetadata; - std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path - std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName - bool m_Clean = false; - bool m_Force = false; - bool m_AppendNewContent = false; - uint8_t m_BlockReuseMinPercentLimit = 85; - bool m_AllowMultiparts = true; - std::string m_AllowPartialBlockRequests = "true"; AuthCommandLineOptions m_AuthOptions; - std::string m_Verb; // list, upload, download - - cxxopts::Options m_ListNamespacesOptions{"list-namespaces", "List available build namespaces"}; - bool m_ListNamespacesRecursive = false; - - cxxopts::Options m_ListOptions{"list", "List available builds"}; - std::filesystem::path m_ListQueryPath; - std::filesystem::path m_ListResultPath; - - cxxopts::Options m_ListBlocksOptions{"list-blocks", "List recent blocks"}; - uint32_t m_ListBlocksMaxCount = 16; - - std::filesystem::path m_Path; - std::string m_IncludeWildcard; std::string m_ExcludeWildcard; - std::string m_ExcludeFolders; std::string m_ExcludeExtensions; - cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; - uint64_t m_FindBlockMaxCount = 10000; - bool m_PostUploadVerify = false; std::filesystem::path m_ChunkingCachePath; - std::filesystem::path m_ManifestPath; - - cxxopts::Options m_DownloadOptions{"download", "Download a folder"}; - std::vector<std::string> m_BuildPartNames; - std::vector<std::string> m_BuildPartIds; - bool m_PostDownloadVerify = false; - bool m_EnableScavenging = true; - std::filesystem::path m_DownloadSpecPath; - cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"}; - std::filesystem::path m_LsResultPath; + bool m_AllowMultiparts = true; + std::string m_AllowPartialBlockRequests = "true"; - cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; - std::filesystem::path m_DiffPath; - bool m_OnlyChunked = false; + bool m_AppendNewContent = false; - cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"}; - std::string m_BlobHash; - - cxxopts::Options m_PrimeCacheOptions{"prime-cache", "Prime cache from a remote store"}; - - cxxopts::Options m_PauseOptions{"pause", "Pause an ongoing zen builds process"}; - cxxopts::Options m_ResumeOptions{"resume", "Resume a paused zen builds process"}; - cxxopts::Options m_AbortOptions{"abort", "Abort an ongoing zen builds process"}; - - int m_ZenProcessId = -1; - - cxxopts::Options m_ValidateBuildPartOptions{"validate-part", "Fetch a build part and validate all referenced attachments"}; - - cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"}; - - cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; - std::vector<std::string> m_BuildIds; +private: + std::filesystem::path m_ZenFolderPath; - cxxopts::Options* m_SubCommands[15] = {&m_ListNamespacesOptions, - &m_ListOptions, - &m_ListBlocksOptions, - &m_UploadOptions, - &m_DownloadOptions, - &m_PauseOptions, - &m_ResumeOptions, - &m_AbortOptions, - &m_DiffOptions, - &m_LsOptions, - &m_FetchBlobOptions, - &m_PrimeCacheOptions, - &m_ValidateBuildPartOptions, - &m_TestOptions, - &m_MultiTestDownloadOptions}; +protected: + BuildsListNamespacesSubCmd m_ListNamespacesSubCmd; + BuildsListSubCmd m_ListSubCmd; + BuildsListBlocksSubCmd m_ListBlocksSubCmd; + BuildsUploadSubCmd m_UploadSubCmd; + BuildsDownloadSubCmd m_DownloadSubCmd; + BuildsLsSubCmd m_LsSubCmd; + BuildsDiffSubCmd m_DiffSubCmd; + BuildsFetchBlobSubCmd m_FetchBlobSubCmd; + BuildsPrimeCacheSubCmd m_PrimeCacheSubCmd; + BuildsPauseSubCmd m_PauseSubCmd; + BuildsResumeSubCmd m_ResumeSubCmd; + BuildsAbortSubCmd m_AbortSubCmd; + BuildsValidatePartSubCmd m_ValidatePartSubCmd; + BuildsTestSubCmd m_TestSubCmd; + BuildsMultiTestDownloadSubCmd m_MultiTestDownloadSubCmd; + + bool OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) override; }; } // namespace zen |