// Copyright Epic Games, Inc. All Rights Reserved. #pragma once #include #include #include #include #include #include "authutils.h" #include "consoleprogress.h" #include namespace zen { class ProgressBase; class AuthMgr; struct CreateBuildStorageOptions { bool RequireNamespace = true; bool RequireBucket = true; bool BoostCacheBackgroundWorkers = false; }; ////////////////////////////////////////////////////////////////////////// struct BuildsConfiguration { std::filesystem::path SystemRootDir; bool UseSparseFiles = true; bool PlainProgress = false; bool LogProgress = false; bool Verbose = false; bool Quiet = false; ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; bool BoostWorkerCount = false; bool BoostWorkerMemory = false; bool BoostWorkers = false; std::string OverrideHost; std::string Host; std::string Url; bool AssumeHttp2 = false; bool VerboseHttp = false; bool AllowRedirect = false; std::string Namespace; std::string Bucket; std::filesystem::path StoragePath; bool WriteMetadataAsJson = false; std::string ZenCacheHost; AuthCommandLineOptions AuthOptions; std::string IncludeWildcard; std::string ExcludeWildcard; std::string ExcludeFolders; std::string ExcludeExtensions; std::filesystem::path ChunkingCachePath; bool AllowMultiparts = true; std::string AllowPartialBlockRequests = "true"; bool AppendNewContent = false; std::filesystem::path ZenFolderPath; void AddSystemOptions(cxxopts::Options& Ops); void AddCloudOptions(cxxopts::Options& Ops); void AddFileOptions(cxxopts::Options& Ops); void AddCacheOptions(cxxopts::Options& Ops); void AddOutputOptions(cxxopts::Options& Ops); void AddWorkerOptions(cxxopts::Options& Ops); void AddZenFolderOptions(cxxopts::Options& Ops); void AddChunkingCacheOptions(cxxopts::Options& Ops); void AddWildcardOptions(cxxopts::Options& Ops); void AddExcludeFolderOption(cxxopts::Options& Ops); void AddExcludeExtensionsOption(cxxopts::Options& Ops); void AddMultipartOptions(cxxopts::Options& Ops); void AddPartialBlockRequestOptions(cxxopts::Options& Ops); void AddAppendNewContentOptions(cxxopts::Options& Ops); }; ////////////////////////////////////////////////////////////////////////// class BuildsSubCmdBase : public ZenSubCmdBase { public: BuildsSubCmdBase(BuildsConfiguration& Config, std::string_view Name, std::string_view Description) : ZenSubCmdBase(Name, Description) , m_Config(Config) { } protected: const BuildsConfiguration& m_Config; std::filesystem::path m_ResolvedZenFolderPath; // Set by EnsureZenFolderExists: true if this command newly created the zen folder // (so CleanZenFolder may remove the whole folder), false if it already existed // (in which case only the temp subfolder is wiped, to preserve any pre-existing state). bool m_CreatedZenFolder = false; struct ResolvedStorage { std::filesystem::path SystemRootDir; std::string Host; std::string Namespace; std::string Bucket; std::filesystem::path StoragePath; }; void LogBanner(); void LogWorkersInfo(const TransferThreadWorkers& Workers); // SystemRootDirOverride / StoragePathOverride: empty = fall back to m_Config. ResolvedStorage ParseStorageOptions(std::string& BuildId, const CreateBuildStorageOptions& Options, cxxopts::Options& SubOpts, const std::filesystem::path& SystemRootDirOverride = {}, const std::filesystem::path& StoragePathOverride = {}); // Builds the storage instance using the supplied Resolved values. ZenFolder is the final zen folder // path (caller resolves; pass GetZenFolderPath() after ResolveZenFolderPath). If ZenFolder is empty // the storage runs without a temp directory and keeps all downloads in memory. // Caller owns Stats/Auth lifetime. Stats must outlive the returned StorageInstance. StorageInstance CreateBuildStorage(const std::filesystem::path& ZenFolder, const CreateBuildStorageOptions& Options, cxxopts::Options& SubOpts, BuildStorageBase::Statistics& OutStorageStats, BuildStorageCache::Statistics& OutCacheStats, std::unique_ptr& OutAuth, const ResolvedStorage& Resolved); Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts); Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts); std::vector ParseBuildPartIds(const std::vector& BuildPartIdStrs, cxxopts::Options& SubOpts); std::vector ParseBuildPartNames(const std::vector& BuildPartNameStrs, cxxopts::Options& SubOpts); CbObject ParseBuildMetadata(bool CreateBuild, std::filesystem::path& BuildMetadataPath, const std::string& BuildMetadata, cxxopts::Options& SubOpts); void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts); IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts); EPartialBlockRequestMode ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts); void ParseZenProcessId(int& ZenProcessId); void ParseFileFilters(std::vector& OutIncludeWildcards, std::vector& OutExcludeWildcards); void ParseExcludeFolderAndExtension(std::vector& OutExcludeFolders, std::vector& OutExcludeExtensions); // Resolves the zen folder using this chain: --zen-folder-path; else LocalPath/ZenFolderName // if LocalPath is non-empty; else cwd/ZenFolderName. Read-only commands that do not need a // zen folder should skip this and pass an empty path to CreateBuildStorage. void ResolveZenFolderPath(const std::filesystem::path& LocalPath); // Assigns the zen folder to --zen-folder-path if given, else to Fallback directly (no // ZenFolderName appended). For commands that want a specific scratch location rather than // the standard LocalPath/.zen chain. void SetZenFolderPath(const std::filesystem::path& Fallback); const std::filesystem::path& GetZenFolderPath() const { return m_ResolvedZenFolderPath; } // Creates the resolved zen folder and records whether it had to be created. // Use together with CleanZenFolder via MakeGuard. void EnsureZenFolderExists(); std::atomic& AbortFlag() const; std::atomic& PauseFlag() const; std::unique_ptr CreateProgress() const; // Wipes temp state produced by the command. If EnsureZenFolderExists created the zen folder // it is removed outright; otherwise only the temp subfolder is wiped so pre-existing state // (e.g. current_state.cbo from a prior download in the same folder) is preserved. void CleanZenFolder(); }; ////////////////////////////////////////////////////////////////////////// class BuildsListNamespacesSubCmd : public BuildsSubCmdBase { public: explicit BuildsListNamespacesSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: bool m_Recursive = false; std::filesystem::path m_ResultPath; }; ////////////////////////////////////////////////////////////////////////// class BuildsListSubCmd : public BuildsSubCmdBase { public: explicit BuildsListSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: std::filesystem::path m_QueryPath; std::filesystem::path m_ResultPath; }; ////////////////////////////////////////////////////////////////////////// class BuildsListBlocksSubCmd : public BuildsSubCmdBase { public: explicit BuildsListBlocksSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: std::string m_BuildId; std::filesystem::path m_ResultPath; uint32_t m_MaxCount = 16; }; ////////////////////////////////////////////////////////////////////////// class BuildsUploadSubCmd : public BuildsSubCmdBase { public: explicit BuildsUploadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: std::filesystem::path m_Path; std::string m_BuildId; std::string m_BuildPartId; std::string m_BuildPartName; bool m_CreateBuild = false; std::filesystem::path m_BuildMetadataPath; std::string m_BuildMetadata; bool m_Clean = false; uint8_t m_BlockReuseMinPercentLimit = 85; uint64_t m_FindBlockMaxCount = 10000; bool m_PostUploadVerify = false; std::filesystem::path m_ManifestPath; bool m_UploadToZenCache = true; }; ////////////////////////////////////////////////////////////////////////// class BuildsDownloadSubCmd : public BuildsSubCmdBase { public: explicit BuildsDownloadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: std::filesystem::path m_Path; std::string m_BuildId; std::vector m_BuildPartIds; std::vector m_BuildPartNames; bool m_Clean = false; bool m_Force = false; bool m_PostDownloadVerify = false; bool m_EnableScavenging = true; std::filesystem::path m_DownloadSpecPath; bool m_UploadToZenCache = true; bool m_AllowFileClone = true; }; ////////////////////////////////////////////////////////////////////////// class BuildsLsSubCmd : public BuildsSubCmdBase { public: explicit BuildsLsSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: std::string m_BuildId; std::vector m_BuildPartIds; std::vector m_BuildPartNames; std::filesystem::path m_ResultPath; }; ////////////////////////////////////////////////////////////////////////// class BuildsDiffSubCmd : public BuildsSubCmdBase { public: explicit BuildsDiffSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: std::filesystem::path m_Path; std::filesystem::path m_DiffPath; bool m_OnlyChunked = false; }; ////////////////////////////////////////////////////////////////////////// class BuildsFetchBlobSubCmd : public BuildsSubCmdBase { public: explicit BuildsFetchBlobSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: std::string m_BuildId; std::string m_BlobHash; }; ////////////////////////////////////////////////////////////////////////// class BuildsPrimeCacheSubCmd : public BuildsSubCmdBase { public: explicit BuildsPrimeCacheSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: std::string m_BuildId; std::vector m_BuildPartIds; std::vector m_BuildPartNames; bool m_Force = false; }; ////////////////////////////////////////////////////////////////////////// class BuildsPauseSubCmd : public BuildsSubCmdBase { public: explicit BuildsPauseSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: int m_ZenProcessId = -1; }; ////////////////////////////////////////////////////////////////////////// class BuildsResumeSubCmd : public BuildsSubCmdBase { public: explicit BuildsResumeSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: int m_ZenProcessId = -1; }; ////////////////////////////////////////////////////////////////////////// class BuildsAbortSubCmd : public BuildsSubCmdBase { public: explicit BuildsAbortSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: int m_ZenProcessId = -1; }; ////////////////////////////////////////////////////////////////////////// class BuildsValidatePartSubCmd : public BuildsSubCmdBase { public: explicit BuildsValidatePartSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: std::string m_BuildId; std::string m_BuildPartId; std::string m_BuildPartName; }; ////////////////////////////////////////////////////////////////////////// class BuildsTestSubCmd : public BuildsSubCmdBase { public: explicit BuildsTestSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: // Fixture-only overrides of SystemRootDir/StoragePath; passed to ParseStorageOptions explicitly. std::filesystem::path m_TestSystemRootDir; std::filesystem::path m_TestStoragePath; std::filesystem::path m_Path; std::string m_BuildPartName; std::string m_BuildId; std::string m_BuildPartId; bool m_CreateBuild = false; uint64_t m_FindBlockMaxCount = 10000; uint8_t m_BlockReuseMinPercentLimit = 85; bool m_UploadToZenCache = true; bool m_EnableScavenging = true; bool m_AllowFileClone = true; }; ////////////////////////////////////////////////////////////////////////// class BuildsMultiTestDownloadSubCmd : public BuildsSubCmdBase { public: explicit BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: // Fixture-only override of SystemRootDir; passed to CreateStorage explicitly. std::filesystem::path m_TestSystemRootDir; std::filesystem::path m_Path; std::vector m_BuildIds; bool m_EnableScavenging = true; bool m_AllowFileClone = true; }; ////////////////////////////////////////////////////////////////////////// class BuildsCommand : public ZenCmdWithSubCommands { public: static constexpr char Name[] = "builds"; static constexpr char Description[] = "Manage builds - list, list-namespaces, ls, upload, download, prime-cache, diff, fetch-blob, validate-part, pause, resume, abort"; BuildsCommand(); ~BuildsCommand(); cxxopts::Options& Options() override { return m_Options; } BuildsConfiguration& GetConfiguration() { return m_Configuration; } const BuildsConfiguration& GetConfiguration() const { return m_Configuration; } private: cxxopts::Options m_Options{Name, Description}; std::string m_SubCommand; BuildsConfiguration m_Configuration; BuildsListNamespacesSubCmd m_ListNamespacesSubCmd; BuildsListSubCmd m_ListSubCmd; BuildsListBlocksSubCmd m_ListBlocksSubCmd; BuildsUploadSubCmd m_UploadSubCmd; BuildsDownloadSubCmd m_DownloadSubCmd; BuildsLsSubCmd m_LsSubCmd; BuildsDiffSubCmd m_DiffSubCmd; BuildsFetchBlobSubCmd m_FetchBlobSubCmd; BuildsPrimeCacheSubCmd m_PrimeCacheSubCmd; BuildsPauseSubCmd m_PauseSubCmd; BuildsResumeSubCmd m_ResumeSubCmd; BuildsAbortSubCmd m_AbortSubCmd; BuildsValidatePartSubCmd m_ValidatePartSubCmd; BuildsTestSubCmd m_TestSubCmd; BuildsMultiTestDownloadSubCmd m_MultiTestDownloadSubCmd; std::optional m_SigIntGuard; #if ZEN_PLATFORM_WINDOWS std::optional m_SigBreakGuard; #endif bool OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) override; }; } // namespace zen