diff options
Diffstat (limited to 'src/zen/cmds')
31 files changed, 3093 insertions, 4797 deletions
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 034d430fd..2580517fa 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -2,6 +2,8 @@ #include "admin_cmd.h" +#include "zenserviceclient.h" + #include <zencore/basicfile.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> @@ -41,14 +43,8 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); HttpClient::KeyValueMap Params{{"skipdelete", ToString(m_DryRun)}, {"skipgc", ToString(m_NoGc)}, @@ -168,12 +164,7 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); HttpClient::KeyValueMap Params; Params.Entries.insert({"smallobjects", m_SmallObjects ? "true" : "false"}); @@ -258,7 +249,7 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } Params.Entries.insert({"enablevalidation", m_EnableValidation ? "true" : "false"}); - HttpClient Http = CreateHttpClient(m_HostName); + HttpClient& Http = Service.Http(); if (HttpClient::Response Response = Http.Post("/admin/gc"sv, HttpClient::Accept(HttpContentType::kJSON), Params)) { ZEN_CONSOLE("OK: {}", Response.ToText()); @@ -290,14 +281,8 @@ GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Response = Http.Get("/admin/gc"sv, HttpClient::Accept(HttpContentType::kJSON))) { ZEN_CONSOLE("OK: {}", Response.ToText()); @@ -328,14 +313,8 @@ GcStopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Response = Http.Post("/admin/gc-stop"sv, HttpClient::Accept(HttpContentType::kJSON))) { if (Response.StatusCode == HttpResponseCode::Accepted) @@ -377,14 +356,8 @@ JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (m_Cancel) { @@ -460,14 +433,8 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); HttpClient::KeyValueMap Parameters; @@ -522,9 +489,10 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw std::runtime_error(fmt::format("Failed to retrieve {} log path", SourceName)); } - if (!CopyFile(SourcePath, TargetPath, {})) + if (std::error_code Ec = CopyFile(SourcePath, TargetPath, {}); Ec) { - throw std::runtime_error( + throw std::system_error( + Ec, fmt::format("Failed to copy {} log file {} to output file '{}'", SourceName, SourcePath, TargetPath)); } }; @@ -579,16 +547,10 @@ FlushCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - zen::HttpClient Http = CreateHttpClient(m_HostName); - - if (zen::HttpClient::Response Response = Http.Post("/admin/flush"sv)) + if (HttpClient::Response Response = Http.Post("/admin/flush"sv)) { ZEN_CONSOLE("OK: {}", Response.ToText()); @@ -624,7 +586,10 @@ Copy(const std::filesystem::path& Source, const std::filesystem::path& Target) CreateDirectories(Target.parent_path()); CopyFileOptions Options; - CopyFile(Source, Target, Options); + if (std::error_code Ec = CopyFile(Source, Target, Options); Ec) + { + throw std::system_error(Ec, fmt::format("Failed to copy '{}' to '{}'", Source, Target)); + } } static bool @@ -638,7 +603,8 @@ TryCopy(const std::filesystem::path& Source, const std::filesystem::path& Target CreateDirectories(Target.parent_path()); CopyFileOptions Options; - return CopyFile(Source, Target, Options); + std::error_code Ec = CopyFile(Source, Target, Options); + return !Ec; } void diff --git a/src/zen/cmds/bench_cmd.cpp b/src/zen/cmds/bench_cmd.cpp index b1639105a..c935179e2 100644 --- a/src/zen/cmds/bench_cmd.cpp +++ b/src/zen/cmds/bench_cmd.cpp @@ -1661,7 +1661,7 @@ BenchDiskSubCmd::RunClone(const std::filesystem::path& Dir) try { std::filesystem::path DstPath = Dir / fmt::format("bench_clone_{}.tmp", FileIndex); - if (TryCloneFile(SrcPath, DstPath)) + if (std::error_code CloneEc = TryCloneFile(SrcPath, DstPath); !CloneEc) { CloneCount.fetch_add(1, std::memory_order_relaxed); } diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index cc8315e0b..84d8424aa 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -4,68 +4,34 @@ #include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryfile.h> -#include <zencore/compactbinaryfmt.h> -#include <zencore/compactbinaryvalue.h> -#include <zencore/compress.h> -#include <zencore/except.h> +#include <zencore/except_fmt.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/parallelwork.h> +#include <zencore/process.h> #include <zencore/scopeguard.h> #include <zencore/session.h> -#include <zencore/stream.h> #include <zencore/string.h> -#include <zencore/trace.h> -#include <zencore/uid.h> +#include <zenhttp/auth/authmgr.h> #include <zenhttp/formatters.h> -#include <zenhttp/httpclient.h> -#include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> -#include <zenremotestore/builds/buildcontent.h> -#include <zenremotestore/builds/buildmanifest.h> -#include <zenremotestore/builds/buildsavedstate.h> -#include <zenremotestore/builds/buildstoragecache.h> -#include <zenremotestore/builds/buildstorageoperations.h> -#include <zenremotestore/builds/buildstorageutil.h> +#include <zenremotestore/builds/buildinspect.h> +#include <zenremotestore/builds/buildprimecache.h> +#include <zenremotestore/builds/buildupdatefolder.h> +#include <zenremotestore/builds/builduploadfolder.h> +#include <zenremotestore/builds/buildvalidatebuildpart.h> #include <zenremotestore/builds/filebuildstorage.h> #include <zenremotestore/builds/jupiterbuildstorage.h> -#include <zenremotestore/chunking/chunkblock.h> -#include <zenremotestore/chunking/chunkedcontent.h> -#include <zenremotestore/chunking/chunkedfile.h> #include <zenremotestore/chunking/chunkingcache.h> #include <zenremotestore/chunking/chunkingcontroller.h> -#include <zenremotestore/filesystemutils.h> -#include <zenremotestore/jupiter/jupiterhost.h> -#include <zenremotestore/operationlogoutput.h> -#include <zenremotestore/transferthreadworkers.h> +#include <zenutil/filesystemutils.h> +#include <zenutil/progress.h> #include <zenutil/wildcard.h> #include <zenutil/workerpools.h> -#include <zenutil/zenserverprocess.h> - -#include "../progressbar.h" #include <signal.h> #include <memory> -#include <numeric> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <tsl/robin_map.h> -#include <tsl/robin_set.h> -#include <json11.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#else -# include <fcntl.h> -# include <sys/file.h> -# include <sys/stat.h> -# include <unistd.h> -#endif - -static const bool DoExtraContentVerify = false; namespace zen { @@ -115,14 +81,6 @@ namespace builds_impl { } }; - struct MemMap - { - void* Handle = nullptr; - void* Data = nullptr; - size_t Size = 0; - std::string Name; - }; - class ZenState { public: @@ -236,1869 +194,103 @@ namespace builds_impl { } } - const std::string ZenFolderName = ".zen"; - std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; } - // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json"; - // } - - std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path) - { - const std::u8string LocalPathString = Path.generic_u8string(); - IoHash PathHash = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length()); - return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash); - } - - const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; - - const std::string UnsyncFolderName = ".unsync"; - - const std::string UGSFolderName = ".ugs"; - const std::string LegacyZenTempFolderName = ".zen-tmp"; - - const std::vector<std::string> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); - const std::vector<std::string> DefaultExcludeExtensions({}); - + // Debugging knobs for file build storage - always 0 in shipped builds const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; - const bool SingleThreaded = false; - bool UseSparseFiles = false; - - static bool IsVerbose = false; - static bool IsQuiet = false; - static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; - -#undef ZEN_CONSOLE_VERBOSE -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ - } - - const std::string DefaultAccessTokenEnvVariableName( -#if ZEN_PLATFORM_WINDOWS - "UE-CloudDataCacheAccessToken"sv -#endif -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - "UE_CloudDataCacheAccessToken"sv -#endif - - ); - - static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory) - { - return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; - } - - class FilteredRate - { - public: - FilteredRate() {} - - void Start() - { - if (StartTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs())) - { - LastTimeUS = StartTimeUS.load(); - } - } - } - void Stop() - { - if (EndTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()); - } - } - - void Update(uint64_t Count) - { - if (LastTimeUS == (uint64_t)-1) - { - return; - } - uint64_t TimeUS = Timer.GetElapsedTimeUs(); - uint64_t TimeDeltaUS = TimeUS - LastTimeUS; - if (TimeDeltaUS >= 2000000) - { - uint64_t Delta = Count - LastCount; - uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS; - - LastPerSecond = PerSecond; - - LastCount = Count; - - FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8; - - LastTimeUS = TimeUS; - } - } - - uint64_t GetCurrent() const // If Stopped - return total count / total time - { - if (LastTimeUS == (uint64_t)-1) - { - return 0; - } - return FilteredPerSecond; - } - - uint64_t GetElapsedTimeUS() const - { - if (StartTimeUS == (uint64_t)-1) - { - return 0; - } - if (EndTimeUS == (uint64_t)-1) - { - return 0; - } - uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS; - return TimeDeltaUS; - } - - bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); } - - private: - Stopwatch Timer; - std::atomic<uint64_t> StartTimeUS = (uint64_t)-1; - std::atomic<uint64_t> EndTimeUS = (uint64_t)-1; - std::atomic<uint64_t> LastTimeUS = (uint64_t)-1; - uint64_t LastCount = 0; - uint64_t LastPerSecond = 0; - uint64_t FilteredPerSecond = 0; - }; - - uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count) + void WriteResultObject(const std::filesystem::path& Path, const CbObject& Response) { - if (ElapsedWallTimeUS == 0) - { - return 0; - } - return Count * 1000000 / ElapsedWallTimeUS; - } - - bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, const std::filesystem::path& Directory) - { - return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory); - } - - void ValidateBuildPart(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - BuildStorageBase& Storage, - const Oid& BuildId, - Oid BuildPartId, - const std::string_view BuildPartName) - { - ZEN_TRACE_CPU("ValidateBuildPart"); - - ProgressBar::SetLogOperationName(ProgressMode, "Validate Part"); - - BuildsOperationValidateBuildPart ValidateOp(Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - BuildPartId, - BuildPartName, - BuildsOperationValidateBuildPart::Options{.IsQuiet = IsQuiet, .IsVerbose = IsVerbose}); - - ValidateOp.Execute(); - - const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount; - const uint64_t DownloadedByteCount = - ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount; - ZEN_CONSOLE("Verified: {:>8} ({}), {}B/sec, {}", - DownloadedCount, - NiceBytes(DownloadedByteCount), - NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)), - NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000)); - } - - struct UploadFolderOptions - { - std::filesystem::path TempDir; - uint64_t FindBlockMaxCount; - uint8_t BlockReuseMinPercentLimit; - bool AllowMultiparts; - bool CreateBuild; - bool IgnoreExistingBlocks; - bool UploadToZenCache; - const std::vector<std::string>& ExcludeFolders = DefaultExcludeFolders; - const std::vector<std::string>& ExcludeExtensions = DefaultExcludeExtensions; - }; - - std::vector<std::pair<Oid, std::string>> UploadFolder(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& Path, - const std::filesystem::path& ManifestPath, - const CbObject& MetaData, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const UploadFolderOptions& Options) - { - ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder"); - - Stopwatch UploadTimer; - - BuildsOperationUploadFolder UploadOp( - Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - Options.CreateBuild, - std::move(MetaData), - BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .DoExtraContentValidation = DoExtraContentVerify, - .FindBlockMaxCount = Options.FindBlockMaxCount, - .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit, - .AllowMultiparts = Options.AllowMultiparts, - .IgnoreExistingBlocks = Options.IgnoreExistingBlocks, - .TempDir = Options.TempDir, - .ExcludeFolders = Options.ExcludeFolders, - .ExcludeExtensions = Options.ExcludeExtensions, - .ZenExcludeManifestName = ZenExcludeManifestName, - .NonCompressableExtensions = DefaultSplitOnlyExtensions, - .PopulateCache = Options.UploadToZenCache}); - - std::vector<std::pair<Oid, std::string>> UploadedParts = - UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache); - if (AbortFlag) - { - return {}; - } - - ZEN_CONSOLE_VERBOSE( - "Folder scanning stats:" - "\n FoundFileCount: {}" - "\n FoundFileByteCount: {}" - "\n AcceptedFileCount: {}" - "\n AcceptedFileByteCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()), - NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Chunking stats:" - "\n FilesProcessed: {}" - "\n FilesChunked: {}" - "\n BytesHashed: {}" - "\n UniqueChunksFound: {}" - "\n UniqueSequencesFound: {}" - "\n UniqueBytesFound: {}" - "\n FilesFoundInCache: {}" - "\n ChunksFoundInCache: {}" - "\n FilesStoredInCache: {}" - "\n ChunksStoredInCache: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_ChunkingStats.FilesProcessed.load(), - UploadOp.m_ChunkingStats.FilesChunked.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()), - UploadOp.m_ChunkingStats.UniqueChunksFound.load(), - UploadOp.m_ChunkingStats.UniqueSequencesFound.load(), - NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()), - UploadOp.m_ChunkingStats.FilesFoundInCache.load(), - UploadOp.m_ChunkingStats.ChunksFoundInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()), - UploadOp.m_ChunkingStats.FilesStoredInCache.load(), - UploadOp.m_ChunkingStats.ChunksStoredInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()), - NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Find block stats:" - "\n FindBlockTimeMS: {}" - "\n PotentialChunkCount: {}" - "\n PotentialChunkByteCount: {}" - "\n FoundBlockCount: {}" - "\n FoundBlockChunkCount: {}" - "\n FoundBlockByteCount: {}" - "\n AcceptedBlockCount: {}" - "\n NewBlocksCount: {}" - "\n NewBlocksChunkCount: {}" - "\n NewBlocksChunkByteCount: {}", - NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS), - UploadOp.m_FindBlocksStats.PotentialChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount), - UploadOp.m_FindBlocksStats.FoundBlockCount, - UploadOp.m_FindBlocksStats.FoundBlockChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount), - UploadOp.m_FindBlocksStats.AcceptedBlockCount, - UploadOp.m_FindBlocksStats.NewBlocksCount, - UploadOp.m_FindBlocksStats.NewBlocksChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Reuse block stats:" - "\n AcceptedChunkCount: {}" - "\n AcceptedByteCount: {}" - "\n AcceptedRawByteCount: {}" - "\n RejectedBlockCount: {}" - "\n RejectedChunkCount: {}" - "\n RejectedByteCount: {}" - "\n AcceptedReduntantChunkCount: {}" - "\n AcceptedReduntantByteCount: {}", - UploadOp.m_ReuseBlocksStats.AcceptedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount), - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount), - UploadOp.m_ReuseBlocksStats.RejectedBlockCount, - UploadOp.m_ReuseBlocksStats.RejectedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount), - UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n GeneratedBlockByteCount: {}" - "\n GeneratedBlockCount: {}" - "\n GenerateBlocksElapsedWallTimeUS: {}", - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n ChunkCount: {}" - "\n ChunkByteCount: {}" - "\n CompressedChunkCount: {}" - "\n CompressChunksElapsedWallTimeUS: {}", - UploadOp.m_LooseChunksStats.ChunkCount, - NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount), - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Disk stats:" - "\n OpenReadCount: {}" - "\n OpenWriteCount: {}" - "\n ReadCount: {}" - "\n ReadByteCount: {}" - "\n WriteCount: {} ({} cloned)" - "\n WriteByteCount: {} ({} cloned)" - "\n CurrentOpenFileCount: {}", - UploadOp.m_DiskStats.OpenReadCount.load(), - UploadOp.m_DiskStats.OpenWriteCount.load(), - UploadOp.m_DiskStats.ReadCount.load(), - NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()), - UploadOp.m_DiskStats.WriteCount.load(), - UploadOp.m_DiskStats.CloneCount.load(), - NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()), - NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()), - UploadOp.m_DiskStats.CurrentOpenFileCount.load()); - - ZEN_CONSOLE_VERBOSE( - "Upload stats:" - "\n BlockCount: {}" - "\n BlocksBytes: {}" - "\n ChunkCount: {}" - "\n ChunksBytes: {}" - "\n ReadFromDiskBytes: {}" - "\n MultipartAttachmentCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()), - UploadOp.m_UploadStats.MultipartAttachmentCount.load(), - NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000)); - - const double DeltaByteCountPercent = - UploadOp.m_ChunkingStats.BytesHashed > 0 - ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) / - (UploadOp.m_ChunkingStats.BytesHashed) - : 0.0; - - const std::string MultipartAttachmentStats = - Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : ""; - - if (!IsQuiet) - { - ZEN_CONSOLE( - "Uploaded part {} ('{}') to build {}, {}\n" - " Scanned files: {:>8} ({}), {}B/sec, {}\n" - " New data: {:>8} ({}) {:.1f}%\n" - " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" - " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" - " Uploaded: {:>8} ({}), {}bits/sec, {}\n" - " Blocks: {:>8} ({})\n" - " Chunks: {:>8} ({}){}", - BuildPartId, - BuildPartName, - BuildId, - NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()), - - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)), - NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes), - DeltaByteCountPercent, - - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount), - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)), - NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), - - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS, - UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)), - NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS, - (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)), - NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - MultipartAttachmentStats); - } - return UploadedParts; - } - - struct VerifyFolderStatistics - { - std::atomic<uint64_t> FilesVerified = 0; - std::atomic<uint64_t> FilesFailed = 0; - std::atomic<uint64_t> ReadBytes = 0; - uint64_t VerifyElapsedWallTimeUs = 0; - }; - - void VerifyFolder(TransferThreadWorkers& Workers, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::filesystem::path& Path, - const std::vector<std::string>& ExcludeFolders, - bool VerifyFileHash, - VerifyFolderStatistics& VerifyFolderStats) - { - ZEN_TRACE_CPU("VerifyFolder"); - - Stopwatch Timer; - - ProgressBar ProgressBar(ProgressMode, "Verify Files"); - - WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool(); - - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size()); - - RwLock ErrorLock; - std::vector<std::string> Errors; - - auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++) - { - if (Work.IsAborted()) - { - break; - } - - Work.ScheduleWork( - VerifyPool, - [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex]( - std::atomic<bool>&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("VerifyFile_work"); - - // TODO: Convert ScheduleWork body to function - - const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred(); - if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) - { - const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; - if (!IsFile(TargetPath)) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); - }); - VerifyFolderStats.FilesFailed++; - } - else - { - std::error_code Ec; - uint64_t SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec)); - if (Ec) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back( - fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value())); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk < ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > 0 && VerifyFileHash) - { - const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex]; - IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath); - IoHash RawHash = IoHash::HashBuffer(Buffer); - if (RawHash != ExpectedRawHash) - { - uint64_t FileOffset = 0; - const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash); - const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex]; - for (uint32_t OrderIndex = OrderOffset; - OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex]; - OrderIndex++) - { - uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; - uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; - IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize); - if (IoHash::HashBuffer(FileChunk) != ChunkHash) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format( - "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. " - "Mismatch at chunk {}", - TargetPath, - ExpectedRawHash, - RawHash, - OrderIndex - OrderOffset)); - }); - break; - } - FileOffset += ChunkSize; - } - VerifyFolderStats.FilesFailed++; - } - VerifyFolderStats.ReadBytes += SizeOnDisk; - } - } - } - VerifyFolderStats.FilesVerified++; - } - }, - [&, PathIndex](std::exception_ptr Ex, std::atomic<bool>&) { - std::string Description; - try - { - std::rethrow_exception(Ex); - } - catch (const std::exception& Ex) - { - Description = Ex.what(); - } - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}", - (Path / Content.Paths[PathIndex]).make_preferred(), - Description)); - }); - VerifyFolderStats.FilesFailed++; - }); - } - - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", - VerifyFolderStats.FilesVerified.load(), - PathCount, - NiceBytes(VerifyFolderStats.ReadBytes.load()), - VerifyFolderStats.FilesFailed.load()); - ProgressBar.UpdateState({.Task = "Verifying files ", - .Details = Details, - .TotalCount = gsl::narrow<uint64_t>(PathCount), - .RemainingCount = gsl::narrow<uint64_t>(PathCount - VerifyFolderStats.FilesVerified.load()), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs(); - - ProgressBar.Finish(); - if (AbortFlag) - { - return; - } - - for (const std::string& Error : Errors) - { - ZEN_CONSOLE_ERROR("{}", Error); - } - if (!Errors.empty()) - { - throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size())); - } - } - - CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId) - { - Stopwatch GetBuildTimer; - CbObject BuildObject = Storage.GetBuild(BuildId); - if (!IsQuiet) - { - ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", - NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["name"sv].AsString(), - NiceBytes(BuildObject.GetSize())); - - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); - } - return BuildObject; - } - - std::vector<std::filesystem::path> GetNewPaths(const std::span<const std::filesystem::path> KnownPaths, - const std::span<const std::filesystem::path> Paths) - { - tsl::robin_set<std::string> KnownPathsSet; - KnownPathsSet.reserve(KnownPaths.size()); - for (const std::filesystem::path& LocalPath : KnownPaths) - { - KnownPathsSet.insert(LocalPath.generic_string()); - } - - std::vector<std::filesystem::path> NewPaths; - for (const std::filesystem::path& UntrackedPath : Paths) - { - if (!KnownPathsSet.contains(UntrackedPath.generic_string())) - { - NewPaths.push_back(UntrackedPath); - } - } - return NewPaths; - } - - BuildSaveState GetLocalStateFromPaths(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - std::span<const std::filesystem::path> PathsToCheck) - { - FolderContent FolderState; - ChunkedFolderContent ChunkedContent; - { - ProgressBar ProgressBar(ProgressMode, "Check Files"); - FolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - PathsToCheck, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - - if (FolderState.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : FolderState.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - FolderState, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - FolderState.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - ChunkingStats += LocalChunkingStats; - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - } - - return BuildSaveState{.State = BuildState{.ChunkedContent = std::move(ChunkedContent)}, - .FolderState = FolderState, - .LocalPath = Path}; - } - - BuildSaveState GetLocalContent(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - const std::filesystem::path& StateFilePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch ReadStateTimer; - bool FileExists = IsFile(StateFilePath); - if (!FileExists) - { - ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path); - return {}; - } - - BuildSaveState SavedLocalState; - try - { - SavedLocalState = ReadBuildSaveStateFile(StateFilePath); - if (!IsQuiet) - { - ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); - } - } - catch (const std::exception& Ex) + const MemoryView ResponseView = Response.GetView(); + if (ToLower(Path.extension().string()) == ".cbo") { - ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what()); - return {}; - } - - FolderContent CurrentLocalFolderState; - { - ProgressBar ProgressBar(ProgressMode, "Check Known Files"); - CurrentLocalFolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - SavedLocalState.FolderState.Paths, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - if (AbortFlag) - { - return {}; - } - - if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState)) - { - const size_t LocalStatePathCount = SavedLocalState.FolderState.Paths.size(); - std::vector<std::filesystem::path> DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths); - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - - if (!IsQuiet) - { - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocalStatePathCount); - } - if (UpdatedContent.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : UpdatedContent.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Known Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - UpdatedContent, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - UpdatedContent.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - - ChunkingStats += LocalChunkingStats; - - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - if (AbortFlag) - { - return {}; - } - SavedLocalState.State.ChunkedContent = - MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}}); - } - } - else - { - // Remove files from LocalContent no longer in LocalFolderState - tsl::robin_set<std::string> LocalFolderPaths; - LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size()); - for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths) - { - LocalFolderPaths.insert(LocalFolderPath.generic_string()); - } - std::vector<std::filesystem::path> DeletedPaths; - for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths) - { - if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) - { - DeletedPaths.push_back(LocalContentPath); - } - } - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - } - - SavedLocalState.FolderState = CurrentLocalFolderState; - - return SavedLocalState; - } - - ChunkedFolderContent ScanAndChunkFolder( - TransferThreadWorkers& Workers, - GetFolderContentStatistics& GetFolderContentStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder, - std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch Timer; - - ZEN_TRACE_CPU("ScanAndChunkFolder"); - - FolderContent Content = GetFolderContent( - GetFolderContentStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - Workers.GetIOWorkerPool(), - GetUpdateDelayMS(ProgressMode), - [](bool, std::ptrdiff_t) {}, - AbortFlag); - if (AbortFlag) - { - return {}; - } - - BuildState LocalContent = GetLocalContent(Workers, - GetFolderContentStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - ChunkController, - ChunkCache) - .State; - - std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths); - - BuildState UntrackedLocalContent = - GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, ChunkCache, UntrackedPaths).State; - - ChunkedFolderContent Result = MergeChunkedFolderContents(LocalContent.ChunkedContent, - std::vector<ChunkedFolderContent>{UntrackedLocalContent.ChunkedContent}); - - const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0)); - const uint64_t ChunkedRawSize = - std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - Result.Paths.size(), - NiceBytes(TotalRawSize), - Result.ChunkedContent.ChunkHashes.size(), - NiceBytes(ChunkedRawSize), - Path, - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); - } - return Result; - }; - - struct DownloadOptions - { - std::filesystem::path SystemRootDir; - std::filesystem::path ZenFolderPath; - bool AllowMultiparts = true; - EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed; - bool CleanTargetFolder = false; - bool PostDownloadVerify = false; - bool PrimeCacheOnly = false; - bool EnableOtherDownloadsScavenging = true; - bool EnableTargetFolderScavenging = true; - bool AllowFileClone = true; - std::vector<std::string> IncludeWildcards; - std::vector<std::string> ExcludeWildcards; - uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; - bool PopulateCache = true; - bool AppendNewContent = false; - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; - }; - - void DownloadFolder(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const BuildStorageCache::Statistics& StorageCacheStats, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - const std::filesystem::path& DownloadSpecPath, - const std::filesystem::path& Path, - const DownloadOptions& Options) - { - ZEN_TRACE_CPU("DownloadFolder"); - - ProgressBar::SetLogOperationName(ProgressMode, "Download Folder"); - - enum TaskSteps : uint32_t - { - CheckState, - CompareState, - Download, - Verify, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - ZEN_ASSERT((!Options.PrimeCacheOnly) || - (Options.PrimeCacheOnly && (Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off))); - - Stopwatch DownloadTimer; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount); - - const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath); - CreateDirectories(ZenTempFolder); - - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - BuildManifest Manifest; - if (!DownloadSpecPath.empty()) - { - const std::filesystem::path AbsoluteDownloadSpecPath = - DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath); - Manifest = ParseBuildManifest(DownloadSpecPath); - } - - std::vector<ChunkedFolderContent> PartContents; - - std::unique_ptr<ChunkingController> ChunkController; - - std::vector<ChunkBlockDescription> BlockDescriptions; - std::vector<IoHash> LooseChunkHashes; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); - - ChunkedFolderContent RemoteContent = GetRemoteContent(Output, - Storage, - BuildId, - AllBuildParts, - Manifest, - Options.IncludeWildcards, - Options.ExcludeWildcards, - ChunkController, - PartContents, - BlockDescriptions, - LooseChunkHashes, - IsQuiet, - IsVerbose, - DoExtraContentVerify); - - const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; - GetFolderContentStatistics LocalFolderScanStats; - ChunkingStatistics ChunkingStats; - - BuildSaveState LocalState; - - if (!Options.PrimeCacheOnly) - { - if (IsDir(Path)) - { - if (!ChunkController && !IsQuiet) - { - ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default"); - ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - } - std::unique_ptr<ChunkingCache> ChunkCache(CreateNullChunkingCache()); - - LocalState = GetLocalContent(Workers, - LocalFolderScanStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - *ChunkController, - *ChunkCache); - - std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths); - - BuildSaveState UntrackedLocalContent = GetLocalStateFromPaths(Workers, - LocalFolderScanStats, - ChunkingStats, - Path, - *ChunkController, - *ChunkCache, - UntrackedPaths); - - if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty()) - { - LocalState.State.ChunkedContent = - MergeChunkedFolderContents(LocalState.State.ChunkedContent, - std::vector<ChunkedFolderContent>{UntrackedLocalContent.State.ChunkedContent}); - - // TODO: Helper - LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.end()); - LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.end()); - LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.end()); - LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.end()); - } - - if (Options.AppendNewContent) - { - RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent, - RemoteContent, - Options.IncludeWildcards, - Options.ExcludeWildcards); - } -#if ZEN_BUILD_DEBUG - ValidateChunkedFolderContent(RemoteContent, - BlockDescriptions, - LooseChunkHashes, - Options.IncludeWildcards, - Options.ExcludeWildcards); -#endif // ZEN_BUILD_DEBUG - } - else - { - CreateDirectories(Path); - } - } - if (AbortFlag) - { - return; - } - - LocalState.LocalPath = Path; - - { - BuildsSelection::Build RemoteBuildState = {.Id = BuildId, - .IncludeWildcards = Options.IncludeWildcards, - .ExcludeWildcards = Options.ExcludeWildcards}; - RemoteBuildState.Parts.reserve(BuildPartIds.size()); - for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++) - { - RemoteBuildState.Parts.push_back( - {BuildsSelection::BuildPart{.Id = BuildPartIds[PartIndex], - .Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}}); - } - - if (Options.AppendNewContent) - { - LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState)); - } - else - { - LocalState.State.Selection.Builds = std::vector<BuildsSelection::Build>{std::move(RemoteBuildState)}; - } - } - - if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder && - CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent)) - { - if (!IsQuiet) - { - ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", - NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); - } - - Stopwatch WriteStateTimer; - - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); + WriteFile(Path, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } else { - ExtendableStringBuilder<128> BuildPartString; - for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts) - { - BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); - } - - uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); - } - - Stopwatch IndexTimer; - - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent); - const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); - - if (!IsQuiet) - { - ZEN_OPERATION_LOG_INFO(Output, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); - - BuildsOperationUpdateFolder Updater( - Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - LocalState.State.ChunkedContent, - LocalLookup, - RemoteContent, - RemoteLookup, - BlockDescriptions, - LooseChunkHashes, - BuildsOperationUpdateFolder::Options{ - .IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .AllowFileClone = Options.AllowFileClone, - .UseSparseFiles = UseSparseFiles, - .SystemRootDir = Options.SystemRootDir, - .ZenFolderPath = Options.ZenFolderPath, - .LargeAttachmentSize = LargeAttachmentSize, - .PreferredMultipartChunkSize = PreferredMultipartChunkSize, - .PartialBlockRequestMode = Options.PartialBlockRequestMode, - .WipeTargetFolder = Options.CleanTargetFolder, - .PrimeCacheOnly = Options.PrimeCacheOnly, - .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, - .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent, - .ValidateCompletedSequences = Options.PostDownloadVerify, - .ExcludeFolders = Options.ExcludeFolders, - .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize, - .PopulateCache = Options.PopulateCache}); - { - ProgressBar::PushLogOperation(ProgressMode, "Download"); - auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); }); - FolderContent UpdatedLocalFolderState; - Updater.Execute(UpdatedLocalFolderState); - - LocalState.State.ChunkedContent = RemoteContent; - LocalState.FolderState = std::move(UpdatedLocalFolderState); - } - - VerifyFolderStatistics VerifyFolderStats; - if (!AbortFlag) - { - if (!Options.PrimeCacheOnly) - { - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); - - VerifyFolder(Workers, - RemoteContent, - RemoteLookup, - Path, - Options.ExcludeFolders, - Options.PostDownloadVerify, - VerifyFolderStats); - - Stopwatch WriteStateTimer; - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - -#if 0 - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(StateObject, SB); - WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); -#endif // 0 - } - const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() + - Updater.m_DownloadStats.DownloadedBlockCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockCount.load(); - const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() + - Updater.m_DownloadStats.DownloadedBlockByteCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load(); - const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs(); - - if (!IsQuiet) - { - std::string CloneInfo; - if (Updater.m_DiskStats.CloneByteCount > 0) - { - CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load())); - } - - std::string DownloadDetails; - { - ExtendableStringBuilder<128> SB; - BuildStorageBase::ExtendedStatistics ExtendedDownloadStats; - if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats)) - { - if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty()) - { - for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append(It.first); - SB.Append(": "sv); - SB.Append(NiceBytes(It.second)); - } - } - } - if (Storage.CacheStorage) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append("Cache: "); - SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load())); - } - if (SB.Size() > 0) - { - DownloadDetails = fmt::format(" ({})", SB.ToView()); - } - } - - ZEN_CONSOLE( - "Downloaded build {}, parts:{} in {}\n" - " Scavenge: {} (Target: {}, Cache: {}, Others: {})\n" - " Download: {} ({}) {}bits/s{}\n" - " Write: {} ({}) {}B/s{}\n" - " Clean: {}\n" - " Finalize: {}\n" - " Verify: {}", - BuildId, - BuildPartString.ToView(), - NiceTimeSpanMs(DownloadTimeMs), - - NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) / - 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000), - - DownloadCount, - NiceBytes(DownloadByteCount), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), - DownloadDetails, - - Updater.m_DiskStats.WriteCount.load(), - NiceBytes(Updater.m_WrittenChunkByteCount.load()), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())), - CloneInfo, - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); - } - } - } - if (Options.PrimeCacheOnly) - { - if (Storage.CacheStorage) - { - Storage.CacheStorage->Flush(5000, [](intptr_t Remaining) { - if (!IsQuiet) - { - if (Remaining == 0) - { - ZEN_CONSOLE("Build cache upload complete"); - } - else - { - ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); - } - } - return !AbortFlag; - }); - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); - - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), ZenTempFolder); - } - - void ListBuild(StorageInstance& Storage, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - std::span<const std::string> IncludeWildcards, - std::span<const std::string> ExcludeWildcards, - CbObjectWriter* OptionalStructuredOutput) - { - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId); - OptionalStructuredOutput->AddObject("build"sv, BuildObject); - } - - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - if (!AllBuildParts.empty()) - { - Stopwatch GetBuildPartTimer; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("parts"sv); - } - - for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) - { - const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; - const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; - CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId); - OptionalStructuredOutput->AddString("partName"sv, BuildPartName); - } - { - if (OptionalStructuredOutput != nullptr) - { - } - else if (!IsQuiet) - { - ZEN_CONSOLE("{}Part: {} ('{}'):\n", - BuildPartIndex > 0 ? "\n" : "", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); - } - - std::vector<std::filesystem::path> Paths; - std::vector<IoHash> RawHashes; - std::vector<uint64_t> RawSizes; - std::vector<uint32_t> Attributes; - - SourcePlatform Platform; - std::vector<IoHash> SequenceRawHashes; - std::vector<uint32_t> ChunkCounts; - std::vector<uint32_t> AbsoluteChunkOrders; - std::vector<IoHash> LooseChunkHashes; - std::vector<uint64_t> LooseChunkRawSizes; - std::vector<IoHash> BlockRawHashes; - - ReadBuildContentFromCompactBinary(BuildPartManifest, - Platform, - Paths, - RawHashes, - RawSizes, - Attributes, - SequenceRawHashes, - ChunkCounts, - AbsoluteChunkOrders, - LooseChunkHashes, - LooseChunkRawSizes, - BlockRawHashes); - - std::vector<size_t> Order(Paths.size()); - std::iota(Order.begin(), Order.end(), 0); - - std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { - const std::filesystem::path& LhsPath = Paths[Lhs]; - const std::filesystem::path& RhsPath = Paths[Rhs]; - return LhsPath < RhsPath; - }); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("files"sv); - } - { - for (size_t Index : Order) - { - const std::filesystem::path& Path = Paths[Index]; - if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true)) - { - const IoHash& RawHash = RawHashes[Index]; - const uint64_t RawSize = RawSizes[Index]; - const uint32_t Attribute = Attributes[Index]; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - { - OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path)); - OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize); - OptionalStructuredOutput->AddHash("rawHash"sv, RawHash); - switch (Platform) - { - case SourcePlatform::Windows: - OptionalStructuredOutput->AddInteger("attributes"sv, Attribute); - break; - case SourcePlatform::MacOS: - case SourcePlatform::Linux: - OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute)); - break; - default: - throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform)); - } - } - OptionalStructuredOutput->EndObject(); - } - else - { - ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); - } - } - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // "files" - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndObject(); - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // parts - } - } - } - - void DiffFolders(TransferThreadWorkers& Workers, - const std::filesystem::path& BasePath, - const std::filesystem::path& ComparePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const std::vector<std::string>& ExcludeFolders, - const std::vector<std::string>& ExcludeExtensions) - { - ZEN_TRACE_CPU("DiffFolders"); - - ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders"); - - enum TaskSteps : uint32_t - { - CheckBase, - CheckCompare, - Diff, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - ChunkedFolderContent BaseFolderContent; - ChunkedFolderContent CompareFolderContent; - - { - auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { - for (const std::string& ExcludeExtension : ExcludeExtensions) - { - if (RelativePath.ends_with(ExcludeExtension)) - { - return false; - } - } - return true; - }; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount); - - GetFolderContentStatistics BaseGetFolderContentStats; - ChunkingStatistics BaseChunkingStats; - BaseFolderContent = ScanAndChunkFolder(Workers, - BaseGetFolderContentStats, - BaseChunkingStats, - BasePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - if (AbortFlag) - { - return; - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount); - - GetFolderContentStatistics CompareGetFolderContentStats; - ChunkingStatistics CompareChunkingStats; - CompareFolderContent = ScanAndChunkFolder(Workers, - CompareGetFolderContentStats, - CompareChunkingStats, - ComparePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - - if (AbortFlag) - { - return; - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount); - - std::vector<IoHash> AddedHashes; - std::vector<IoHash> RemovedHashes; - uint64_t RemovedSize = 0; - uint64_t AddedSize = 0; - - tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BaseRawHashLookup; - for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - BaseRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CompareRawHashLookup; - for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex]; - if (!BaseRawHashLookup.contains(RawHash)) - { - AddedHashes.push_back(RawHash); - AddedSize += CompareFolderContent.RawSizes[PathIndex]; - } - CompareRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - if (!CompareRawHashLookup.contains(RawHash)) - { - RemovedHashes.push_back(RawHash); - RemovedSize += BaseFolderContent.RawSizes[PathIndex]; - } - } - - uint64_t BaseTotalRawSize = 0; - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) - { - BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex]; - } - - double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0; - - ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept", - RemovedHashes.size(), - NiceBytes(RemovedSize), - AddedHashes.size(), - NiceBytes(AddedSize), - BaseFolderContent.Paths.size() - RemovedHashes.size(), - NiceBytes(BaseTotalRawSize - RemovedSize), - KeptPercent); - - uint64_t CompareTotalRawSize = 0; - - uint64_t FoundChunkCount = 0; - uint64_t FoundChunkSize = 0; - uint64_t NewChunkCount = 0; - uint64_t NewChunkSize = 0; - const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent); - for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) - { - const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex]; - if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash)) - { - FoundChunkCount++; - FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - else - { - NewChunkCount++; - NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(ResponseView, SB); + WriteFile(Path, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - - double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0; - double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0; - - ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", - FoundChunkCount, - NiceBytes(FoundChunkSize), - FoundPercent, - CompareFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(CompareTotalRawSize), - BaseFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(BaseTotalRawSize), - NewChunkCount, - NiceBytes(NewChunkSize), - NewPercent); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } } // namespace builds_impl -////////////////////////////////////////////////////////////////////////////////////////////////////// -// BuildsCommand — Option-adding helpers -// +////////////////////////////////////////////////////////////////////////// void -BuildsCommand::AddSystemOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddSystemOptions(cxxopts::Options& Ops) { - Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>"); + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(SystemRootDir), "<systemdir>"); Ops.add_option("", "", "use-sparse-files", "Enable use of sparse files when writing large files. Defaults to true.", - cxxopts::value(m_UseSparseFiles), + cxxopts::value(UseSparseFiles), "<usesparsefiles>"); } void -BuildsCommand::AddCloudOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCloudOptions(cxxopts::Options& Ops) { - m_AuthOptions.AddOptions(Ops); + AuthOptions.AddOptions(Ops); - Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>"); - Ops.add_option("cloud build", - "", - "url", - "Cloud Builds host url (legacy - use --override-host)", - cxxopts::value(m_OverrideHost), - "<url>"); - Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>"); - Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>"); + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(OverrideHost), "<override-host>"); + Ops.add_option("cloud build", "", "url", "Cloud Builds host url (legacy - use --override-host)", cxxopts::value(OverrideHost), "<url>"); + Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(Url), "<cloud-url>"); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(Host), "<host>"); Ops.add_option("cloud build", "", "assume-http2", "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", - cxxopts::value(m_AssumeHttp2), + cxxopts::value(AssumeHttp2), "<assumehttp2>"); Ops.add_option("cloud build", "", "verbose-http", "Enable verbose option for http client", - cxxopts::value(m_VerboseHttp), + cxxopts::value(VerboseHttp), "<verbosehttp>"); - Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>"); - Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>"); - Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(m_AllowRedirect), "<allow-redirect>"); + Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(Namespace), "<namespace>"); + Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(Bucket), "<bucket>"); + Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(AllowRedirect), "<allow-redirect>"); } void -BuildsCommand::AddFileOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddFileOptions(cxxopts::Options& Ops) { - Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>"); + Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(StoragePath), "<storagepath>"); Ops.add_option("filestorage", "", "json-metadata", "Write build, part and block metadata as .json files in addition to .cb files", - cxxopts::value(m_WriteMetadataAsJson), + cxxopts::value(WriteMetadataAsJson), "<jsonmetadata>"); } void -BuildsCommand::AddCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCacheOptions(cxxopts::Options& Ops) { - Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>"); + Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(ZenCacheHost), "<zenhost>"); } void -BuildsCommand::AddOutputOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddOutputOptions(cxxopts::Options& Ops) { - Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<plainprogress>"); - Ops.add_option("output", - "", - "log-progress", - "Write @progress style progress to output", - cxxopts::value(m_LogProgress), - "<logprogress>"); - Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); - Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>"); + Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(PlainProgress), "<plainprogress>"); + Ops.add_option("output", "", "log-progress", "Write @progress style progress to output", cxxopts::value(LogProgress), "<logprogress>"); + Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(Verbose), "<verbose>"); + Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(Quiet), "<quiet>"); } void -BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWorkerOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "boost-worker-count", "Increase the number of worker threads - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkerCount), + cxxopts::value(BoostWorkerCount), "<boostworkercount>"); Ops.add_option("", @@ -2106,47 +298,47 @@ BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) "boost-worker-memory", "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to " "be less responsive due to high memory usage", - cxxopts::value(m_BoostWorkerMemory), + cxxopts::value(BoostWorkerMemory), "<boostworkermemory>"); Ops.add_option("", "", "boost-workers", "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkers), + cxxopts::value(BoostWorkers), "<boostworkermemory>"); } void -BuildsCommand::AddZenFolderOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddZenFolderOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "zen-folder-path", - fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", builds_impl::ZenFolderName), - cxxopts::value(m_ZenFolderPath), - "<boostworkers>"); + "Path to the zen state and temp folder used by this command. Overrides the command-specific default.", + cxxopts::value(ZenFolderPath), + "<path>"); } void -BuildsCommand::AddChunkingCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddChunkingCacheOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "chunking-cache-path", "Path to cache for chunking information of scanned files. Default is empty resulting in no caching", - cxxopts::value(m_ChunkingCachePath), + cxxopts::value(ChunkingCachePath), "<chunkingcachepath>"); } void -BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWildcardOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;", - cxxopts::value(m_IncludeWildcard), + cxxopts::value(IncludeWildcard), "<wildcard>"); Ops.add_option("", @@ -2154,46 +346,46 @@ BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) "exclude-wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard " "include filter", - cxxopts::value(m_ExcludeWildcard), + cxxopts::value(ExcludeWildcard), "<excludewildcard>"); } void -BuildsCommand::AddExcludeFolderOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeFolderOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-folders", "Names of folders to exclude, separated by ;", - cxxopts::value(m_ExcludeFolders), + cxxopts::value(ExcludeFolders), "<excludefolders>"); } void -BuildsCommand::AddExcludeExtensionsOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeExtensionsOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-extensions", "Extensions to exclude, separated by ;" "include filter", - cxxopts::value(m_ExcludeExtensions), + cxxopts::value(ExcludeExtensions), "<excludeextensions>"); } void -BuildsCommand::AddMultipartOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddMultipartOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "allow-multipart", "Allow large attachments to be transfered using multipart protocol. Defaults to true.", - cxxopts::value(m_AllowMultiparts), + cxxopts::value(AllowMultiparts), "<allowmultipart>"); } void -BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddPartialBlockRequestOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2206,12 +398,12 @@ BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) "allowed to host\n" " true = multiple partial block ranges requests per block allowed to zen cache and host\n" "Defaults to 'mixed'.", - cxxopts::value(m_AllowPartialBlockRequests), + cxxopts::value(AllowPartialBlockRequests), "<allowpartialblockrequests>"); } void -BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddAppendNewContentOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2220,26 +412,26 @@ BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) " false = the local content will be replaced by the remote content\n" " true = the remote data will be overlayed on top of local data\n" "Defaults to false.", - cxxopts::value(m_AppendNewContent), + cxxopts::value(AppendNewContent), "<append>"); } BuildsCommand::BuildsCommand() -: m_ListNamespacesSubCmd(*this) -, m_ListSubCmd(*this) -, m_ListBlocksSubCmd(*this) -, m_UploadSubCmd(*this) -, m_DownloadSubCmd(*this) -, m_LsSubCmd(*this) -, m_DiffSubCmd(*this) -, m_FetchBlobSubCmd(*this) -, m_PrimeCacheSubCmd(*this) -, m_PauseSubCmd(*this) -, m_ResumeSubCmd(*this) -, m_AbortSubCmd(*this) -, m_ValidatePartSubCmd(*this) -, m_TestSubCmd(*this) -, m_MultiTestDownloadSubCmd(*this) +: m_ListNamespacesSubCmd(m_Configuration) +, m_ListSubCmd(m_Configuration) +, m_ListBlocksSubCmd(m_Configuration) +, m_UploadSubCmd(m_Configuration) +, m_DownloadSubCmd(m_Configuration) +, m_LsSubCmd(m_Configuration) +, m_DiffSubCmd(m_Configuration) +, m_FetchBlobSubCmd(m_Configuration) +, m_PrimeCacheSubCmd(m_Configuration) +, m_PauseSubCmd(m_Configuration) +, m_ResumeSubCmd(m_Configuration) +, m_AbortSubCmd(m_Configuration) +, m_ValidatePartSubCmd(m_Configuration) +, m_TestSubCmd(m_Configuration) +, m_MultiTestDownloadSubCmd(m_Configuration) { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), ""); @@ -2269,174 +461,250 @@ BuildsCommand::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/) { using namespace builds_impl; - signal(SIGINT, SignalCallbackHandler); + // Held in member variables so the handlers stay installed through the + // subcommand's Run() and are restored when BuildsCommand is destroyed. + m_SigIntGuard.emplace(SIGINT, SignalCallbackHandler); #if ZEN_PLATFORM_WINDOWS - signal(SIGBREAK, SignalCallbackHandler); -#endif // ZEN_PLATFORM_WINDOWS + m_SigBreakGuard.emplace(SIGBREAK, SignalCallbackHandler); +#endif // Validate output options - if (m_Verbose && m_Quiet) + if (m_Configuration.Verbose && m_Configuration.Quiet) { throw OptionParseException("'--verbose' conflicts with '--quiet'", {}); } - if (m_LogProgress && m_PlainProgress) + if (m_Configuration.LogProgress && m_Configuration.PlainProgress) { throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", {}); } - if (m_LogProgress && m_Quiet) + if (m_Configuration.LogProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--log-progress'", {}); } - if (m_PlainProgress && m_Quiet) + if (m_Configuration.PlainProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--plain-progress'", {}); } - IsVerbose = m_Verbose; - IsQuiet = m_Quiet; - if (m_LogProgress) - { - ProgressMode = ProgressBar::Mode::Log; - } - else if (m_PlainProgress) + if (m_Configuration.LogProgress) { - ProgressMode = ProgressBar::Mode::Plain; + m_Configuration.ProgressMode = ConsoleProgressMode::Log; } - else if (m_Verbose) + else if (m_Configuration.PlainProgress) { - ProgressMode = ProgressBar::Mode::Plain; + m_Configuration.ProgressMode = ConsoleProgressMode::Plain; } - else if (IsQuiet) + else if (m_Configuration.Quiet) { - ProgressMode = ProgressBar::Mode::Quiet; + m_Configuration.ProgressMode = ConsoleProgressMode::Quiet; } else { - ProgressMode = ProgressBar::Mode::Pretty; + m_Configuration.ProgressMode = ConsoleProgressMode::Pretty; } - if (m_BoostWorkers) + if (m_Configuration.BoostWorkers) { - m_BoostWorkerCount = true; - m_BoostWorkerMemory = true; + m_Configuration.BoostWorkerCount = true; + m_Configuration.BoostWorkerMemory = true; } // Parse system options - if (m_SystemRootDir.empty()) + if (m_Configuration.SystemRootDir.empty()) { - m_SystemRootDir = PickDefaultSystemRootDirectory(); + m_Configuration.SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathInPlace(m_SystemRootDir); - - UseSparseFiles = m_UseSparseFiles; + MakeSafeAbsolutePathInPlace(m_Configuration.SystemRootDir); + MakeSafeAbsolutePathInPlace(m_Configuration.ChunkingCachePath); return true; } +std::atomic<bool>& +BuildsSubCmdBase::AbortFlag() const +{ + return builds_impl::AbortFlag; +} + +std::atomic<bool>& +BuildsSubCmdBase::PauseFlag() const +{ + return builds_impl::PauseFlag; +} + +std::unique_ptr<ProgressBase> +BuildsSubCmdBase::CreateProgress() const +{ + return std::unique_ptr<ProgressBase>(CreateConsoleProgress(m_Config.ProgressMode)); +} + +////////////////////////////////////////////////////////////////////////// + +void +BuildsSubCmdBase::LogBanner() +{ + if (!m_Config.Quiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); + } +} + void -BuildsCommand::ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts) +BuildsSubCmdBase::LogWorkersInfo(const TransferThreadWorkers& Workers) { - if (!m_Url.empty()) + if (!m_Config.Quiet) { - if (!m_Host.empty()) + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } +} + +void +BuildsSubCmdBase::EnsureZenFolderExists() +{ + m_CreatedZenFolder = CreateDirectories(GetZenFolderPath()); +} + +void +BuildsSubCmdBase::CleanZenFolder() +{ + using namespace builds_impl; + WorkerThreadPool& Pool = GetSmallWorkerPool(EWorkloadType::Burst); + if (m_CreatedZenFolder) + { + CleanAndRemoveDirectory(Pool, AbortFlag(), PauseFlag(), GetZenFolderPath()); + } + else + { + CleanAndRemoveDirectory(Pool, AbortFlag(), PauseFlag(), ZenTempFolderPath(GetZenFolderPath())); + } +} + +////////////////////////////////////////////////////////////////////////// + +BuildsSubCmdBase::ResolvedStorage +BuildsSubCmdBase::ParseStorageOptions(std::string& BuildId, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + const std::filesystem::path& SystemRootDirOverride, + const std::filesystem::path& StoragePathOverride) +{ + ResolvedStorage Resolved{.SystemRootDir = SystemRootDirOverride.empty() ? m_Config.SystemRootDir : SystemRootDirOverride, + .Host = m_Config.Host, + .Namespace = m_Config.Namespace, + .Bucket = m_Config.Bucket, + .StoragePath = StoragePathOverride.empty() ? m_Config.StoragePath : StoragePathOverride}; + + if (!m_Config.Url.empty()) + { + if (!Resolved.Host.empty()) { - throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", Resolved.Host, m_Config.Url), + SubOpts.help()); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", Resolved.Bucket, m_Config.Url), + SubOpts.help()); } if (!BuildId.empty()) { - throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Config.Url), + SubOpts.help()); } - if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, BuildId)) + if (!ParseBuildStorageUrl(m_Config.Url, Resolved.Host, Resolved.Namespace, Resolved.Bucket, BuildId)) { - throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOpts.help()); + throw OptionParseException( + fmt::format("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", m_Config.Url), + SubOpts.help()); } } - if (!m_OverrideHost.empty() || !m_Host.empty()) + if (!m_Config.OverrideHost.empty() || !Resolved.Host.empty()) { - if (!m_StoragePath.empty()) + if (!Resolved.StoragePath.empty()) { throw OptionParseException( - fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath), + fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", Resolved.StoragePath), SubOpts.help()); } - if (RequireNamespace && m_Namespace.empty()) + if (Options.RequireNamespace && Resolved.Namespace.empty()) { throw OptionParseException("'--namespace' is required", SubOpts.help()); } - if (RequireBucket && m_Bucket.empty()) + if (Options.RequireBucket && Resolved.Bucket.empty()) { throw OptionParseException("'--bucket' is required", SubOpts.help()); } } - else if (m_StoragePath.empty()) + else if (Resolved.StoragePath.empty()) { throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - MakeSafeAbsolutePathInPlace(m_StoragePath); + MakeSafeAbsolutePathInPlace(Resolved.StoragePath); + return Resolved; } StorageInstance -BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, - BuildStorageCache::Statistics& StorageCacheStats, - const std::filesystem::path& TempPath, - std::string& BuildId, - bool RequireNamespace, - bool RequireBucket, - bool BoostCacheBackgroundWorkerPool, - std::unique_ptr<AuthMgr>& Auth, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::CreateBuildStorage(const std::filesystem::path& ZenFolder, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + BuildStorageBase::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + std::unique_ptr<AuthMgr>& Auth, + const ResolvedStorage& Resolved) { using namespace builds_impl; - ParseStorageOptions(BuildId, RequireNamespace, RequireBucket, SubOpts); + // Empty ZenFolder => operate without a temp directory and keep all downloads in memory. + const bool HasZenFolder = !ZenFolder.empty(); + const std::filesystem::path StorageTempPath = HasZenFolder ? ZenTempFolderPath(ZenFolder) / "storage" : std::filesystem::path{}; + const std::filesystem::path CacheTempPath = HasZenFolder ? ZenTempFolderPath(ZenFolder) / "zencache" : std::filesystem::path{}; - HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 2, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; + HttpClientSettings ClientSettings{ + .LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_Config.AssumeHttp2, + .AllowResume = true, + .RetryCount = 2, + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = HasZenFolder ? GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory) + : std::numeric_limits<uint64_t>::max()}; std::string StorageDescription; std::string CacheDescription; StorageInstance Result; - if (!m_Host.empty() || !m_OverrideHost.empty()) + if (!Resolved.Host.empty() || !m_Config.OverrideHost.empty()) { - m_AuthOptions.ParseOptions(SubOpts, - m_SystemRootDir, - ClientSettings, - m_Host.empty() ? m_OverrideHost : m_Host, - Auth, - IsQuiet, - /*Hidden*/ false, - m_Verbose); - - BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*CreateConsoleLogOutput(ProgressMode), + AuthCommandLineOptions AuthOpts = m_Config.AuthOptions; + AuthOpts.ParseOptions(SubOpts, + Resolved.SystemRootDir, + ClientSettings, + Resolved.Host.empty() ? m_Config.OverrideHost : Resolved.Host, + Auth, + m_Config.Quiet, + /*Hidden*/ false, + m_Config.Verbose); + + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(), ClientSettings, - m_Host, - m_OverrideHost, - m_ZenCacheHost, + Resolved.Host, + m_Config.OverrideHost, + m_Config.ZenCacheHost, ZenCacheResolveMode::All, - m_Verbose); + m_Config.Verbose); if (!ResolveRes.Cloud.Address.empty()) { ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; Result.BuildStorageHttp = - std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); + std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, [this]() { return AbortFlag().load(); }); Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, - m_Namespace, - m_Bucket, - m_AllowRedirect, - TempPath / "storage"); + Resolved.Namespace, + Resolved.Bucket, + m_Config.AllowRedirect, + StorageTempPath); Result.BuildStorageHost = ResolveRes.Cloud; uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0; @@ -2446,8 +714,8 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, ResolveRes.Cloud.Name, (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket, + Resolved.Namespace, + Resolved.Bucket, NiceLatencyNs(HostLatencyNs)); if (!ResolveRes.Cache.Address.empty()) @@ -2461,19 +729,21 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); + Resolved.Namespace, + Resolved.Bucket, + CacheTempPath, + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); Result.CacheHost = ResolveRes.Cache; + Result.SetupCacheSession(ResolveRes.Cache.Address, fmt::format("builds {}", m_SubOptions.program()), GetSessionId()); + uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; CacheDescription = @@ -2483,69 +753,71 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, Result.CacheHttp->GetSessionId(), NiceLatencyNs(CacheLatencyNs)); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } } - else if (!m_StoragePath.empty()) + else if (!Resolved.StoragePath.empty()) { - StorageDescription = fmt::format("folder {}", m_StoragePath); - Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageDescription = fmt::format("folder {}", Resolved.StoragePath); + Result.BuildStorage = CreateFileBuildStorage(Resolved.StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), + Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = Resolved.StoragePath.generic_string(), .Name = "Disk", .LatencySec = 1.0 / 100000, // 1 us .Caps = {.MaxRangeCountPerRequest = 2048u}}; - if (!m_ZenCacheHost.empty()) + if (!m_Config.ZenCacheHost.empty()) { - ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); + ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_Config.ZenCacheHost, m_Config.AssumeHttp2, m_Config.VerboseHttp); if (TestResult.Success) { Result.CacheHttp = std::make_unique<HttpClient>( - m_ZenCacheHost, + m_Config.ZenCacheHost, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = m_AssumeHttp2, + .AssumeHttp2 = m_Config.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, - .Name = m_ZenCacheHost, - .AssumeHttp2 = m_AssumeHttp2, + Resolved.Namespace, + Resolved.Bucket, + CacheTempPath, + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_Config.ZenCacheHost, + .Name = m_Config.ZenCacheHost, + .AssumeHttp2 = m_Config.AssumeHttp2, .LatencySec = TestResult.LatencySeconds, .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; + Result.SetupCacheSession(m_Config.ZenCacheHost, fmt::format("builds {}", m_SubOptions.program()), GetSessionId()); + CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } @@ -2555,7 +827,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Remote: {}", StorageDescription); if (!Result.CacheHost.Name.empty()) @@ -2567,45 +839,33 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, } Oid -BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) { - if (BuildIdStr.length() != Oid::StringLength) + const Oid BuildId = Oid::TryFromHexString(BuildIdStr); + if (BuildId == Oid::Zero) { throw OptionParseException( - fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildIdStr, Oid::StringLength), + fmt::format("'--build-id' ('{}') is malformed, expected {} hex characters", BuildIdStr, Oid::StringLength), SubOpts.help()); } - else if (Oid BuildId = Oid::FromHexString(BuildIdStr); BuildId == Oid::Zero) - { - throw OptionParseException(fmt::format("'--build-id' ('{}') is invalid", BuildIdStr), SubOpts.help()); - } - else - { - return BuildId; - } + return BuildId; } Oid -BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) { - if (BuildPartIdStr.length() != Oid::StringLength) + const Oid BuildPartId = Oid::TryFromHexString(BuildPartIdStr); + if (BuildPartId == Oid::Zero) { throw OptionParseException( - fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildPartIdStr, Oid::StringLength), + fmt::format("'--build-part-id' ('{}') is malformed, expected {} hex characters", BuildPartIdStr, Oid::StringLength), SubOpts.help()); } - else if (Oid BuildPartId = Oid::FromHexString(BuildPartIdStr); BuildPartId == Oid::Zero) - { - throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildPartIdStr), SubOpts.help()); - } - else - { - return BuildPartId; - } + return BuildPartId; } std::vector<Oid> -BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts) { std::vector<Oid> BuildPartIds; for (const std::string& BuildPartId : BuildPartIdStrs) @@ -2620,7 +880,7 @@ BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs } std::vector<std::string> -BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts) { std::vector<std::string> BuildPartNames; for (const std::string& BuildPartName : BuildPartNameStrs) @@ -2635,10 +895,10 @@ BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartName } CbObject -BuildsCommand::ParseBuildMetadata(bool CreateBuild, - std::filesystem::path& BuildMetadataPath, - const std::string& BuildMetadata, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts) { if (CreateBuild) { @@ -2697,7 +957,7 @@ BuildsCommand::ParseBuildMetadata(bool CreateBuild, } void -BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) { if (Path.empty()) { @@ -2707,7 +967,7 @@ BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) } IoHash -BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) { if (BlobHashStr.empty()) { @@ -2731,7 +991,7 @@ BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& S } void -BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) +BuildsSubCmdBase::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) { auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) { ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) { @@ -2757,12 +1017,13 @@ BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, s }); }; - SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards); - SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards); + SplitAndAppendWildcard(m_Config.IncludeWildcard, OutIncludeWildcards); + SplitAndAppendWildcard(m_Config.ExcludeWildcard, OutExcludeWildcards); } void -BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) +BuildsSubCmdBase::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, + std::vector<std::string>& OutExcludeExtensions) { auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) { ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) { @@ -2779,38 +1040,50 @@ BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExclu }); }; - SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders); - SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions); + SplitAndAppendExclusion(m_Config.ExcludeFolders, OutExcludeFolders); + SplitAndAppendExclusion(m_Config.ExcludeExtensions, OutExcludeExtensions); +} + +void +BuildsSubCmdBase::SetZenFolderPath(const std::filesystem::path& Fallback) +{ + m_ResolvedZenFolderPath = m_Config.ZenFolderPath.empty() ? Fallback : m_Config.ZenFolderPath; + MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath); } void -BuildsCommand::ResolveZenFolderPath(const std::filesystem::path& DefaultPath) +BuildsSubCmdBase::ResolveZenFolderPath(const std::filesystem::path& LocalPath) { - if (m_ZenFolderPath.empty()) + using namespace builds_impl; + if (!m_Config.ZenFolderPath.empty()) + { + m_ResolvedZenFolderPath = m_Config.ZenFolderPath; + } + else if (!LocalPath.empty()) + { + m_ResolvedZenFolderPath = LocalPath / ZenFolderName; + } + else { - m_ZenFolderPath = DefaultPath; + m_ResolvedZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath); } EPartialBlockRequestMode -BuildsCommand::ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts) { - if (PrimeCacheOnly) - { - return EPartialBlockRequestMode::Off; - } - EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests); + EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_Config.AllowPartialBlockRequests); if (Mode == EPartialBlockRequestMode::Invalid) { - throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests), + throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_Config.AllowPartialBlockRequests), SubOpts.help()); } return Mode; } void -BuildsCommand::ParseZenProcessId(int& ZenProcessId) +BuildsSubCmdBase::ParseZenProcessId(int& ZenProcessId) { if (ZenProcessId == -1) { @@ -2831,20 +1104,15 @@ BuildsCommand::ParseZenProcessId(int& ZenProcessId) ////////////////////////////////////////////////////////////////////////// -// --------------------------------------------------------------------------- -// Subcommand implementations -// --------------------------------------------------------------------------- - -BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-namespaces", "List all namespaces and optionally their buckets") -, m_Parent(Parent) +BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-namespaces", "List all namespaces and optionally their buckets") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "recursive", "Enable fetch of buckets within namespaces also", cxxopts::value(m_Recursive), "<recursive>"); Opts.add_option("", "", @@ -2859,36 +1127,20 @@ BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) void BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); + cxxopts::Options& Opts = SubOptions(); - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ false, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + std::string DummyBuildId; + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireNamespace = false, .RequireBucket = false}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); CbObject Response = Storage.BuildStorage->ListNamespaces(m_Recursive); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -2900,29 +1152,20 @@ BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - std::filesystem::path ResultPath = MakeSafeAbsolutePath(m_ResultPath); - if (ToLower(ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(MakeSafeAbsolutePath(m_ResultPath), Response); } } -BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list", "List builds matching a query"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListSubCmd::BuildsListSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "list", "List builds matching a query") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "query-path", @@ -2942,19 +1185,16 @@ BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list" void BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + if (!m_ResultPath.empty()) + { + LogBanner(); + } + + cxxopts::Options& Opts = SubOptions(); MakeSafeAbsolutePathInPlace(m_QueryPath); MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) - { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - } std::string JsonQuery; if (m_QueryPath.empty()) { @@ -2991,25 +1231,13 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } } + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireBucket = false}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -3021,28 +1249,20 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-blocks", "List blocks for a build") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-blocks", "List blocks for a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", @@ -3058,50 +1278,34 @@ BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) void BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } + cxxopts::Options& Opts = SubOptions(); + + MakeSafeAbsolutePathInPlace(m_ResultPath); + if (m_MaxCount == 0) { throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_MaxCount), Opts.help()); } BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_MaxCount); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response); - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size()); } @@ -3116,35 +1320,25 @@ BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("upload", "Upload a folder to build storage") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "upload", "Upload a folder to build storage") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", @@ -3192,7 +1386,7 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) cxxopts::value(m_UploadToZenCache), "<uploadtozencache>"); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); Opts.add_option("", "", @@ -3217,50 +1411,31 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) void BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - ZenState InstanceState; + ParsePath(m_Path, Opts); - m_Parent.ParsePath(m_Path, Opts); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); if (m_BuildPartName.empty() && m_ManifestPath.empty()) { m_BuildPartName = m_Path.filename().string(); } - const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(m_BuildId, Opts); if (m_BuildId.empty()) { m_BuildId = BuildId.ToString(); @@ -3269,28 +1444,31 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Oid BuildPartId; if (!m_BuildPartId.empty()) { - BuildPartId = m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + BuildPartId = ParseBuildPartId(m_BuildPartId, Opts); } - CbObject MetaData = m_Parent.ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); + CbObject MetaData = ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); - const std::filesystem::path TempDir = ZenTempFolderPath(m_Parent.GetZenFolderPath()); + const std::filesystem::path TempDir = ZenTempFolderPath(GetZenFolderPath()); std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); std::vector<std::pair<Oid, std::string>> UploadedParts = - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -3302,89 +1480,89 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = TempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = m_CreateBuild, .IgnoreExistingBlocks = m_Clean, .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, .ExcludeFolders = ExcludeFolders, .ExcludeExtensions = ExcludeExtensions}); - if (!AbortFlag) + if (!AbortFlag()) { if (m_PostUploadVerify) { for (const auto& Part : UploadedParts) { - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + Part.first, + Part.second); } } } - if (true) + if (!m_Config.Quiet) { - if (!IsQuiet) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.BuildStorageHost.Name, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } + const BuildStorageResolveResult::Host& Host = Storage.BuildStorageHost; + const BuildStorageBase::Statistics& Stats = StorageStats; + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + Host.Name, + NiceBytes(Stats.TotalBytesRead.load()), + NiceBytes(Stats.TotalBytesWritten.load()), + Stats.TotalRequestCount.load(), + Stats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(Stats.TotalExecutionTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0, + Stats.TotalRequestCount.load() > 0 ? NiceTimeSpanMs(Stats.TotalRequestTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Upload aborted"); } } -BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("download", "Download a build to a local folder") -, m_Parent(Parent) -{ - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddExcludeFolderOption(Opts); +////////////////////////////////////////////////////////////////////////// - Opts.add_option("cache", - "", - "cache-prime-only", - "Only download blobs missing in cache and upload to cache", - cxxopts::value(m_PrimeCacheOnly), - "<cacheprimeonly>"); +BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "download", "Download a build to a local folder") +{ + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddExcludeFolderOption(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - Opts.add_option("", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); + Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>"); Opts.add_option("", "", "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", + "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, " + "the part named 'default' is selected. Use '*' (alone) to select all parts.", cxxopts::value(m_BuildPartNames), "<name>"); Opts.add_option("", @@ -3405,9 +1583,9 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) "Upload data downloaded from remote host to zen cache", cxxopts::value(m_UploadToZenCache), "<uploadtozencache>"); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); Opts.add_option( "", @@ -3437,139 +1615,107 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) void BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - ZenState InstanceState; - - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ m_PrimeCacheOnly, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); - - if (m_PostDownloadVerify && m_PrimeCacheOnly) - { - throw OptionParseException("'--cache-prime-only' conflicts with '--verify'", Opts.help()); - } + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + // Preserve zen folder on exit - <zen>/current_state.cbo tracks what has been downloaded into + // this folder and is consulted by later download operations against the same target. + ResolveZenFolderPath(m_Path); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); - if (m_Clean && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled"); - } - - if (m_Force && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled"); - } - - if (m_Parent.m_AllowPartialBlockRequests != "false" && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled"); - } + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); + NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help()); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(m_PrimeCacheOnly, Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); - if (m_Parent.m_AppendNewContent && m_Clean) + if (m_Config.AppendNewContent && m_Clean) { throw OptionParseException("'--append' conflicts with '--clean'", Opts.help()); } std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); DownloadFolder( - *Output, + ConsoleLog(), + *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, BuildPartIds, BuildPartNames, m_DownloadSpecPath, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_Config.SystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = m_Clean, .PostDownloadVerify = m_PostDownloadVerify, - .PrimeCacheOnly = m_PrimeCacheOnly, .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force, .EnableTargetFolderScavenging = !m_Force, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Parent.m_BoostWorkerMemory), + .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory), .PopulateCache = m_UploadToZenCache, - .AppendNewContent = m_Parent.m_AppendNewContent, + .AppendNewContent = m_Config.AppendNewContent, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles, .ExcludeFolders = ExcludeFolders}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Download aborted"); } } -BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "List files in a build"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsLsSubCmd::BuildsLsSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "ls", "List files in a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - Opts.add_option("", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); + Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>"); Opts.add_option("", "", "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", + "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, " + "the part named 'default' is selected. Use '*' (alone) to select all parts.", cxxopts::value(m_BuildPartNames), "<name>"); @@ -3594,43 +1740,31 @@ BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "Lis void BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); + + builds_impl::ZenState InstanceState; std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); - - m_Parent.ResolveZenFolderPath(m_Parent.m_StoragePath); // ls uses storage path context + ParseFileFilters(IncludeWildcards, ExcludeWildcards); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); + NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help()); std::unique_ptr<CbObjectWriter> StructuredOutput; if (!m_ResultPath.empty()) @@ -3639,38 +1773,30 @@ BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) StructuredOutput = std::make_unique<CbObjectWriter>(); } - ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); + ListBuild(m_Config.Quiet, Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); if (StructuredOutput) { CbObject Response = StructuredOutput->Save(); - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("List build aborted"); } } -BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff", "Diff two local folders"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "diff", "Diff two local folders") { - auto& Opts = SubOptions(); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>"); Opts.add_option("", @@ -3686,38 +1812,28 @@ BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff" void BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); if (m_DiffPath.empty()) { throw OptionParseException("'--compare-path' is required", Opts.help()); } MakeSafeAbsolutePathInPlace(m_DiffPath); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); StandardChunkingControllerSettings ChunkingSettings; std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); if (m_OnlyChunked) { @@ -3729,24 +1845,37 @@ BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ChunkingSettings.SplitAndCompressExtensions.end()); } - DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions); - if (AbortFlag) + std::unique_ptr<ProgressBase> Progress = CreateProgress(); + + DiffFolders(*Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + Workers, + m_Path, + m_DiffPath, + *ChunkController, + *ChunkCache, + ExcludeFolders, + ExcludeExtensions); + if (AbortFlag()) { throw std::runtime_error("Diff folders aborted"); } } -BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("fetch-blob", "Fetch and validate a specific blob") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "fetch-blob", "Fetch and validate a specific blob") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>"); Opts.parse_positional({"build-id", "blob-hash"}); @@ -3756,80 +1885,55 @@ BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) void BuildsFetchBlobSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - IoHash BlobHash = m_Parent.ParseBlobHash(m_BlobHash, Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - const Oid BuildId = Oid::FromHexString(m_BuildId); + IoHash BlobHash = ParseBlobHash(m_BlobHash, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); - if (AbortFlag) + ValidateBlob(AbortFlag(), *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); + if (AbortFlag()) { throw std::runtime_error("Fetch blob aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize); } } -BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("prime-cache", "Prime the zen cache with build data") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "prime-cache", "Prime the zen cache with build data") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - Opts.add_option("", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); + Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>"); Opts.add_option("", "", "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", + "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, " + "the part named 'default' is selected. Use '*' (alone) to select all parts.", cxxopts::value(m_BuildPartNames), "<name>"); Opts.add_option("", @@ -3845,47 +1949,31 @@ BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) void BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = + CreateBuildStorage(GetZenFolderPath(), {.BoostCacheBackgroundWorkers = true}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ true, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); - - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); + NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help()); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); + CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, m_Config.Quiet); std::vector<std::pair<Oid, std::string>> AllBuildParts = ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); @@ -3897,100 +1985,101 @@ BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) AllBuildPartIds.push_back(BuildPart.first); } - ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache"); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); + Progress->SetLogOperationName("Prime Cache"); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - - BuildsOperationPrimeCache PrimeOp(*Output, + BuildsOperationPrimeCache PrimeOp(ConsoleLog(), + *Progress, Storage, - AbortFlag, - PauseFlag, + AbortFlag(), + PauseFlag(), Workers.GetNetworkPool(), BuildId, AllBuildPartIds, - BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .ZenFolderPath = m_Parent.GetZenFolderPath(), + BuildsOperationPrimeCache::Options{.IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .ZenFolderPath = GetZenFolderPath(), .LargeAttachmentSize = PreferredMultipartChunkSize * 4u, .PreferredMultipartChunkSize = PreferredMultipartChunkSize, .ForceUpload = m_Force}, - StorageCacheStats); + CacheStats); PrimeOp.Execute(); - if (!IsQuiet) + if (!m_Config.Quiet && Storage.CacheStorage) { - if (Storage.CacheStorage) - { - ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", - StorageCacheStats.PutBlobCount.load(), - NiceBytes(StorageCacheStats.PutBlobByteCount), - Storage.CacheHost.Name); - } + ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", + CacheStats.PutBlobCount.load(), + NiceBytes(CacheStats.PutBlobByteCount), + Storage.CacheHost.Name); } } -BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("pause", "Pause a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +namespace { + void AddProcessIdOption(cxxopts::Options& Opts, int& ZenProcessId) + { + Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(ZenProcessId), "<pid>"); + Opts.parse_positional({"process-id"}); + Opts.positional_help("process-id"); + } +} // namespace + +BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "pause", "Pause a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsPauseSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(1); } -BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("resume", "Resume a paused zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "resume", "Resume a paused zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsResumeSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(false); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(0); } -BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("abort", "Abort a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "abort", "Abort a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsAbortSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Abort.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Abort.store(1); } -BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("validate-part", "Validate a build part") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "validate-part", "Validate a build part") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", @@ -4011,42 +2100,24 @@ BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) void BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + builds_impl::ZenState InstanceState; - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + Oid BuildId = ParseBuildId(m_BuildId, Opts); if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) { @@ -4055,33 +2126,46 @@ BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Opts.help()); } - const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(m_BuildPartId, Opts); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + BuildPartId, + m_BuildPartName); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Validate build part failed"); } } -BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test", "Run an upload/download test cycle"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsTestSubCmd::BuildsTestSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "test", "Run an upload/download test cycle") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); - Parent.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddChunkingCacheOptions(Opts); + Config.AddMultipartOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "", "enable-scavenge", @@ -4101,38 +2185,35 @@ BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test" void BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + const bool CreatedTempStorage = m_Config.OverrideHost.empty() && m_Config.StoragePath.empty(); + if (CreatedTempStorage) { - m_Parent.m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.m_StoragePath); - CreateDirectories(m_Parent.m_StoragePath); - m_Parent.m_StoragePath = m_Parent.m_StoragePath.generic_string(); + m_TestStoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), m_TestStoragePath); + CreateDirectories(m_TestStoragePath); + m_TestStoragePath = m_TestStoragePath.generic_string(); } - auto StorageGuard = MakeGuard([this]() { - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + auto StorageGuard = MakeGuard([this, CreatedTempStorage]() { + if (CreatedTempStorage) { - DeleteDirectories(m_Parent.m_StoragePath); + DeleteDirectories(m_TestStoragePath); } }); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -4143,37 +2224,35 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3"); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); - auto DownloadGuard = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() { - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + auto DownloadGuard = MakeGuard([this, &Workers, DownloadPath, DownloadPath2, DownloadPath3]() { + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); }); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::unique_ptr<AuthMgr> Auth; std::string TestBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - TestBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - m_BuildId = Oid::NewOid().ToString(); - m_BuildPartId = Oid::NewOid().ToString(); - m_CreateBuild = true; - - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + const ResolvedStorage Resolved = ParseStorageOptions(TestBuildId, {}, Opts, m_TestSystemRootDir, m_TestStoragePath); + // Place scratch next to the folder under test so upload does not self-include, and so the + // scratch area shares the volume of the source data. + { + const std::u8string PathStr = m_Path.generic_u8string(); + const IoHash PathHash = IoHash::HashBuffer(PathStr.data(), PathStr.length()); + SetZenFolderPath(m_Path.parent_path() / fmt::format("zen_{}", PathHash)); + } + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, StorageCacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto ZenGuard = MakeGuard([this]() { CleanZenFolder(); }); + + const Oid BuildId = Oid::NewOid(); + const Oid BuildPartId = Oid::NewOid(); + m_BuildId = BuildId.ToString(); + m_BuildPartId = BuildPartId.ToString(); + m_CreateBuild = true; auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -4196,18 +2275,21 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); } - const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); + const std::filesystem::path UploadTempDir = ZenTempFolderPath(GetZenFolderPath()); std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -4219,12 +2301,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload build)"); } @@ -4232,9 +2316,12 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path); - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), Oid::NewOid(), Oid::NewOid(), m_BuildPartName, @@ -4246,52 +2333,70 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload again, chunking is cached)"); } } - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + BuildPartId, + m_BuildPartName); - if (!m_Parent.m_IncludeWildcard.empty() || !m_Parent.m_ExcludeWildcard.empty()) + if (!m_Config.IncludeWildcard.empty() || !m_Config.ExcludeWildcard.empty()) { - auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); }); + auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); }); ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4301,66 +2406,79 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = ExcludeWildcards, .ExcludeWildcards = IncludeWildcards, - .AppendNewContent = true}); - if (AbortFlag) + .AppendNewContent = true, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = {}, .ExcludeWildcards = {}, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4368,25 +2486,30 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4394,17 +2517,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download identical target)"); } @@ -4436,7 +2561,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) return true; }; - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + ParallelWork Work(AbortFlag(), PauseFlag(), WorkerThreadPool::EMode::EnableBacklog); uint32_t Randomizer = 0; auto FileSizeIt = DownloadContent.FileSizes.begin(); @@ -4454,8 +2579,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { Work.ScheduleWork( Workers.GetIOWorkerPool(), - [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { - if (!AbortFlag) + [this, SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { + if (!AbortFlag()) { bool WasReadOnly = SetFileReadOnly(FilePath, false); { @@ -4483,7 +2608,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) case 1: { (void)SetFileReadOnly(FilePath, false); - (void)RemoveFile(FilePath); + std::error_code Ec; + const bool Removed = RemoveFile(FilePath, Ec); + if (!Removed || Ec) + { + throw zen::runtime_error("ScrambleDir: failed to delete '{}'. Reason: '{}'", + FilePath, + Ec ? Ec.message() : "file no longer present"); + } } break; default: @@ -4496,15 +2628,18 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_UNUSED(IsAborted, IsPaused); ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); }); - ZEN_ASSERT(!AbortFlag.load()); + ZEN_ASSERT(!AbortFlag().load()); ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }; ScrambleDir(DownloadPath); ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4512,17 +2647,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled target)"); } @@ -4539,9 +2676,12 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); } - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId2, BuildPartId2, m_BuildPartName, @@ -4553,22 +2693,38 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload scrambled)"); } - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + BuildPartId, + m_BuildPartName); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4576,133 +2732,156 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download scrambled)"); } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled)"); } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath2, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath2 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath3, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath3 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } } -BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("multi-test-download", "Download multiple builds sequentially as a test") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "multi-test-download", "Download multiple builds sequentially as a test") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>"); Opts.add_option("", @@ -4724,42 +2903,37 @@ BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Pare void BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - m_Parent.ParsePath(m_Path, Opts); - - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts); + ParsePath(m_Path, Opts); + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {}, Opts, m_TestSystemRootDir); + // Place scratch next to the download target so the scratch area shares its volume. + { + const std::u8string PathStr = m_Path.generic_u8string(); + const IoHash PathHash = IoHash::HashBuffer(PathStr.data(), PathStr.length()); + SetZenFolderPath(m_Path.parent_path() / fmt::format("zen_{}", PathHash)); + } + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto ZenGuard = MakeGuard([this]() { CleanZenFolder(); }); - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); + + std::unique_ptr<ProgressBase> Progress = CreateProgress(); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -4769,35 +2943,40 @@ BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), Opts.help()); } - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, /*BuildPartIds,*/ {}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = BuildIdString == m_BuildIds.front(), .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Multitest aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("\n"); } } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 7ef71e176..338318757 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -2,65 +2,227 @@ #pragma once -#include "../authutils.h" -#include "../zen.h" - -#include <zenhttp/auth/authmgr.h> -#include <zenhttp/httpclientauth.h> +#include <zencore/scopeguard.h> #include <zenremotestore/builds/buildstoragecache.h> #include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/partialblockrequestmode.h> -#include <filesystem> +#include <zenremotestore/transferthreadworkers.h> + +#include "authutils.h" +#include "consoleprogress.h" + +#include <optional> namespace zen { -class BuildsCommand; +class ProgressBase; +class AuthMgr; + +struct CreateBuildStorageOptions +{ + bool RequireNamespace = true; + bool RequireBucket = true; + bool BoostCacheBackgroundWorkers = false; +}; + +////////////////////////////////////////////////////////////////////////// -class BuildsListNamespacesSubCmd : public ZenSubCmdBase +struct BuildsConfiguration +{ + std::filesystem::path SystemRootDir; + bool UseSparseFiles = true; + + bool PlainProgress = false; + bool LogProgress = false; + bool Verbose = false; + bool Quiet = false; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; + bool BoostWorkerCount = false; + bool BoostWorkerMemory = false; + bool BoostWorkers = false; + + std::string OverrideHost; + std::string Host; + std::string Url; + bool AssumeHttp2 = false; + bool VerboseHttp = false; + bool AllowRedirect = false; + std::string Namespace; + std::string Bucket; + + std::filesystem::path StoragePath; + bool WriteMetadataAsJson = false; + + std::string ZenCacheHost; + + AuthCommandLineOptions AuthOptions; + + std::string IncludeWildcard; + std::string ExcludeWildcard; + std::string ExcludeFolders; + std::string ExcludeExtensions; + + std::filesystem::path ChunkingCachePath; + + bool AllowMultiparts = true; + std::string AllowPartialBlockRequests = "true"; + + bool AppendNewContent = false; + + std::filesystem::path ZenFolderPath; + + void AddSystemOptions(cxxopts::Options& Ops); + void AddCloudOptions(cxxopts::Options& Ops); + void AddFileOptions(cxxopts::Options& Ops); + void AddCacheOptions(cxxopts::Options& Ops); + void AddOutputOptions(cxxopts::Options& Ops); + void AddWorkerOptions(cxxopts::Options& Ops); + void AddZenFolderOptions(cxxopts::Options& Ops); + void AddChunkingCacheOptions(cxxopts::Options& Ops); + void AddWildcardOptions(cxxopts::Options& Ops); + void AddExcludeFolderOption(cxxopts::Options& Ops); + void AddExcludeExtensionsOption(cxxopts::Options& Ops); + void AddMultipartOptions(cxxopts::Options& Ops); + void AddPartialBlockRequestOptions(cxxopts::Options& Ops); + void AddAppendNewContentOptions(cxxopts::Options& Ops); +}; + +////////////////////////////////////////////////////////////////////////// + +class BuildsSubCmdBase : public ZenSubCmdBase { public: - explicit BuildsListNamespacesSubCmd(BuildsCommand& Parent); + BuildsSubCmdBase(BuildsConfiguration& Config, std::string_view Name, std::string_view Description) + : ZenSubCmdBase(Name, Description) + , m_Config(Config) + { + } + +protected: + const BuildsConfiguration& m_Config; + std::filesystem::path m_ResolvedZenFolderPath; + // Set by EnsureZenFolderExists: true if this command newly created the zen folder + // (so CleanZenFolder may remove the whole folder), false if it already existed + // (in which case only the temp subfolder is wiped, to preserve any pre-existing state). + bool m_CreatedZenFolder = false; + + struct ResolvedStorage + { + std::filesystem::path SystemRootDir; + std::string Host; + std::string Namespace; + std::string Bucket; + std::filesystem::path StoragePath; + }; + + void LogBanner(); + void LogWorkersInfo(const TransferThreadWorkers& Workers); + + // SystemRootDirOverride / StoragePathOverride: empty = fall back to m_Config. + ResolvedStorage ParseStorageOptions(std::string& BuildId, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + const std::filesystem::path& SystemRootDirOverride = {}, + const std::filesystem::path& StoragePathOverride = {}); + + // Builds the storage instance using the supplied Resolved values. ZenFolder is the final zen folder + // path (caller resolves; pass GetZenFolderPath() after ResolveZenFolderPath). If ZenFolder is empty + // the storage runs without a temp directory and keeps all downloads in memory. + // Caller owns Stats/Auth lifetime. Stats must outlive the returned StorageInstance. + StorageInstance CreateBuildStorage(const std::filesystem::path& ZenFolder, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + BuildStorageBase::Statistics& OutStorageStats, + BuildStorageCache::Statistics& OutCacheStats, + std::unique_ptr<AuthMgr>& OutAuth, + const ResolvedStorage& Resolved); + Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts); + Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts); + std::vector<Oid> ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts); + std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts); + CbObject ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts); + void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts); + IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts); + EPartialBlockRequestMode ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts); + void ParseZenProcessId(int& ZenProcessId); + void ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards); + void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions); + + // Resolves the zen folder using this chain: --zen-folder-path; else LocalPath/ZenFolderName + // if LocalPath is non-empty; else cwd/ZenFolderName. Read-only commands that do not need a + // zen folder should skip this and pass an empty path to CreateBuildStorage. + void ResolveZenFolderPath(const std::filesystem::path& LocalPath); + // Assigns the zen folder to --zen-folder-path if given, else to Fallback directly (no + // ZenFolderName appended). For commands that want a specific scratch location rather than + // the standard LocalPath/.zen chain. + void SetZenFolderPath(const std::filesystem::path& Fallback); + const std::filesystem::path& GetZenFolderPath() const { return m_ResolvedZenFolderPath; } + // Creates the resolved zen folder and records whether it had to be created. + // Use together with CleanZenFolder via MakeGuard. + void EnsureZenFolderExists(); + + std::atomic<bool>& AbortFlag() const; + std::atomic<bool>& PauseFlag() const; + std::unique_ptr<ProgressBase> CreateProgress() const; + + // Wipes temp state produced by the command. If EnsureZenFolderExists created the zen folder + // it is removed outright; otherwise only the temp subfolder is wiped so pre-existing state + // (e.g. current_state.cbo from a prior download in the same folder) is preserved. + void CleanZenFolder(); +}; + +////////////////////////////////////////////////////////////////////////// + +class BuildsListNamespacesSubCmd : public BuildsSubCmdBase +{ +public: + explicit BuildsListNamespacesSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; bool m_Recursive = false; std::filesystem::path m_ResultPath; }; -class BuildsListSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsListSubCmd : public BuildsSubCmdBase { public: - explicit BuildsListSubCmd(BuildsCommand& Parent); + explicit BuildsListSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_QueryPath; std::filesystem::path m_ResultPath; }; -class BuildsListBlocksSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsListBlocksSubCmd : public BuildsSubCmdBase { public: - explicit BuildsListBlocksSubCmd(BuildsCommand& Parent); + explicit BuildsListBlocksSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::string m_BuildId; std::filesystem::path m_ResultPath; uint32_t m_MaxCount = 16; }; -class BuildsUploadSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsUploadSubCmd : public BuildsSubCmdBase { public: - explicit BuildsUploadSubCmd(BuildsCommand& Parent); + explicit BuildsUploadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_Path; std::string m_BuildId; std::string m_BuildPartId; @@ -76,14 +238,15 @@ private: bool m_UploadToZenCache = true; }; -class BuildsDownloadSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsDownloadSubCmd : public BuildsSubCmdBase { public: - explicit BuildsDownloadSubCmd(BuildsCommand& Parent); + explicit BuildsDownloadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_Path; std::string m_BuildId; std::vector<std::string> m_BuildPartIds; @@ -94,117 +257,129 @@ private: bool m_EnableScavenging = true; std::filesystem::path m_DownloadSpecPath; bool m_UploadToZenCache = true; - bool m_PrimeCacheOnly = false; bool m_AllowFileClone = true; }; -class BuildsLsSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsLsSubCmd : public BuildsSubCmdBase { public: - explicit BuildsLsSubCmd(BuildsCommand& Parent); + explicit BuildsLsSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::string m_BuildId; std::vector<std::string> m_BuildPartIds; std::vector<std::string> m_BuildPartNames; std::filesystem::path m_ResultPath; }; -class BuildsDiffSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsDiffSubCmd : public BuildsSubCmdBase { public: - explicit BuildsDiffSubCmd(BuildsCommand& Parent); + explicit BuildsDiffSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_Path; std::filesystem::path m_DiffPath; bool m_OnlyChunked = false; }; -class BuildsFetchBlobSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsFetchBlobSubCmd : public BuildsSubCmdBase { public: - explicit BuildsFetchBlobSubCmd(BuildsCommand& Parent); + explicit BuildsFetchBlobSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - std::string m_BuildId; - std::string m_BlobHash; + std::string m_BuildId; + std::string m_BlobHash; }; -class BuildsPrimeCacheSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsPrimeCacheSubCmd : public BuildsSubCmdBase { public: - explicit BuildsPrimeCacheSubCmd(BuildsCommand& Parent); + explicit BuildsPrimeCacheSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::string m_BuildId; std::vector<std::string> m_BuildPartIds; std::vector<std::string> m_BuildPartNames; bool m_Force = false; }; -class BuildsPauseSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsPauseSubCmd : public BuildsSubCmdBase { public: - explicit BuildsPauseSubCmd(BuildsCommand& Parent); + explicit BuildsPauseSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - int m_ZenProcessId = -1; + int m_ZenProcessId = -1; }; -class BuildsResumeSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsResumeSubCmd : public BuildsSubCmdBase { public: - explicit BuildsResumeSubCmd(BuildsCommand& Parent); + explicit BuildsResumeSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - int m_ZenProcessId = -1; + int m_ZenProcessId = -1; }; -class BuildsAbortSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsAbortSubCmd : public BuildsSubCmdBase { public: - explicit BuildsAbortSubCmd(BuildsCommand& Parent); + explicit BuildsAbortSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - int m_ZenProcessId = -1; + int m_ZenProcessId = -1; }; -class BuildsValidatePartSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsValidatePartSubCmd : public BuildsSubCmdBase { public: - explicit BuildsValidatePartSubCmd(BuildsCommand& Parent); + explicit BuildsValidatePartSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - std::string m_BuildId; - std::string m_BuildPartId; - std::string m_BuildPartName; + std::string m_BuildId; + std::string m_BuildPartId; + std::string m_BuildPartName; }; -class BuildsTestSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsTestSubCmd : public BuildsSubCmdBase { public: - explicit BuildsTestSubCmd(BuildsCommand& Parent); + explicit BuildsTestSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; + // Fixture-only overrides of SystemRootDir/StoragePath; passed to ParseStorageOptions explicitly. + std::filesystem::path m_TestSystemRootDir; + std::filesystem::path m_TestStoragePath; + std::filesystem::path m_Path; std::string m_BuildPartName; std::string m_BuildId; @@ -217,21 +392,27 @@ private: bool m_AllowFileClone = true; }; -class BuildsMultiTestDownloadSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsMultiTestDownloadSubCmd : public BuildsSubCmdBase { public: - explicit BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent); + explicit BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; + // Fixture-only override of SystemRootDir; passed to CreateStorage explicitly. + std::filesystem::path m_TestSystemRootDir; + std::filesystem::path m_Path; std::vector<std::string> m_BuildIds; bool m_EnableScavenging = true; bool m_AllowFileClone = true; }; -class BuildsCommand : public CacheStoreCmdWithSubCommands +////////////////////////////////////////////////////////////////////////// + +class BuildsCommand : public ZenCmdWithSubCommands { public: static constexpr char Name[] = "builds"; @@ -243,99 +424,14 @@ public: cxxopts::Options& Options() override { return m_Options; } - // Option-adding helpers (called by subcommand constructors) - void AddSystemOptions(cxxopts::Options& Ops); - void AddCloudOptions(cxxopts::Options& Ops); - void AddFileOptions(cxxopts::Options& Ops); - void AddCacheOptions(cxxopts::Options& Ops); - void AddOutputOptions(cxxopts::Options& Ops); - void AddWorkerOptions(cxxopts::Options& Ops); - void AddZenFolderOptions(cxxopts::Options& Ops); - void AddChunkingCacheOptions(cxxopts::Options& Ops); - void AddWildcardOptions(cxxopts::Options& Ops); - void AddExcludeFolderOption(cxxopts::Options& Ops); - void AddExcludeExtensionsOption(cxxopts::Options& Ops); - void AddMultipartOptions(cxxopts::Options& Ops); - void AddPartialBlockRequestOptions(cxxopts::Options& Ops); - void AddAppendNewContentOptions(cxxopts::Options& Ops); - - // Shared parsing/factory methods used by subcommand Run() implementations - void ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts); - StorageInstance CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, - BuildStorageCache::Statistics& StorageCacheStats, - const std::filesystem::path& TempPath, - std::string& BuildId, - bool RequireNamespace, - bool RequireBucket, - bool BoostCacheBackgroundWorkerPool, - std::unique_ptr<AuthMgr>& Auth, - cxxopts::Options& SubOpts); - Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts); - Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts); - std::vector<Oid> ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts); - std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts); - CbObject ParseBuildMetadata(bool CreateBuild, - std::filesystem::path& BuildMetadataPath, - const std::string& BuildMetadata, - cxxopts::Options& SubOpts); - void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts); - IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts); - EPartialBlockRequestMode ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts); - void ParseZenProcessId(int& ZenProcessId); - void ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards); - void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions); - - void ResolveZenFolderPath(const std::filesystem::path& DefaultPath); - const std::filesystem::path& GetZenFolderPath() const { return m_ZenFolderPath; } - - cxxopts::Options m_Options{Name, Description}; - std::string m_SubCommand; - - // Shared state populated by AddXxxOptions helpers (bound via cxxopts::value references) - std::filesystem::path m_SystemRootDir; - bool m_UseSparseFiles = true; - - bool m_PlainProgress = false; - bool m_LogProgress = false; - bool m_Verbose = false; - bool m_Quiet = false; - bool m_BoostWorkerCount = false; - bool m_BoostWorkerMemory = false; - bool m_BoostWorkers = false; - - // cloud builds - std::string m_OverrideHost; - std::string m_Host; - std::string m_Url; - bool m_AssumeHttp2 = false; - bool m_VerboseHttp = false; - bool m_AllowRedirect = false; - std::string m_Namespace; - std::string m_Bucket; - - std::filesystem::path m_StoragePath; - bool m_WriteMetadataAsJson = false; - - std::string m_ZenCacheHost; - - AuthCommandLineOptions m_AuthOptions; - - std::string m_IncludeWildcard; - std::string m_ExcludeWildcard; - std::string m_ExcludeFolders; - std::string m_ExcludeExtensions; - - std::filesystem::path m_ChunkingCachePath; - - bool m_AllowMultiparts = true; - std::string m_AllowPartialBlockRequests = "true"; - - bool m_AppendNewContent = false; + BuildsConfiguration& GetConfiguration() { return m_Configuration; } + const BuildsConfiguration& GetConfiguration() const { return m_Configuration; } private: - std::filesystem::path m_ZenFolderPath; + cxxopts::Options m_Options{Name, Description}; + std::string m_SubCommand; + BuildsConfiguration m_Configuration; -protected: BuildsListNamespacesSubCmd m_ListNamespacesSubCmd; BuildsListSubCmd m_ListSubCmd; BuildsListBlocksSubCmd m_ListBlocksSubCmd; @@ -352,6 +448,11 @@ protected: BuildsTestSubCmd m_TestSubCmd; BuildsMultiTestDownloadSubCmd m_MultiTestDownloadSubCmd; + std::optional<ScopedSignalHandler> m_SigIntGuard; +#if ZEN_PLATFORM_WINDOWS + std::optional<ScopedSignalHandler> m_SigBreakGuard; +#endif + bool OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) override; }; diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index a8c15f119..f93a5318c 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -2,25 +2,40 @@ #include "cache_cmd.h" +#include "zenserviceclient.h" + #include <zencore/compactbinarybuilder.h> #include <zencore/compress.h> #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> +#include <zencore/process.h> #include <zencore/scopeguard.h> +#include <zencore/session.h> +#include <zencore/stream.h> #include <zencore/thread.h> +#include <zencore/timer.h> #include <zencore/workthreadpool.h> +#include <zenhttp/formatters.h> #include <zenhttp/httpclient.h> #include <zenhttp/httpcommon.h> #include <zenhttp/packageformat.h> #include <zenstore/cache/cachepolicy.h> +#include <zenutil/rpcrecording.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <fmt/format.h> +#include <gsl/gsl-lite.hpp> +ZEN_THIRD_PARTY_INCLUDES_END #include <memory> #include <random> namespace zen { +using namespace std::literals; + namespace { IoBuffer CreateRandomBlob(uint64_t Size) { @@ -56,37 +71,112 @@ namespace { } } // namespace -DropCommand::DropCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheCommand + +CacheCommand::CacheCommand() { m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>"); - m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>"); - m_Options.parse_positional({"namespace", "bucket"}); + + AddSubCommand(m_DetailsSubCmd); + AddSubCommand(m_DropSubCmd); + AddSubCommand(m_GenSubCmd); + AddSubCommand(m_GetSubCmd); + AddSubCommand(m_InfoSubCmd); + AddSubCommand(m_RecordSubCmd); + AddSubCommand(m_ReplaySubCmd); + AddSubCommand(m_StatsSubCmd); } -DropCommand::~DropCommand() = default; +CacheCommand::~CacheCommand() = default; -void -DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +//////////////////////////////////////////////////////////////////////////////// +// CacheSubCmdBase + +CacheSubCmdBase::CacheSubCmdBase(std::string_view Name, std::string_view Description) : ZenSubCmdBase(Name, Description) { - ZEN_UNUSED(GlobalOptions); + m_SubOptions.add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); +} - if (!ParseOptions(argc, argv)) +void +CacheSubCmdBase::ResolveHost() +{ + m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); + if (m_HostName.empty()) { - return; + throw OptionParseException("Unable to resolve server specification", m_SubOptions.help()); } +} - m_HostName = ResolveTargetHostSpec(m_HostName); +//////////////////////////////////////////////////////////////////////////////// +// Legacy shim dispatcher - if (m_HostName.empty()) +namespace cache_legacy_shim { + static void Dispatch(std::span<const std::string_view> Injected, const ZenCliOptions& GlobalOptions, int argc, char** argv) { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); + // cxxopts treats argv as writable char** in the style of C main(argv). + // Stage the injected tokens in writable std::string storage so we never + // hand out pointers to string literals. + std::vector<std::string> Storage; + Storage.reserve(Injected.size()); + for (std::string_view Token : Injected) + { + Storage.emplace_back(Token); + } + + std::vector<char*> NewArgv; + NewArgv.reserve(static_cast<size_t>(argc) + Storage.size()); + NewArgv.push_back(argv[0]); + for (std::string& Token : Storage) + { + NewArgv.push_back(Token.data()); + } + for (int i = 1; i < argc; ++i) + { + NewArgv.push_back(argv[i]); + } + + CacheCommand Impl; + Impl.Run(GlobalOptions, static_cast<int>(NewArgv.size()), NewArgv.data()); } + void RunAs(const char* SubCommandName, const ZenCliOptions& GlobalOptions, int argc, char** argv) + { + const std::string_view Tokens[] = {std::string_view(SubCommandName)}; + Dispatch(Tokens, GlobalOptions, argc, argv); + } +} // namespace cache_legacy_shim + +// RpcStopRecordingCommand is unique among legacy shims in that it needs to +// inject two tokens ("record" and "stop") rather than a single subcommand name. +void +RpcStopRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + using namespace std::literals; + const std::string_view Tokens[] = {"record"sv, "stop"sv}; + cache_legacy_shim::Dispatch(Tokens, GlobalOptions, argc, argv); +} + +//////////////////////////////////////////////////////////////////////////////// +// CacheDropSubCmd + +CacheDropSubCmd::CacheDropSubCmd() : CacheSubCmdBase("drop", "Drop cache namespace or bucket") +{ + m_SubOptions.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>"); + m_SubOptions.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>"); + m_SubOptions.parse_positional({"namespace", "bucket"}); +} + +void +CacheDropSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "drop"}); + HttpClient& Http = Service.Http(); + if (m_NamespaceName.empty()) { - throw OptionParseException("'--namespace' is required", m_Options.help()); + throw OptionParseException("'--namespace' is required", m_SubOptions.help()); } std::string Url; @@ -94,18 +184,16 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BucketName.empty()) { - DropDescription = fmt::format("cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); + DropDescription = fmt::format("cache namespace '{}' from '{}'", m_NamespaceName, Service.HostSpec()); Url = fmt::format("/z$/{}", m_NamespaceName); } else { - DropDescription = fmt::format("cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); + DropDescription = fmt::format("cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, Service.HostSpec()); Url = fmt::format("/z$/{}/{}", m_NamespaceName, m_BucketName); } ZEN_CONSOLE("Dropping {}", DropDescription); - - HttpClient Http = CreateHttpClient(m_HostName); if (HttpClient::Response Response = Http.Delete(Url)) { ZEN_CONSOLE("{}", Response.ToText()); @@ -116,54 +204,43 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } -CacheInfoCommand::CacheInfoCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheInfoSubCmd + +CacheInfoSubCmd::CacheInfoSubCmd() : CacheSubCmdBase("info", "Info on cache, namespace or bucket") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>"); - m_Options.add_option("", - "", - "bucketsizes", - "Comma delimited list of bucket names to get size info from, * to get info on all buckets", - cxxopts::value(m_SizeInfoBucketNames), - "<bucketnames>"); - m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>"); - m_Options.add_option("", "", "bucketsize", "Show detailed bucket size info", cxxopts::value(m_BucketSizeInfo), "<bucketsize>"); - - m_Options.parse_positional({"namespace", "bucket"}); + m_SubOptions.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>"); + m_SubOptions.add_option("", + "", + "bucketsizes", + "Comma delimited list of bucket names to get size info from, * to get info on all buckets", + cxxopts::value(m_SizeInfoBucketNames), + "<bucketnames>"); + m_SubOptions.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>"); + m_SubOptions.add_option("", "", "bucketsize", "Show detailed bucket size info", cxxopts::value(m_BucketSizeInfo), "<bucketsize>"); + m_SubOptions.add_option("", "y", "yaml", "Output as YAML instead of JSON", cxxopts::value(m_YAML), "<yaml>"); + m_SubOptions.parse_positional({"namespace", "bucket"}); } -CacheInfoCommand::~CacheInfoCommand() = default; - void -CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheInfoSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "info"}); + HttpClient& Http = Service.Http(); std::string Url; - if (m_HostName.empty()) + if (m_NamespaceName.empty()) { if (!m_SizeInfoBucketNames.empty()) { - throw OptionParseException("'--bucketsizes' requires '--namespace'", m_Options.help()); + throw OptionParseException("'--bucketsizes' requires '--namespace'", m_SubOptions.help()); } if (m_BucketSizeInfo) { - throw OptionParseException("'--bucketsize' requires '--namespace' and '--bucket'", m_Options.help()); + throw OptionParseException("'--bucketsize' requires '--namespace' and '--bucket'", m_SubOptions.help()); } - ZEN_CONSOLE("Info on cache from '{}'", m_HostName); + ZEN_CONSOLE("Info on cache from '{}'", Service.HostSpec()); Url = "/z$"; } else if (m_BucketName.empty()) @@ -171,18 +248,18 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BucketSizeInfo) { throw OptionParseException(fmt::format("'--bucketsize' requires '--namespace' and '--bucket' ('{}')", m_BucketName), - m_Options.help()); + m_SubOptions.help()); } - ZEN_CONSOLE("Info on cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); + ZEN_CONSOLE("Info on cache namespace '{}' from '{}'", m_NamespaceName, Service.HostSpec()); Url = fmt::format("/z$/{}", m_NamespaceName); } else { if (!m_SizeInfoBucketNames.empty()) { - throw OptionParseException("'--bucketsizes' conflicts with '--bucket'", m_Options.help()); + throw OptionParseException("'--bucketsizes' conflicts with '--bucket'", m_SubOptions.help()); } - ZEN_CONSOLE("Info on cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); + ZEN_CONSOLE("Info on cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, Service.HostSpec()); Url = fmt::format("/z$/{}/{}", m_NamespaceName, m_BucketName); } @@ -196,8 +273,9 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Parameters.Entries.insert({"bucketsize", "true"}); } - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON), Parameters)) + const ZenContentType AcceptType = m_YAML ? ZenContentType::kYAML : ZenContentType::kJSON; + + if (HttpClient::Response Response = Http.Get(Url, HttpClient::Accept(AcceptType), Parameters)) { ZEN_CONSOLE("{}", Response.ToText()); } @@ -207,76 +285,62 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } -CacheStatsCommand::CacheStatsCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheStatsSubCmd + +CacheStatsSubCmd::CacheStatsSubCmd() : CacheSubCmdBase("stats", "Stats on cache") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); + m_SubOptions.add_option("", "y", "yaml", "Output as YAML instead of JSON", cxxopts::value(m_YAML), "<yaml>"); } -CacheStatsCommand::~CacheStatsCommand() = default; - void -CacheStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheStatsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "stats"}); + HttpClient& Http = Service.Http(); - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + const ZenContentType AcceptType = m_YAML ? ZenContentType::kYAML : ZenContentType::kJSON; - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = Http.Get("/stats/z$", HttpClient::Accept(ZenContentType::kJSON))) + if (HttpClient::Response Response = Http.Get("/stats/z$", HttpClient::Accept(AcceptType))) { ZEN_CONSOLE("{}", Response.ToText()); } else { - Response.ThrowError("Info failed"); + Response.ThrowError("Stats failed"); } } -CacheDetailsCommand::CacheDetailsCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheDetailsSubCmd + +CacheDetailsSubCmd::CacheDetailsSubCmd() : CacheSubCmdBase("details", "Details on cache") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "c", "csv", "Info on csv format", cxxopts::value(m_CSV), "<csv>"); - m_Options.add_option("", "d", "details", "Get detailed information about records", cxxopts::value(m_Details), "<details>"); - m_Options.add_option("", - "a", - "attachmentdetails", - "Get detailed information about attachments", - cxxopts::value(m_AttachmentDetails), - "<attachmentdetails>"); - m_Options.add_option("", "n", "namespace", "Namespace name to get info for", cxxopts::value(m_Namespace), "<namespace>"); - m_Options.add_option("", "b", "bucket", "Filter on bucket name", cxxopts::value(m_Bucket), "<bucket>"); - m_Options.add_option("", "v", "valuekey", "Filter on value key hash string", cxxopts::value(m_ValueKey), "<valuekey>"); + m_SubOptions.add_option("", "c", "csv", "Output as CSV instead of JSON", cxxopts::value(m_CSV), "<csv>"); + m_SubOptions.add_option("", "y", "yaml", "Output as YAML instead of JSON", cxxopts::value(m_YAML), "<yaml>"); + m_SubOptions.add_option("", "d", "details", "Get detailed information about records", cxxopts::value(m_Details), "<details>"); + m_SubOptions.add_option("", + "a", + "attachmentdetails", + "Get detailed information about attachments", + cxxopts::value(m_AttachmentDetails), + "<attachmentdetails>"); + m_SubOptions.add_option("", "n", "namespace", "Namespace name to get info for", cxxopts::value(m_Namespace), "<namespace>"); + m_SubOptions.add_option("", "b", "bucket", "Filter on bucket name", cxxopts::value(m_Bucket), "<bucket>"); + m_SubOptions.add_option("", "v", "valuekey", "Filter on value key hash string", cxxopts::value(m_ValueKey), "<valuekey>"); } -CacheDetailsCommand::~CacheDetailsCommand() = default; - void -CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheDetailsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "details"}); + HttpClient& Http = Service.Http(); - if (!ParseOptions(argc, argv)) + if (m_CSV && m_YAML) { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); + throw OptionParseException("'--csv' conflicts with '--yaml'", m_SubOptions.help()); } HttpClient::KeyValueMap Parameters; @@ -296,7 +360,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar } else { - Headers = HttpClient::Accept(ZenContentType::kJSON); + Headers = HttpClient::Accept(m_YAML ? ZenContentType::kYAML : ZenContentType::kJSON); } std::string Url; @@ -304,11 +368,11 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar { if (m_Namespace.empty()) { - throw OptionParseException("'--namespace' is required", m_Options.help()); + throw OptionParseException("'--namespace' is required", m_SubOptions.help()); } if (m_Bucket.empty()) { - throw OptionParseException("'--bucket' is required", m_Options.help()); + throw OptionParseException("'--bucket' is required", m_SubOptions.help()); } Url = fmt::format("/z$/details$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey); } @@ -316,7 +380,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar { if (m_Namespace.empty()) { - throw OptionParseException("'--namespace' is required", m_Options.help()); + throw OptionParseException("'--namespace' is required", m_SubOptions.help()); } Url = fmt::format("/z$/details$/{}/{}", m_Namespace, m_Bucket); } @@ -329,61 +393,49 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar Url = "/z$/details$"; } - HttpClient Http = CreateHttpClient(m_HostName); if (HttpClient::Response Response = Http.Get(Url, Headers, Parameters)) { ZEN_CONSOLE("{}", Response.ToText()); } else { - Response.ThrowError("Info failed"); + Response.ThrowError("Details failed"); } } -CacheGenerateCommand::CacheGenerateCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheGenSubCmd + +CacheGenSubCmd::CacheGenSubCmd() : CacheSubCmdBase("gen", "Generates cache values into a bucket") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options + m_SubOptions .add_option("", "n", "namespace", "Namespace to generate cache values/records for", cxxopts::value(m_Namespace), "<namespace>"); - m_Options.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), "<bucket>"); - m_Options.add_option("", "", "count", "Number of cache values/records to generate", cxxopts::value(m_Count), "<count>"); - m_Options.add_option("", "", "min-size", "Minimum size of cache value/attachments", cxxopts::value(m_MinSize), "<min>"); - m_Options.add_option("", "", "max-size", "Maximum size of cache value/attachments", cxxopts::value(m_MaxSize), "<max>"); - m_Options.add_option("", - "", - "min-attachments", - "Minimum number of attachments when creating record based values", - cxxopts::value(m_MinAttachmentCount), - "<minattachments>"); - m_Options.add_option("", - "", - "max-attachments", - "Minimum number of attachments when creating record based values, 0 to only create cache values", - cxxopts::value(m_MaxAttachmentCount), - "<maxattachments>"); - m_Options.parse_positional({"namespace", "bucket", "count"}); - m_Options.positional_help("namespace bucket count"); + m_SubOptions.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), "<bucket>"); + m_SubOptions.add_option("", "", "count", "Number of cache values/records to generate", cxxopts::value(m_Count), "<count>"); + m_SubOptions.add_option("", "", "min-size", "Minimum size of cache value/attachments", cxxopts::value(m_MinSize), "<min>"); + m_SubOptions.add_option("", "", "max-size", "Maximum size of cache value/attachments", cxxopts::value(m_MaxSize), "<max>"); + m_SubOptions.add_option("", + "", + "min-attachments", + "Minimum number of attachments when creating record based values", + cxxopts::value(m_MinAttachmentCount), + "<minattachments>"); + m_SubOptions.add_option("", + "", + "max-attachments", + "Minimum number of attachments when creating record based values, 0 to only create cache values", + cxxopts::value(m_MaxAttachmentCount), + "<maxattachments>"); + m_SubOptions.parse_positional({"namespace", "bucket", "count"}); + m_SubOptions.positional_help("namespace bucket count"); } -CacheGenerateCommand::~CacheGenerateCommand() = default; - void -CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheGenSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "gen"}); + HttpClient& Http = Service.Http(); if (m_MaxSize == 0 && m_MinSize == 0) { @@ -400,6 +452,16 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } } + // The size-range expansion below requires MinSize >= 1 (it uses + // `MinSize - 1` as a uniform distribution upper bound, which would + // underflow on an unsigned zero) and MaxSize >= MinSize. + if (m_MinSize == 0 || m_MaxSize < m_MinSize) + { + throw OptionParseException( + fmt::format("'--min-size' ({}) must be >= 1 and '--max-size' ({}) must be >= '--min-size'", m_MinSize, m_MaxSize), + m_SubOptions.help()); + } + std::vector<std::uniform_int_distribution<uint64_t>> Variations; std::vector<size_t> SizeRanges; SizeRanges.push_back(m_MinSize); @@ -431,8 +493,6 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a std::uniform_int_distribution<uint64_t> KeyDistribution; - HttpClient Http = CreateHttpClient(m_HostName); - auto GeneratePutCacheValueRequest( [this, &KeyDistribution, &Generator](std::span<std::uint64_t> BatchSizes, uint64_t RequestIndex) -> CbPackage { CbPackage Package; @@ -583,68 +643,56 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } } -CacheGetCommand::CacheGetCommand() +//////////////////////////////////////////////////////////////////////////////// +// CacheGetSubCmd + +CacheGetSubCmd::CacheGetSubCmd() : CacheSubCmdBase("get", "Get cache values/records or attachments from a bucket") { - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options - .add_option("", "n", "namespace", "Namespace to generate cache values/records for", cxxopts::value(m_Namespace), "<namespace>"); - m_Options.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), "<bucket>"); - m_Options.add_option("", "v", "valuekey", "Cache entry iohash id", cxxopts::value(m_ValueKey), "<valuekey>"); - m_Options.add_option("", - "a", - "attachmenthash", - "For a cache entry record, get a particular attachment based on the 'RawHash'", - cxxopts::value(m_AttachmentHash), - "<attachmenthash>"); - m_Options.add_option("", "o", "output-path", "File path for output data", cxxopts::value(m_OutputPath), "<path>"); - m_Options.add_option("", "t", "text", "Ouput content of cache entry record as text", cxxopts::value(m_AsText), "<text>"); - m_Options + m_SubOptions.add_option("", "n", "namespace", "Namespace of the cache entry", cxxopts::value(m_Namespace), "<namespace>"); + m_SubOptions.add_option("", "b", "bucket", "Bucket of the cache entry", cxxopts::value(m_Bucket), "<bucket>"); + m_SubOptions.add_option("", "v", "valuekey", "Cache entry iohash id", cxxopts::value(m_ValueKey), "<valuekey>"); + m_SubOptions.add_option("", + "a", + "attachmenthash", + "For a cache entry record, get a particular attachment based on the 'RawHash'", + cxxopts::value(m_AttachmentHash), + "<attachmenthash>"); + m_SubOptions.add_option("", "o", "output-path", "File path for output data", cxxopts::value(m_OutputPath), "<path>"); + m_SubOptions.add_option("", "t", "text", "Output content of cache entry record as text", cxxopts::value(m_AsText), "<text>"); + m_SubOptions .add_option("", "d", "decompress", "Decompress data when applicable. Default = true", cxxopts::value(m_Decompress), "<decompress>"); - m_Options.parse_positional({"namespace", "bucket", "valuekey", "attachmenthash"}); - m_Options.positional_help("namespace bucket valuekey attachmenthash"); + m_SubOptions.parse_positional({"namespace", "bucket", "valuekey", "attachmenthash"}); + m_SubOptions.positional_help("namespace bucket valuekey attachmenthash"); } -CacheGetCommand::~CacheGetCommand() = default; - void -CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +CacheGetSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - ZEN_UNUSED(GlobalOptions); - using namespace std::literals; - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "get"}); + HttpClient& Http = Service.Http(); if (m_Namespace.empty()) { - throw OptionParseException("'--namespace' is required", m_Options.help()); + throw OptionParseException("'--namespace' is required", m_SubOptions.help()); } if (m_Bucket.empty()) { - throw OptionParseException("'--bucket' is required", m_Options.help()); + throw OptionParseException("'--bucket' is required", m_SubOptions.help()); } if (m_ValueKey.empty()) { - throw OptionParseException("'--valuekey' is required", m_Options.help()); + throw OptionParseException("'--valuekey' is required", m_SubOptions.help()); } IoHash ValueId; if (!IoHash::TryParse(m_ValueKey, ValueId)) { - throw OptionParseException(fmt::format("'--value-key' ('{}') is malformed", m_ValueKey), m_Options.help()); + throw OptionParseException(fmt::format("'--valuekey' ('{}') is malformed", m_ValueKey), m_SubOptions.help()); } IoHash AttachmentHash; @@ -652,11 +700,14 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!IoHash::TryParse(m_AttachmentHash, AttachmentHash)) { - throw OptionParseException(fmt::format("'--attachmenthash' ('{}') is malformed", m_AttachmentHash), m_Options.help()); + throw OptionParseException(fmt::format("'--attachmenthash' ('{}') is malformed", m_AttachmentHash), m_SubOptions.help()); } } - HttpClient Http = CreateHttpClient(m_HostName); + if (m_OutputPath.empty() && !m_AsText) + { + throw OptionParseException("'--output-path' is required (or pass '--as-text' to print to stdout)", m_SubOptions.help()); + } if (!m_OutputPath.empty()) { @@ -669,18 +720,19 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) CreateDirectories(m_OutputPath.parent_path()); } } - if (m_OutputPath.empty()) - { - m_OutputPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); - } - std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey); + std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, ValueId); if (AttachmentHash != IoHash::Zero) { Url = fmt::format("{}/{}", Url, AttachmentHash); } if (HttpClient::Response Result = Http.Download(Url, std::filesystem::temp_directory_path()); Result) { + // `Http.Download` parks the payload in the system temp dir and returns + // a buffer that already has delete-on-close set, so every exit path + // (exception, fallback WriteFile, `--as-text` console print) reaps it. + // A successful MoveToFile below clears the flag so the payload's + // handle-close doesn't delete the caller's output afterwards. auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer { IoHash RawHash; uint64_t RawSize; @@ -707,7 +759,17 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - if (!MoveToFile(m_OutputPath, ChunkData)) + if (std::error_code MoveEc = MoveToFile(m_OutputPath, ChunkData); MoveEc) + { + // The file was renamed into place; clearing DeleteOnClose prevents + // the move'd-out file at m_OutputPath from being deleted when the + // payload's handle closes. When m_Decompress is false ChunkData + // shares a core with ResponsePayload so either clear suffices; + // when decompressed ChunkData is in-memory and MoveToFile would + // have failed, so we don't reach this branch. + Result.ResponsePayload.SetDeleteOnClose(false); + } + else { WriteFile(m_OutputPath, ChunkData); } @@ -720,4 +782,428 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } +//////////////////////////////////////////////////////////////////////////////// +// CacheRecordSubCmd + +CacheRecordSubCmd::CacheRecordSubCmd() +: CacheSubCmdBase("record", "Start recording cache rpc requests ('cache record <path>'), or stop ('cache record stop')") +{ + m_SubOptions.add_option("", "p", "path", "Recording file path, or 'stop' to stop recording", cxxopts::value(m_Path), "<path>"); + m_SubOptions.parse_positional("path"); + m_SubOptions.positional_help("<path>|stop"); +} + +void +CacheRecordSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + ResolveHost(); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "record"}); + HttpClient& Http = Service.Http(); + + if (m_Path == "stop") + { + if (HttpClient::Response Response = Http.Post("/z$/exec$/stop-recording"sv)) + { + ZEN_CONSOLE("{}", Response.ToText()); + } + else + { + Response.ThrowError("Failed to stop recording"); + } + return; + } + + if (m_Path.empty()) + { + throw OptionParseException("recording path is required (use '<path>' to start, 'stop' to stop)", m_SubOptions.help()); + } + + if (HttpClient::Response Response = + Http.Post("/z$/exec$/start-recording"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap({{"path", m_Path}}))) + { + ZEN_CONSOLE("{}", Response.ToText()); + } + else + { + Response.ThrowError("Failed to start recording"); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// CacheReplaySubCmd + +CacheReplaySubCmd::CacheReplaySubCmd() : CacheSubCmdBase("replay", "Replays a previously recorded session of rpc requests") +{ + m_SubOptions.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), "<path>"); + m_SubOptions.add_option("", "", "dry", "Do a dry run", cxxopts::value(m_DryRun), "<enable>"); + m_SubOptions.add_option("", + "w", + "numthreads", + "Number of worker threads per process", + cxxopts::value(m_ThreadCount)->default_value(fmt::format("{}", GetHardwareConcurrency())), + "<count>"); + m_SubOptions.add_option("", "", "onhost", "Replay on host, bypassing http/network layer", cxxopts::value(m_OnHost), "<onhost>"); + m_SubOptions.add_option("", + "", + "showmethodstats", + "Show statistics of which RPC methods are used", + cxxopts::value(m_ShowMethodStats), + "<showmethodstats>"); + m_SubOptions.add_option("", + "", + "offset", + "Offset into request recording to start replay", + cxxopts::value(m_Offset)->default_value("0"), + "<offset>"); + m_SubOptions.add_option("", + "", + "stride", + "Stride for request recording when replaying requests", + cxxopts::value(m_Stride)->default_value("1"), + "<stride>"); + m_SubOptions.add_option("", "", "numproc", "Number of worker processes", cxxopts::value(m_ProcessCount)->default_value("1"), "<count>"); + m_SubOptions.add_option("", + "", + "forceallowlocalrefs", + "Force enable local refs in requests", + cxxopts::value(m_ForceAllowLocalRefs), + "<enable>"); + m_SubOptions + .add_option("", "", "disablelocalrefs", "Force disable local refs in requests", cxxopts::value(m_DisableLocalRefs), "<enable>"); + m_SubOptions.add_option("", + "", + "forceallowlocalhandlerefs", + "Force enable local refs as handles in requests", + cxxopts::value(m_ForceAllowLocalHandleRef), + "<enable>"); + m_SubOptions.add_option("", + "", + "disablelocalhandlerefs", + "Force disable local refs as handles in requests", + cxxopts::value(m_DisableLocalHandleRefs), + "<enable>"); + m_SubOptions.add_option("", + "", + "forceallowpartiallocalrefs", + "Force enable local refs for all sizes", + cxxopts::value(m_ForceAllowPartialLocalRefs), + "<enable>"); + m_SubOptions.add_option("", + "", + "disablepartiallocalrefs", + "Force disable local refs for all sizes", + cxxopts::value(m_DisablePartialLocalRefs), + "<enable>"); + m_SubOptions.parse_positional("path"); +} + +void +CacheReplaySubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) +{ + if (m_RecordingPath.empty()) + { + throw OptionParseException("'--path' is required", m_SubOptions.help()); + } + + if (!IsDir(m_RecordingPath)) + { + throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath)); + } + + if (m_Stride == 0) + { + throw OptionParseException("'--stride' must be >= 1", m_SubOptions.help()); + } + + m_ThreadCount = Max(m_ThreadCount, 1); + + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "replay"}); + m_HostName = Service.HostSpec(); + + ZEN_CONSOLE("Replay '{}' (start offset {}, stride {}) to '{}', {} threads", + m_RecordingPath, + m_Offset, + m_Stride, + m_HostName, + m_ThreadCount); + + Stopwatch TotalTimer; + + if (m_OnHost) + { + HttpClient& Http = Service.Http(); + if (HttpClient::Response Response = + Http.Post("/z$/exec$/replay-recording"sv, + HttpClient::KeyValueMap{}, + HttpClient::KeyValueMap({{"path", m_RecordingPath}, {"thread-count", fmt::format("{}", m_ThreadCount)}}))) + { + ZEN_CONSOLE("{}", Response.ToText()); + + return; + } + else + { + Response.ThrowError("Failed to start replay"); + } + } + + std::unique_ptr<cache::IRpcRequestReplayer> Replayer = cache::MakeDiskRequestReplayer(m_RecordingPath, true); + uint64_t EntryCount = Replayer->GetRequestCount(); + + if (m_Offset >= EntryCount) + { + ZEN_CONSOLE("Offset {} is at or past the end of the recording ({} entries); nothing to replay", m_Offset, EntryCount); + return; + } + + std::atomic_uint64_t EntryOffset = m_Offset; + std::atomic_uint64_t BytesSent = 0; + std::atomic_uint64_t BytesReceived = 0; + + Stopwatch Timer; + + // The subcommand API does not receive argv, so look the zen executable path + // up from the current process to spawn child workers. + const std::filesystem::path SelfExePath = GetRunningExecutablePath(); + + if (m_ProcessCount > 1) + { + std::vector<std::unique_ptr<ProcessHandle>> WorkerProcesses; + WorkerProcesses.resize(m_ProcessCount); + + ProcessMonitor Monitor; + for (int ProcessIndex = 0; ProcessIndex < m_ProcessCount; ++ProcessIndex) + { + std::string CommandLine = + fmt::format("{} cache replay --hosturl {} --path \"{}\" --offset {} --stride {} --numthreads {} --numproc {}"sv, + SelfExePath.string(), + m_HostName, + m_RecordingPath, + m_Stride == 1 ? 0 : m_Offset + ProcessIndex, + m_Stride, + m_ThreadCount, + 1); + CreateProcResult Result(CreateProc(SelfExePath, CommandLine)); + WorkerProcesses[ProcessIndex] = std::make_unique<ProcessHandle>(); + WorkerProcesses[ProcessIndex]->Initialize(Result); + Monitor.AddPid(WorkerProcesses[ProcessIndex]->Pid()); + } + while (Monitor.IsRunning()) + { + ZEN_CONSOLE("Waiting for worker processes..."); + Sleep(1000); + } + return; + } + else + { + std::map<std::string, size_t> MethodTypes; + RwLock MethodTypesLock; + + WorkerThreadPool WorkerPool(m_ThreadCount); + + Latch WorkLatch(m_ThreadCount); + for (int WorkerIndex = 0; WorkerIndex < m_ThreadCount; ++WorkerIndex) + { + WorkerPool.ScheduleWork( + [this, &WorkLatch, EntryCount, &EntryOffset, &Replayer, &BytesSent, &BytesReceived, &MethodTypes, &MethodTypesLock]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + + std::map<std::string, size_t> LocalMethodTypes; + + auto ReduceTypes = MakeGuard([&] { + RwLock::ExclusiveLockScope __(MethodTypesLock); + + for (auto& Entry : LocalMethodTypes) + { + MethodTypes[Entry.first] += Entry.second; + } + }); + + HttpClient Http = CacheCommand::CreateHttpClient(m_HostName); + + uint64_t EntryIndex = EntryOffset.fetch_add(m_Stride); + while (EntryIndex < EntryCount) + { + IoBuffer Payload; + const zen::cache::RecordedRequestInfo RequestInfo = Replayer->GetRequest(EntryIndex, /* out */ Payload); + + if (RequestInfo != zen::cache::RecordedRequestInfo::NullRequest) + { + CbPackage RequestPackage; + CbObject Request; + + switch (RequestInfo.ContentType) + { + case ZenContentType::kCbPackage: + { + if (ParsePackageMessageWithLegacyFallback(Payload, RequestPackage)) + { + Request = RequestPackage.GetObject(); + } + } + break; + case ZenContentType::kCbObject: + { + Request = LoadCompactBinaryObject(Payload); + } + break; + } + + RpcAcceptOptions OriginalAcceptOptions = static_cast<RpcAcceptOptions>(Request["AcceptFlags"sv].AsUInt16(0u)); + int OriginalProcessPid = Request["Pid"sv].AsInt32(0); + + int AdjustedPid = 0; + RpcAcceptOptions AdjustedAcceptOptions = RpcAcceptOptions::kNone; + + if (!m_DisableLocalRefs) + { + if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowLocalReferences) || + m_ForceAllowLocalRefs) + { + AdjustedAcceptOptions |= RpcAcceptOptions::kAllowLocalReferences; + if (!m_DisablePartialLocalRefs) + { + if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowPartialLocalReferences) || + m_ForceAllowPartialLocalRefs) + { + AdjustedAcceptOptions |= RpcAcceptOptions::kAllowPartialLocalReferences; + } + } + if (!m_DisableLocalHandleRefs) + { + if (OriginalProcessPid != 0 || m_ForceAllowLocalHandleRef) + { + AdjustedPid = GetCurrentProcessId(); + } + } + } + } + + if (m_ShowMethodStats) + { + std::string MethodName = std::string(Request["Method"sv].AsString()); + if (auto It = LocalMethodTypes.find(MethodName); It != LocalMethodTypes.end()) + { + It->second++; + } + else + { + LocalMethodTypes[MethodName] = 1; + } + } + + if (OriginalAcceptOptions != AdjustedAcceptOptions || OriginalProcessPid != AdjustedPid) + { + CbObjectWriter RequestCopyWriter; + for (const CbFieldView& Field : Request) + { + if (!Field.HasName()) + { + RequestCopyWriter.AddField(Field); + continue; + } + std::string_view FieldName = Field.GetName(); + if (FieldName == "Pid"sv) + { + continue; + } + if (FieldName == "AcceptFlags"sv) + { + continue; + } + RequestCopyWriter.AddField(FieldName, Field); + } + if (AdjustedPid != 0) + { + RequestCopyWriter.AddInteger("Pid"sv, AdjustedPid); + } + if (AdjustedAcceptOptions != RpcAcceptOptions::kNone) + { + RequestCopyWriter.AddInteger("AcceptFlags"sv, static_cast<uint16_t>(AdjustedAcceptOptions)); + } + + if (RequestInfo.ContentType == ZenContentType::kCbPackage) + { + RequestPackage.SetObject(RequestCopyWriter.Save()); + std::vector<IoBuffer> Buffers = FormatPackageMessage(RequestPackage); + std::vector<SharedBuffer> SharedBuffers(Buffers.begin(), Buffers.end()); + Payload = CompositeBuffer(std::move(SharedBuffers)).Flatten().AsIoBuffer(); + } + else + { + RequestCopyWriter.Finalize(); + Payload = IoBuffer(RequestCopyWriter.GetSaveSize()); + RequestCopyWriter.Save(Payload.GetMutableView()); + } + } + + if (!m_DryRun) + { + Http.SetSessionId(RequestInfo.SessionId); + Payload.SetContentType(RequestInfo.ContentType); + + HttpClient::Response Response = + Http.Post("/z$/$rpc", Payload, {HttpClient::Accept(RequestInfo.AcceptType)}); + + BytesSent.fetch_add(Payload.GetSize()); + if (!Response) + { + ZEN_CONSOLE_ERROR("{}", Response); + break; + } + BytesReceived.fetch_add(Response.DownloadedBytes); + } + } + + EntryIndex = EntryOffset.fetch_add(m_Stride); + } + }, + WorkerThreadPool::EMode::EnableBacklog); + } + + while (!WorkLatch.Wait(1000)) + { + // EntryCount > m_Offset is guaranteed by the early-return above. + // EntryOffset atomically overshoots EntryCount (fetch_add past the + // end) when the workload finishes, so clamp before subtracting. + const uint64_t RequestsTotal = (EntryCount - m_Offset) / m_Stride; + const uint64_t CurrentOffset = EntryOffset.load(); + const uint64_t RequestsRemaining = CurrentOffset < EntryCount ? (EntryCount - CurrentOffset) / m_Stride : 0; + const uint64_t PercentDone = RequestsTotal > 0 ? (RequestsTotal - RequestsRemaining) * 100 / RequestsTotal : 100; + + ZEN_CONSOLE("[{:3}%] [{}] {} requests, {} remaining (sent {}, received {})", + PercentDone, + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + RequestsTotal, + RequestsRemaining, + NiceBytes(BytesSent.load()), + NiceBytes(BytesReceived.load())); + } + + if (m_ShowMethodStats) + { + for (const auto& It : MethodTypes) + { + ZEN_CONSOLE("{:18}: {:10}", It.first, It.second); + } + } + } + + const uint64_t RequestsSent = (EntryOffset.load() - m_Offset) / m_Stride; + const uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); + const uint64_t Sent = BytesSent.load(); + const uint64_t Received = BytesReceived.load(); + + ZEN_CONSOLE("Processed requests: {} ({}), payloads sent {} ({}), payloads received {} ({}) in {}.\nTotal runtime: {}", + RequestsSent, + NiceRate(RequestsSent, ElapsedMS, "req"), + NiceBytes(Sent), + NiceByteRate(Sent, ElapsedMS), + NiceBytes(Received), + NiceByteRate(Received, ElapsedMS), + NiceTimeSpanMs(ElapsedMS), + NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs())); +} + } // namespace zen diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h index 4f5b90f4d..a2834f73d 100644 --- a/src/zen/cmds/cache_cmd.h +++ b/src/zen/cmds/cache_cmd.h @@ -4,131 +4,319 @@ #include "../zen.h" +#include <filesystem> + namespace zen { -class DropCommand : public CacheStoreCommand +// Base for `cache` subcommands. Registers the shared --hosturl option and +// exposes ResolveHost() which subcommands must call before issuing HTTP +// requests (it normalises m_HostName and throws if no host could be resolved). +class CacheSubCmdBase : public ZenSubCmdBase { public: - static constexpr char Name[] = "drop"; - static constexpr char Description[] = "Drop cache namespace or bucket"; + CacheSubCmdBase(std::string_view Name, std::string_view Description); + +protected: + void ResolveHost(); + + std::string m_HostName; +}; + +class CacheDetailsSubCmd : public CacheSubCmdBase +{ +public: + CacheDetailsSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + bool m_CSV = false; + bool m_YAML = false; + bool m_Details = false; + bool m_AttachmentDetails = false; + std::string m_Namespace; + std::string m_Bucket; + std::string m_ValueKey; +}; + +class CacheDropSubCmd : public CacheSubCmdBase +{ +public: + CacheDropSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_NamespaceName; + std::string m_BucketName; +}; + +class CacheGenSubCmd : public CacheSubCmdBase +{ +public: + CacheGenSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_Namespace; + std::string m_Bucket; + uint64_t m_Count = 1; + uint64_t m_MinSize = 0; + uint64_t m_MaxSize = 0; + uint32_t m_MinAttachmentCount = 0; + uint32_t m_MaxAttachmentCount = 0; +}; + +class CacheGetSubCmd : public CacheSubCmdBase +{ +public: + CacheGetSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_Namespace; + std::string m_Bucket; + std::string m_ValueKey; + std::string m_AttachmentHash; + std::filesystem::path m_OutputPath; + bool m_AsText = false; + bool m_Decompress = true; +}; + +class CacheInfoSubCmd : public CacheSubCmdBase +{ +public: + CacheInfoSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_NamespaceName; + std::string m_SizeInfoBucketNames; + bool m_BucketSizeInfo = false; + bool m_YAML = false; + std::string m_BucketName; +}; - DropCommand(); - ~DropCommand(); +class CacheRecordSubCmd : public CacheSubCmdBase +{ +public: + CacheRecordSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_Path; +}; + +class CacheReplaySubCmd : public CacheSubCmdBase +{ +public: + CacheReplaySubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } +private: + std::string m_RecordingPath; + bool m_OnHost = false; + bool m_ShowMethodStats = false; + int m_ProcessCount = 1; + int m_ThreadCount = 0; + uint64_t m_Offset = 0; + uint64_t m_Stride = 1; + bool m_ForceAllowLocalRefs = false; + bool m_DisableLocalRefs = false; + bool m_ForceAllowLocalHandleRef = false; + bool m_DisableLocalHandleRefs = false; + bool m_ForceAllowPartialLocalRefs = false; + bool m_DisablePartialLocalRefs = false; + bool m_DryRun = false; +}; + +class CacheStatsSubCmd : public CacheSubCmdBase +{ +public: + CacheStatsSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + bool m_YAML = false; +}; + +class CacheCommand : public CacheStoreCmdWithSubCommands +{ +public: + static constexpr char Name[] = "cache"; + static constexpr char Description[] = "Manage cache - info, stats, details, get, gen, drop, record, replay"; + + CacheCommand(); + ~CacheCommand(); + + cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; + + CacheDetailsSubCmd m_DetailsSubCmd; + CacheDropSubCmd m_DropSubCmd; + CacheGenSubCmd m_GenSubCmd; + CacheGetSubCmd m_GetSubCmd; + CacheInfoSubCmd m_InfoSubCmd; + CacheRecordSubCmd m_RecordSubCmd; + CacheReplaySubCmd m_ReplaySubCmd; + CacheStatsSubCmd m_StatsSubCmd; +}; + +// --------------------------------------------------------------------------- +// Deprecated legacy top-level commands. These forward to the corresponding +// 'cache <sub>' subcommand so that existing scripts keep working. They are +// hidden from the top-level `zen --help` listing; `zen cache --help` is the +// canonical discovery surface now. + +namespace cache_legacy_shim { + void RunAs(const char* SubCommandName, const ZenCliOptions& GlobalOptions, int argc, char** argv); +} + +class DeprecatedCacheStoreCommand : public CacheStoreCommand +{ +public: + bool IsHidden() const override { return true; } +}; + +class DropCommand : public DeprecatedCacheStoreCommand +{ +public: + static constexpr char Name[] = "drop"; + static constexpr char Description[] = "(deprecated, use 'cache drop') Drop cache namespace or bucket"; + + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("drop", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_NamespaceName; - std::string m_BucketName; }; -class CacheInfoCommand : public CacheStoreCommand +class CacheInfoCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-info"; - static constexpr char Description[] = "Info on cache, namespace or bucket"; + static constexpr char Description[] = "(deprecated, use 'cache info') Info on cache, namespace or bucket"; - CacheInfoCommand(); - ~CacheInfoCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("info", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_NamespaceName; - std::string m_SizeInfoBucketNames; - bool m_BucketSizeInfo = false; - std::string m_BucketName; }; -class CacheStatsCommand : public CacheStoreCommand +class CacheStatsCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-stats"; - static constexpr char Description[] = "Stats on cache"; + static constexpr char Description[] = "(deprecated, use 'cache stats') Stats on cache"; - CacheStatsCommand(); - ~CacheStatsCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("stats", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; }; -class CacheDetailsCommand : public CacheStoreCommand +class CacheDetailsCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-details"; - static constexpr char Description[] = "Details on cache"; + static constexpr char Description[] = "(deprecated, use 'cache details') Details on cache"; - CacheDetailsCommand(); - ~CacheDetailsCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("details", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - bool m_CSV = false; - bool m_Details = false; - bool m_AttachmentDetails = false; - std::string m_Namespace; - std::string m_Bucket; - std::string m_ValueKey; }; -class CacheGenerateCommand : public CacheStoreCommand +class CacheGenerateCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-gen"; - static constexpr char Description[] = "Generates cache values into a bucket"; + static constexpr char Description[] = "(deprecated, use 'cache gen') Generates cache values into a bucket"; - CacheGenerateCommand(); - ~CacheGenerateCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("gen", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_Namespace; - std::string m_Bucket; - uint64_t m_Count = 1; - - uint64_t m_MinSize = 0; - uint64_t m_MaxSize = 0; - uint32_t m_MinAttachmentCount = 0; - uint32_t m_MaxAttachmentCount = 0; }; -class CacheGetCommand : public CacheStoreCommand +class CacheGetCommand : public DeprecatedCacheStoreCommand { public: static constexpr char Name[] = "cache-get"; - static constexpr char Description[] = "Get cache values/records or attachments from a bucket"; + static constexpr char Description[] = "(deprecated, use 'cache get') Get cache values/records or attachments from a bucket"; - CacheGetCommand(); - ~CacheGetCommand(); + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("get", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } +private: + cxxopts::Options m_Options{Name, Description}; +}; + +class RpcStartRecordingCommand : public DeprecatedCacheStoreCommand +{ +public: + static constexpr char Name[] = "rpc-record-start"; + static constexpr char Description[] = "(deprecated, use 'cache record <path>') Starts recording of cache rpc requests on a host"; + + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("record", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_Namespace; - std::string m_Bucket; - std::string m_ValueKey; - std::string m_AttachmentHash; - std::filesystem::path m_OutputPath; - bool m_AsText = false; - bool m_Decompress = true; + cxxopts::Options m_Options{Name, Description}; +}; + +class RpcStopRecordingCommand : public DeprecatedCacheStoreCommand +{ +public: + static constexpr char Name[] = "rpc-record-stop"; + static constexpr char Description[] = "(deprecated, use 'cache record stop') Stops recording of cache rpc requests on a host"; + + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; +}; + +class RpcReplayCommand : public DeprecatedCacheStoreCommand +{ +public: + static constexpr char Name[] = "rpc-record-replay"; + static constexpr char Description[] = "(deprecated, use 'cache replay') Replays a previously recorded session of rpc requests"; + + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override + { + cache_legacy_shim::RunAs("replay", GlobalOptions, argc, argv); + } + cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; }; } // namespace zen diff --git a/src/zen/cmds/compute_cmd.cpp b/src/zen/cmds/compute_cmd.cpp new file mode 100644 index 000000000..9a350c69c --- /dev/null +++ b/src/zen/cmds/compute_cmd.cpp @@ -0,0 +1,88 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "compute_cmd.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "zenserviceclient.h" + +# include <zencore/compactbinary.h> +# include <zencore/logging.h> +# include <zenhttp/httpclient.h> + +using namespace std::literals; + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +// ComputeRecordStartSubCmd + +ComputeRecordStartSubCmd::ComputeRecordStartSubCmd() : ZenSubCmdBase("record-start", "Start recording compute actions") +{ + SubOptions().add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); +} + +void +ComputeRecordStartSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "record-start"}); + HttpClient& Http = Service.Http(); + if (HttpClient::Response Response = Http.Post("/compute/record/start"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) + { + CbObject Obj = Response.AsObject(); + std::string_view Path = Obj["path"sv].AsString(); + ZEN_CONSOLE("recording started: " ZEN_BRIGHT_GREEN("{}"), Path); + } + else + { + Response.ThrowError("Failed to start recording"); + } +} + +////////////////////////////////////////////////////////////////////////// +// ComputeRecordStopSubCmd + +ComputeRecordStopSubCmd::ComputeRecordStopSubCmd() : ZenSubCmdBase("record-stop", "Stop recording compute actions") +{ + SubOptions().add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); +} + +void +ComputeRecordStopSubCmd::Run(const ZenCliOptions& GlobalOptions) +{ + ZEN_UNUSED(GlobalOptions); + + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "record-stop"}); + HttpClient& Http = Service.Http(); + if (HttpClient::Response Response = Http.Post("/compute/record/stop"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) + { + CbObject Obj = Response.AsObject(); + std::string_view Path = Obj["path"sv].AsString(); + ZEN_CONSOLE("recording stopped: " ZEN_BRIGHT_GREEN("{}"), Path); + } + else + { + Response.ThrowError("Failed to stop recording"); + } +} + +////////////////////////////////////////////////////////////////////////// +// ComputeCommand + +ComputeCommand::ComputeCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), ""); + m_Options.parse_positional({"subcommand"}); + + AddSubCommand(m_RecordStartSubCmd); + AddSubCommand(m_RecordStopSubCmd); +} + +ComputeCommand::~ComputeCommand() = default; + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zen/cmds/compute_cmd.h b/src/zen/cmds/compute_cmd.h new file mode 100644 index 000000000..b26f639c4 --- /dev/null +++ b/src/zen/cmds/compute_cmd.h @@ -0,0 +1,53 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +#include <string> + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen { + +class ComputeRecordStartSubCmd : public ZenSubCmdBase +{ +public: + ComputeRecordStartSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_HostName; +}; + +class ComputeRecordStopSubCmd : public ZenSubCmdBase +{ +public: + ComputeRecordStopSubCmd(); + void Run(const ZenCliOptions& GlobalOptions) override; + +private: + std::string m_HostName; +}; + +class ComputeCommand : public ZenCmdWithSubCommands +{ +public: + static constexpr char Name[] = "compute"; + static constexpr char Description[] = "Compute service operations"; + + ComputeCommand(); + ~ComputeCommand(); + + cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; + std::string m_SubCommand; + ComputeRecordStartSubCmd m_RecordStartSubCmd; + ComputeRecordStopSubCmd m_RecordStopSubCmd; +}; + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp deleted file mode 100644 index 530661607..000000000 --- a/src/zen/cmds/copy_cmd.cpp +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "copy_cmd.h" - -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/string.h> -#include <zencore/timer.h> - -namespace zen { - -CopyCommand::CopyCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_options()("no-clone", "Do not perform block clone", cxxopts::value(m_NoClone)->default_value("false")); - m_Options.add_options()("must-clone", - "Always perform block clone (fails if clone is not possible)", - cxxopts::value(m_MustClone)->default_value("false")); - m_Options.add_option("", "s", "source", "Copy source", cxxopts::value(m_CopySource), "<file/directory>"); - m_Options.add_option("", "t", "target", "Copy target", cxxopts::value(m_CopyTarget), "<file/directory>"); - m_Options.parse_positional({"source", "target"}); -} - -CopyCommand::~CopyCommand() = default; - -void -CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return; - } - - // Validate arguments - - if (m_CopySource.empty()) - throw OptionParseException("'--source' is required", m_Options.help()); - - if (m_CopyTarget.empty()) - throw OptionParseException("'--target' is required", m_Options.help()); - - std::filesystem::path FromPath = m_CopySource; - std::filesystem::path ToPath = m_CopyTarget; - - std::error_code Ec; - std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec); - - if (!Ec) - { - std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec); - - if (!Ec) - { - if (FromCanonical == ToCanonical) - { - throw std::runtime_error("Target and source must be distinct files or directories"); - } - } - } - - const bool IsFileCopy = IsFile(m_CopySource); - const bool IsDirCopy = IsDir(m_CopySource); - - if (!IsFileCopy && !IsDirCopy) - { - throw std::runtime_error("Invalid source specification (neither directory nor file)"); - } - - if (IsFileCopy && IsDirCopy) - { - throw std::runtime_error("Invalid source specification (both directory AND file!?)"); - } - - if (IsDirCopy) - { - if (IsFile(ToPath)) - { - throw std::runtime_error("Attempted copy of directory into file"); - } - - if (!IsDir(ToPath)) - { - CreateDirectories(ToPath); - } - - std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec); - - if (!Ec) - { - if (ToCanonical.generic_string().starts_with(FromCanonical.generic_string()) || - FromCanonical.generic_string().starts_with(ToCanonical.generic_string())) - { - throw std::runtime_error("Invalid parent/child relationship for source/target directories"); - } - } - - // Multi file copy - - ZEN_CONSOLE("copying {} -> {}", FromPath, ToPath); - - zen::Stopwatch Timer; - - struct CopyVisitor : public FileSystemTraversal::TreeVisitor - { - CopyVisitor(std::filesystem::path InBasePath, zen::CopyFileOptions InCopyOptions) - : BasePath(InBasePath) - , CopyOptions(InCopyOptions) - { - } - - virtual void VisitFile(const std::filesystem::path& Parent, - const path_view& File, - uint64_t FileSize, - uint32_t, - uint64_t) override - { - ZEN_UNUSED(FileSize); - std::error_code Ec; - const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec); - - if (Ec) - { - FailedFileCount++; - } - else - { - const std::filesystem::path FromPath = Parent / File; - const std::filesystem::path ToPath = TargetPath / Relative / File; - - try - { - zen::CreateDirectories(TargetPath / Relative); - if (zen::CopyFile(FromPath, ToPath, CopyOptions)) - { - ++FileCount; - ByteCount += FileSize; - } - else - { - throw std::logic_error("CopyFile failed in an unexpected way"); - } - } - catch (const std::exception& Ex) - { - ++FailedFileCount; - - ZEN_CONSOLE_ERROR("Failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); - } - } - } - - virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; } - - std::filesystem::path BasePath; - std::filesystem::path TargetPath; - zen::CopyFileOptions CopyOptions; - int FileCount = 0; - uint64_t ByteCount = 0; - int FailedFileCount = 0; - }; - - zen::CopyFileOptions CopyOptions; - CopyOptions.EnableClone = !m_NoClone; - CopyOptions.MustClone = m_MustClone; - - CopyVisitor Visitor{FromPath, CopyOptions}; - Visitor.TargetPath = ToPath; - - FileSystemTraversal Traversal; - Traversal.TraverseFileSystem(FromPath, Visitor); - - ZEN_CONSOLE("Copy of {} files ({}) completed in {} ({})", - Visitor.FileCount, - NiceBytes(Visitor.ByteCount), - zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - zen::NiceRate(Visitor.ByteCount, (uint32_t)Timer.GetElapsedTimeMs())); - - if (Visitor.FailedFileCount) - { - throw std::runtime_error(fmt::format("{} file copy operations FAILED", Visitor.FailedFileCount)); - } - } - else - { - // Single file copy - - zen::Stopwatch Timer; - - zen::CopyFileOptions CopyOptions; - CopyOptions.EnableClone = !m_NoClone; - - zen::CreateDirectories(ToPath.parent_path()); - if (zen::CopyFile(FromPath, ToPath, CopyOptions)) - { - ZEN_CONSOLE("Copy completed in {}", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - } - else - { - throw std::runtime_error(fmt::format("Failed to copy '{}' to '{}'", FromPath, ToPath)); - } - } -} - -} // namespace zen diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h deleted file mode 100644 index 757a8e691..000000000 --- a/src/zen/cmds/copy_cmd.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -/** Copy files, possibly using block cloning - */ -class CopyCommand : public ZenCmdBase -{ -public: - static constexpr char Name[] = "copy"; - static constexpr char Description[] = "Copy file(s)"; - - CopyCommand(); - ~CopyCommand(); - - virtual cxxopts::Options& Options() override { return m_Options; } - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } - -private: - cxxopts::Options m_Options{Name, Description}; - std::filesystem::path m_CopySource; - std::filesystem::path m_CopyTarget; - bool m_NoClone = false; - bool m_MustClone = false; -}; - -} // namespace zen diff --git a/src/zen/cmds/dedup_cmd.cpp b/src/zen/cmds/dedup_cmd.cpp index 9ef50a97d..18ad56aec 100644 --- a/src/zen/cmds/dedup_cmd.cpp +++ b/src/zen/cmds/dedup_cmd.cpp @@ -240,7 +240,12 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { zen::BLAKE3Stream b3s; - zen::ScanFile(Entry.path(), 64 * 1024, [&](const void* Data, size_t Size) { b3s.Append(Data, Size); }); + if (std::error_code ScanEc = + zen::ScanFile(Entry.path(), 64 * 1024, [&](const void* Data, size_t Size) { b3s.Append(Data, Size); }); + ScanEc) + { + throw std::system_error(ScanEc, fmt::format("Failed to scan file '{}'", Entry.path())); + } Hash = b3s.GetHash(); } @@ -279,7 +284,11 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Options.EnableClone = true; Options.MustClone = true; - zen::CopyFile(Dupe->path(), Entry.path(), Options); + if (std::error_code Ec = zen::CopyFile(Dupe->path(), Entry.path(), Options); Ec) + { + ZEN_ERROR("Failed to clone '{}' to '{}': {}", Dupe->path(), Entry.path(), Ec.message()); + continue; + } DupeBytes += Entry.file_size(); } diff --git a/src/zen/cmds/exec_cmd.cpp b/src/zen/cmds/exec_cmd.cpp index 9719fce77..60968b521 100644 --- a/src/zen/cmds/exec_cmd.cpp +++ b/src/zen/cmds/exec_cmd.cpp @@ -2,6 +2,8 @@ #include "exec_cmd.h" +#include "zenserviceclient.h" + #include <zencompute/computeservice.h> #include <zencompute/recordingreader.h> #include <zencore/compactbinary.h> @@ -23,6 +25,8 @@ #include <zenhttp/httpclient.h> #include <zenhttp/packageformat.h> +#include "consoleprogress.h" + #include <EASTL/hash_map.h> #include <EASTL/hash_set.h> #include <EASTL/map.h> @@ -114,7 +118,7 @@ namespace { } // namespace ////////////////////////////////////////////////////////////////////////// -// ExecSessionConfig — read-only configuration for a session run +// ExecSessionConfig - read-only configuration for a session run struct ExecSessionConfig { @@ -124,17 +128,18 @@ struct ExecSessionConfig std::vector<ExecFunctionDefinition>& FunctionList; // mutable for EmitFunctionListOnce std::string_view OrchestratorUrl; const std::filesystem::path& OutputPath; - int Offset = 0; - int Stride = 1; - int Limit = 0; - bool Verbose = false; - bool Quiet = false; - bool DumpActions = false; - bool Binary = false; + int Offset = 0; + int Stride = 1; + int Limit = 0; + bool Verbose = false; + bool Quiet = false; + bool DumpActions = false; + bool Binary = false; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; }; ////////////////////////////////////////////////////////////////////////// -// ExecSessionRunner — owns per-run state, drives the session lifecycle +// ExecSessionRunner - owns per-run state, drives the session lifecycle class ExecSessionRunner { @@ -345,8 +350,6 @@ ExecSessionRunner::DrainCompletedJobs() } m_PendingJobs.Remove(CompleteLsn); - - ZEN_CONSOLE("completed: LSN {} ({} still pending)", CompleteLsn, m_PendingJobs.GetSize()); } } } @@ -897,17 +900,22 @@ ExecSessionRunner::Run() // Then submit work items - int FailedWorkCounter = 0; - size_t RemainingWorkItems = m_Config.RecordingReader.GetActionCount(); - int SubmittedWorkItems = 0; + std::atomic<int> FailedWorkCounter{0}; + std::atomic<size_t> RemainingWorkItems{m_Config.RecordingReader.GetActionCount()}; + std::atomic<int> SubmittedWorkItems{0}; + size_t TotalWorkItems = RemainingWorkItems.load(); - ZEN_CONSOLE("submitting {} work items", RemainingWorkItems); + std::unique_ptr<ProgressBase> ProgressOwner(CreateConsoleProgress(m_Config.ProgressMode)); + std::unique_ptr<ProgressBase::ProgressBar> SubmitProgress = ProgressOwner->CreateProgressBar("Submit"); + SubmitProgress->UpdateState( + {.Task = "Submitting work items", .TotalCount = TotalWorkItems, .RemainingCount = RemainingWorkItems.load()}, + false); int OffsetCounter = m_Config.Offset; int StrideCounter = m_Config.Stride; auto ShouldSchedule = [&]() -> bool { - if (m_Config.Limit && SubmittedWorkItems >= m_Config.Limit) + if (m_Config.Limit && SubmittedWorkItems.load() >= m_Config.Limit) { // Limit reached, ignore @@ -1005,17 +1013,14 @@ ExecSessionRunner::Run() { const int32_t LsnField = EnqueueResult.Lsn; - --RemainingWorkItems; - ++SubmittedWorkItems; + size_t Remaining = --RemainingWorkItems; + int Submitted = ++SubmittedWorkItems; - if (!m_Config.Quiet) - { - ZEN_CONSOLE("submitted work item #{} - LSN {} - {}. {} remaining", - SubmittedWorkItems, - LsnField, - NiceTimeSpanMs(SubmitTimer.GetElapsedTimeMs()), - RemainingWorkItems); - } + SubmitProgress->UpdateState({.Task = "Submitting work items", + .Details = fmt::format("#{} LSN {}", Submitted, LsnField), + .TotalCount = TotalWorkItems, + .RemainingCount = Remaining}, + false); if (!m_Config.OutputPath.empty()) { @@ -1055,22 +1060,37 @@ ExecSessionRunner::Run() }, TargetParallelism); + SubmitProgress->Finish(); + // Wait until all pending work is complete + size_t TotalPendingJobs = m_PendingJobs.GetSize(); + + std::unique_ptr<ProgressBase::ProgressBar> CompletionProgress = ProgressOwner->CreateProgressBar("Execute"); + while (!m_PendingJobs.IsEmpty()) { - // TODO: improve this logic - zen::Sleep(500); + size_t PendingCount = m_PendingJobs.GetSize(); + CompletionProgress->UpdateState( + {.Task = "Executing work items", + .Details = fmt::format("{} completed, {} remaining", TotalPendingJobs - PendingCount, PendingCount), + .TotalCount = TotalPendingJobs, + .RemainingCount = PendingCount}, + false); + + zen::Sleep(ProgressOwner->GetProgressUpdateDelayMS()); DrainCompletedJobs(); SendOrchestratorHeartbeat(); } + CompletionProgress->Finish(); + // Write summary files WriteSummaryFiles(); - if (FailedWorkCounter) + if (FailedWorkCounter.load()) { return 1; } @@ -1089,7 +1109,7 @@ ExecHttpSubCmd::ExecHttpSubCmd(ExecCommand& Parent) : ZenSubCmdBase("http", "For void ExecHttpSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = ExecCommand::Name}); ZEN_ASSERT(m_Parent.m_ChunkResolver); ChunkResolver& Resolver = *m_Parent.m_ChunkResolver; @@ -1097,7 +1117,7 @@ ExecHttpSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); zen::compute::ComputeServiceSession ComputeSession(Resolver); - ComputeSession.AddRemoteRunner(Resolver, TempPath, m_HostName); + ComputeSession.AddRemoteRunner(Resolver, TempPath, Service.HostSpec()); Stopwatch ExecTimer; int ReturnValue = m_Parent.RunSession(ComputeSession); @@ -1423,6 +1443,12 @@ ExecCommand::OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) int ExecCommand::RunSession(zen::compute::ComputeServiceSession& ComputeSession, std::string_view OrchestratorUrl) { + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; + if (m_QuietLogging) + { + ProgressMode = ConsoleProgressMode::Quiet; + } + ExecSessionConfig Config{ .Resolver = *m_ChunkResolver, .RecordingReader = *m_RecordingReader, @@ -1437,6 +1463,7 @@ ExecCommand::RunSession(zen::compute::ComputeServiceSession& ComputeSession, std .Quiet = m_QuietLogging, .DumpActions = m_DumpActions, .Binary = m_Binary, + .ProgressMode = ProgressMode, }; ExecSessionRunner Runner(ComputeSession, Config); diff --git a/src/zen/cmds/history_cmd.cpp b/src/zen/cmds/history_cmd.cpp new file mode 100644 index 000000000..27faae1eb --- /dev/null +++ b/src/zen/cmds/history_cmd.cpp @@ -0,0 +1,228 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "history_cmd.h" + +#include <zenbase/zenbase.h> +#include <zencore/except_fmt.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/process.h> +#include <zenutil/consoletui.h> +#include <zenutil/invocationhistory.h> + +#include <algorithm> + +namespace zen { + +HistoryCommand::HistoryCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_options()("filter", + "Filter by executable ('zen' or 'zenserver' / 'server')", + cxxopts::value<std::string>(m_Filter)->default_value("")); + m_Options.add_options()("print", "Print the selected command line instead of running it", cxxopts::value<bool>(m_Print)); + m_Options.add_options()("l,list", "List all invocations to stdout instead of showing the picker", cxxopts::value<bool>(m_List)); +} + +HistoryCommand::~HistoryCommand() +{ +} + +namespace { + + bool KeepRecord(const HistoryRecord& Rec, std::string_view Filter) + { + if (Filter.empty()) + { + return true; + } + if (Filter == "zen") + { + return Rec.Exe == "zen"; + } + if (Filter == "zenserver" || Filter == "server") + { + return Rec.Exe == "zenserver"; + } + return true; + } + + std::string BuildLabel(const HistoryRecord& Rec, int32_t TerminalCols) + { + constexpr int32_t kIndicator = 3; // " > " or " " prefix from TuiPickOne + constexpr int32_t kEllipsis = 3; // "..." + + std::string Exe = Rec.Exe.empty() ? std::string("?") : Rec.Exe; + std::string Lbl = fmt::format("{} {:<9} pid {:<7}", Rec.Ts, Exe, Rec.Pid); + + if (!Rec.CmdLine.empty()) + { + int32_t Available = TerminalCols - kIndicator - 2 - static_cast<int32_t>(Lbl.size()); + if (Available > kEllipsis) + { + Lbl += " "; + if (static_cast<int32_t>(Rec.CmdLine.size()) <= Available) + { + Lbl += Rec.CmdLine; + } + else + { + Lbl.append(Rec.CmdLine, 0, static_cast<size_t>(Available - kEllipsis)); + Lbl += "..."; + } + } + } + return Lbl; + } + + int SpawnZen(const HistoryRecord& Rec) + { + const std::filesystem::path TargetPath = GetRunningExecutablePath(); + ZEN_CONSOLE("Running: {}", Rec.CmdLine); + + CreateProcOptions Opts{}; + CreateProcResult Result = CreateProc(TargetPath, Rec.CmdLine, Opts); + if (!Result) + { + throw zen::runtime_error("failed to launch '{}'", TargetPath); + } + + ProcessHandle Proc; + Proc.Initialize(Result); + return Proc.WaitExitCode(); + } + + void SpawnZenServerDetached(const HistoryRecord& Rec) + { + const std::filesystem::path TargetPath = GetRunningExecutablePath().parent_path() / ("zenserver" ZEN_EXE_SUFFIX_LITERAL); + + ZEN_CONSOLE("Launching detached: {}", Rec.CmdLine); + + CreateProcOptions Opts{}; + Opts.Flags = CreateProcOptions::Flag_NoConsole | CreateProcOptions::Flag_NewProcessGroup; + CreateProcResult Result = CreateProc(TargetPath, Rec.CmdLine, Opts); + if (!Result) + { + throw zen::runtime_error("failed to launch '{}'", TargetPath); + } + +#if ZEN_PLATFORM_WINDOWS + // Take ownership of the handle just to release it; we do not wait. + ProcessHandle Proc; + Proc.Initialize(Result); + ZEN_CONSOLE("Launched {} (pid {})", TargetPath, Proc.Pid()); +#else + ZEN_CONSOLE("Launched {} (pid {})", TargetPath, static_cast<int>(Result)); +#endif + } + + void PrintPlainTable(const std::vector<HistoryRecord>& Records) + { + if (Records.empty()) + { + ZEN_CONSOLE("No invocation history available."); + return; + } + for (const HistoryRecord& Rec : Records) + { + std::string Line = fmt::format("{} {:<9} pid {:<7}", Rec.Ts, Rec.Exe, Rec.Pid); + if (!Rec.CmdLine.empty()) + { + Line += " "; + Line += Rec.CmdLine; + } + ZEN_CONSOLE("{}", Line); + } + } + +} // namespace + +void +HistoryCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return; + } + + if (m_List && m_Print) + { + throw OptionParseException("'--list' conflicts with '--print'", m_Options.help()); + } + + std::vector<HistoryRecord> All = ReadInvocationHistory(); + + std::vector<HistoryRecord> Filtered; + Filtered.reserve(All.size()); + for (HistoryRecord& Rec : All) + { + if (KeepRecord(Rec, m_Filter)) + { + Filtered.push_back(std::move(Rec)); + } + } + + std::reverse(Filtered.begin(), Filtered.end()); + + if (Filtered.empty()) + { + ZEN_CONSOLE("No invocation history available{}.", m_Filter.empty() ? "" : " for this filter"); + return; + } + + if (m_List || !IsTuiAvailable()) + { + PrintPlainTable(Filtered); + return; + } + + const int32_t Cols = static_cast<int32_t>(TuiConsoleColumns()); + std::vector<std::string> Labels; + Labels.reserve(Filtered.size()); + for (const HistoryRecord& Rec : Filtered) + { + Labels.push_back(BuildLabel(Rec, Cols)); + } + + int Selected = TuiPickOne("Recent invocations. Select one to re-run:", Labels); + if (Selected < 0) + { + return; + } + + const HistoryRecord& Pick = Filtered[Selected]; + + if (m_Print) + { + ZEN_CONSOLE("{}", Pick.CmdLine); + return; + } + + if (Pick.CmdLine.empty()) + { + throw zen::runtime_error("selected record has no command line"); + } + + if (Pick.Exe == "zenserver") + { + SpawnZenServerDetached(Pick); + return; + } + + if (Pick.Exe == "zen") + { + int ExitCode = SpawnZen(Pick); + if (ExitCode != 0) + { + throw ErrorWithReturnCode(fmt::format("zen exited with code {}", ExitCode), ExitCode); + } + return; + } + + throw zen::runtime_error("unknown executable '{}' in history record", Pick.Exe); +} + +} // namespace zen diff --git a/src/zen/cmds/trace_cmd.h b/src/zen/cmds/history_cmd.h index 6eb0ba22b..d7dd2f078 100644 --- a/src/zen/cmds/trace_cmd.h +++ b/src/zen/cmds/history_cmd.h @@ -6,24 +6,23 @@ namespace zen { -class TraceCommand : public ZenCmdBase +class HistoryCommand : public ZenCmdBase { public: - static constexpr char Name[] = "trace"; - static constexpr char Description[] = "Control zen realtime tracing"; + HistoryCommand(); + ~HistoryCommand(); - TraceCommand(); - ~TraceCommand(); + static constexpr char Name[] = "history"; + static constexpr char Description[] = "List recent zen/zenserver invocations and optionally re-run one"; virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - bool m_Stop = false; - std::string m_TraceHost; - std::string m_TraceFile; + std::string m_Filter; + bool m_Print = false; + bool m_List = false; }; } // namespace zen diff --git a/src/zen/cmds/hub_cmd.cpp b/src/zen/cmds/hub_cmd.cpp index 5bdd3a922..75408a5e9 100644 --- a/src/zen/cmds/hub_cmd.cpp +++ b/src/zen/cmds/hub_cmd.cpp @@ -2,6 +2,8 @@ #include "hub_cmd.h" +#include "zenserviceclient.h" + #include <zencore/compactbinary.h> #include <zencore/compactbinaryutil.h> #include <zencore/filesystem.h> @@ -210,17 +212,13 @@ HubProvisionSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } if (m_ModuleId.empty()) { throw OptionParseException("moduleid is required", SubOptions().help()); } - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Resp = Http.Post(fmt::format("/hub/modules/{}/provision", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) { @@ -251,17 +249,13 @@ HubDeprovisionSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } if (m_ModuleId.empty()) { throw OptionParseException("moduleid is required", SubOptions().help()); } - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Resp = Http.Post(fmt::format("/hub/modules/{}/deprovision", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) { @@ -290,17 +284,13 @@ HubHibernateSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } if (m_ModuleId.empty()) { throw OptionParseException("moduleid is required", SubOptions().help()); } - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Resp = Http.Post(fmt::format("/hub/modules/{}/hibernate", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) { @@ -329,17 +319,13 @@ HubWakeSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } if (m_ModuleId.empty()) { throw OptionParseException("moduleid is required", SubOptions().help()); } - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Resp = Http.Post(fmt::format("/hub/modules/{}/wake", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{})) { @@ -369,13 +355,8 @@ HubStatusSubCmd::Run(const ZenCliOptions& GlobalOptions) { ZEN_UNUSED(GlobalOptions); - m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName); - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve hub host specification", SubOptions().help()); - } - - HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name}); + HttpClient& Http = Service.Http(); if (!m_ModuleId.empty()) { diff --git a/src/zen/cmds/info_cmd.cpp b/src/zen/cmds/info_cmd.cpp index 9faad5691..b147b93c3 100644 --- a/src/zen/cmds/info_cmd.cpp +++ b/src/zen/cmds/info_cmd.cpp @@ -2,6 +2,8 @@ #include "info_cmd.h" +#include "zenserviceclient.h" + #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/string.h> @@ -31,14 +33,8 @@ InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/admin/info", HttpClient::Accept(ZenContentType::kJSON))) { @@ -46,7 +42,7 @@ InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - Result.ThrowError(fmt::format("Failed getting info from {}", m_HostName)); + Result.ThrowError(fmt::format("Failed getting info from {}", Service.HostSpec())); } } diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index d31c34fd0..c6a3434f8 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -2,16 +2,21 @@ #include "projectstore_cmd.h" +#include "zenserviceclient.h" + #include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinaryutil.h> #include <zencore/compress.h> +#include <zencore/except_fmt.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> +#include <zencore/logging/broadcastsink.h> #include <zencore/parallelwork.h> #include <zencore/process.h> #include <zencore/scopeguard.h> +#include <zencore/session.h> #include <zencore/stream.h> #include <zencore/timer.h> #include <zencore/workthreadpool.h> @@ -21,16 +26,20 @@ #include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> #include <zenremotestore/builds/buildstoragecache.h> +#include <zenremotestore/builds/buildstorageresolve.h> #include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/builds/jupiterbuildstorage.h> #include <zenremotestore/jupiter/jupiterhost.h> -#include <zenremotestore/operationlogoutput.h> #include <zenremotestore/projectstore/projectstoreoperations.h> #include <zenremotestore/projectstore/remoteprojectstore.h> #include <zenremotestore/transferthreadworkers.h> +#include <zenutil/authutils.h> +#include <zenutil/logging.h> +#include <zenutil/progress.h> +#include <zenutil/sessionsclient.h> #include <zenutil/workerpools.h> -#include "../progressbar.h" +#include "consoleprogress.h" ZEN_THIRD_PARTY_INCLUDES_START #include <json11.hpp> @@ -112,6 +121,26 @@ namespace projectstore_impl { } } + // `OplogMirrorCommand::Run` uses a latching boolean flag rather than the + // SignalCounter above, because it drives a worker pool that aborts on any + // interrupt. Kept separate from SignalCallbackHandler so neither interferes + // with the other when both are installed in the same process. + static std::atomic<bool> MirrorAbortFlag{false}; + + static void MirrorSignalCallbackHandler(int SigNum) + { + if (SigNum == SIGINT) + { + MirrorAbortFlag.store(true); + } +#if ZEN_PLATFORM_WINDOWS + if (SigNum == SIGBREAK) + { + MirrorAbortFlag.store(true); + } +#endif + } + void ExecuteAsyncOperation(HttpClient& Http, std::string_view Url, IoBuffer&& Payload, bool PlainProgress) { signal(SIGINT, SignalCallbackHandler); @@ -131,13 +160,16 @@ namespace projectstore_impl { throw std::runtime_error(fmt::format("invalid job id returned, received '{}'", JobIdText)); } - ProgressBar ProgressBar(PlainProgress ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty, ""sv); + std::unique_ptr<ProgressBase> ProgressOwner( + CreateConsoleProgress(PlainProgress ? ConsoleProgressMode::Plain : ConsoleProgressMode::Pretty)); + std::unique_ptr<ProgressBase::ProgressBar> Bar = ProgressOwner->CreateProgressBar(""sv); + std::string ActiveTask; - auto OuputMessages = [&](CbObjectView StatusObject) { + auto OutputMessages = [&](CbObjectView StatusObject) { CbArrayView Messages = StatusObject["Messages"sv].AsArrayView(); if (Messages.Num() > 0) { - ProgressBar.ForceLinebreak(); + Bar->ForceLinebreak(); for (auto M : Messages) { std::string_view Message = M.AsString(); @@ -169,33 +201,36 @@ namespace projectstore_impl { uint64_t RemainingCount = StatusObject["RemainingCount"sv].AsUInt64(); uint64_t ProgressElapsedTimeMs = StatusObject["ProgressElapsedTimeMs"sv].AsUInt64((uint64_t)-1); - if (!ProgressBar.IsSameTask(CurrentOp)) + if (ActiveTask != CurrentOp) { - ProgressBar.Finish(); + Bar->Finish(); + ActiveTask = ""; } - if (!ProgressBar.HasActiveTask()) + if (ActiveTask.empty()) { - OuputMessages(StatusObject); + OutputMessages(StatusObject); MessagesDone = true; + ActiveTask = std::string(CurrentOp); } - ProgressBar.UpdateState({.Task = std::string(CurrentOp), - .Details = std::string(CurrentOpDetails), - .TotalCount = TotalCount, - .RemainingCount = RemainingCount, - .OptionalElapsedTime = ProgressElapsedTimeMs}, - false); + Bar->UpdateState({.Task = std::string(CurrentOp), + .Details = std::string(CurrentOpDetails), + .TotalCount = TotalCount, + .RemainingCount = RemainingCount, + .OptionalElapsedTime = ProgressElapsedTimeMs}, + false); } if ((Status == "Complete") || (Status == "Aborted")) { - ProgressBar.Finish(); + Bar->Finish(); + ActiveTask = ""; } if (!MessagesDone) { - OuputMessages(StatusObject); + OutputMessages(StatusObject); } if (Status == "Complete") @@ -246,13 +281,13 @@ namespace projectstore_impl { #endif // ZEN_PLATFORM_WINDOWS if (HttpClient::Response DeleteResult = Http.Delete(fmt::format("/admin/jobs/{}", JobId))) { - ProgressBar.ForceLinebreak(); + Bar->ForceLinebreak(); ZEN_CONSOLE("Requested cancel..."); Cancelled = true; } else { - ProgressBar.ForceLinebreak(); + Bar->ForceLinebreak(); ZEN_CONSOLE("Failed cancelling job {}", DeleteResult); } continue; @@ -530,14 +565,8 @@ DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -569,7 +598,7 @@ DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_OplogName = ResolveOplog(Http, m_ProjectName, m_OplogName); if (m_OplogName.empty()) { - throw std::runtime_error(fmt::format("Can't find oplog in project '{}'", m_OplogName, m_ProjectName)); + throw zen::runtime_error("Can't find oplog '{}' in project '{}'", m_OplogName, m_ProjectName); } if (m_DryRun) { @@ -620,19 +649,13 @@ ProjectInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (!m_OplogName.empty() && m_ProjectName.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); std::string Url; if (m_ProjectName.empty()) @@ -709,19 +732,13 @@ CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (m_ProjectId.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); std::string Url = fmt::format("/prj/{}", m_ProjectId); @@ -779,20 +796,14 @@ CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (m_ProjectId.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ProjectId = ResolveProject(Http, m_ProjectId); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ProjectId = ResolveProject(Http, m_ProjectId); if (m_ProjectId.empty()) { throw std::runtime_error("Project can not be found"); @@ -1010,20 +1021,14 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_BoostWorkerMemory = true; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (m_ProjectName.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ProjectName = ResolveProject(Http, m_ProjectName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) { throw std::runtime_error("Project can not be found"); @@ -1114,7 +1119,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg std::string TargetUrlBase = m_ZenUrl; if (TargetUrlBase.find("://") == std::string::npos) { - // Assume https URL + // Assume http URL TargetUrlBase = fmt::format("http://{}", TargetUrlBase); } @@ -1303,7 +1308,14 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg std::filesystem::path MetadataPath(m_BuildsMetadataPath); IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); - CbFieldIterator MetaData = LoadCompactBinaryFromJson(Json); + std::string JsonError; + CbFieldIterator MetaData = LoadCompactBinaryFromJson(Json, JsonError); + if (!JsonError.empty()) + { + throw zen::runtime_error("builds metadata file '{}' is malformed. Reason: '{}'", + MetadataPath.string(), + JsonError); + } Writer.AddBinary("metadata"sv, MetaData.GetBuffer()); } if (!m_BuildsMetadata.empty()) @@ -1313,7 +1325,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg size_t SplitPos = Pair.find('='); if (SplitPos == std::string::npos || SplitPos == 0) { - throw std::runtime_error(fmt::format("builds metadata key-value pair '{}' is malformed", Pair)); + throw zen::runtime_error("builds metadata key-value pair '{}' is malformed", Pair); } MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); return true; @@ -1323,7 +1335,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } } Writer.EndObject(); // "builds" - TargetDescription = fmt::format("[builds] {}/{}/{}/{}", m_CloudUrl, m_JupiterNamespace, m_JupiterBucket, m_BuildsId); + TargetDescription = fmt::format("[builds] {}/{}/{}/{}", m_BuildsUrl, m_JupiterNamespace, m_JupiterBucket, m_BuildsId); } if (!m_ZenUrl.empty()) { @@ -1517,13 +1529,6 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_BoostWorkerMemory = true; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - if (m_ProjectName.empty()) { throw OptionParseException("'--project' is required", m_Options.help()); @@ -1541,8 +1546,9 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_Options.help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ProjectName = ResolveProject(Http, m_ProjectName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) { throw std::runtime_error("Project can not be found"); @@ -1738,7 +1744,11 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } } Writer.EndObject(); // "builds" - SourceDescription = fmt::format("[builds] {}/{}/{}/{}", m_CloudUrl, m_JupiterNamespace, m_JupiterBucket, m_BuildsId); + SourceDescription = fmt::format("[builds] {}/{}/{}/{}", + m_BuildsHost.empty() ? m_BuildsOverrideHost : m_BuildsHost, + m_JupiterNamespace, + m_JupiterBucket, + m_BuildsId); } if (!m_ZenUrl.empty()) { @@ -1806,14 +1816,8 @@ SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (m_ProjectName.empty()) { @@ -1869,14 +1873,8 @@ ProjectStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/stats/prj", HttpClient::Accept(ZenContentType::kJSON))) { ZEN_CONSOLE("{}", Result.ToText()); @@ -1922,14 +1920,8 @@ ProjectOpDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char* return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -2038,14 +2030,8 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -2108,16 +2094,59 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg std::unordered_set<std::u8string> FileNames; std::atomic<uint64_t> WrittenByteCount = 0; - std::atomic<bool> AbortFlag(false); + // Install Ctrl-C handler so SIGINT aborts the worker pool rather than killing + // the process. Without this the local AbortFlag would shadow whatever global + // handler is installed elsewhere and interrupts would be dropped. RAII so + // the previous handler is restored when the function returns or throws. + MirrorAbortFlag.store(false); + ScopedSignalHandler SigIntGuard(SIGINT, MirrorSignalCallbackHandler); +#if ZEN_PLATFORM_WINDOWS + ScopedSignalHandler SigBreakGuard(SIGBREAK, MirrorSignalCallbackHandler); +#endif + std::atomic<bool>& AbortFlag = MirrorAbortFlag; Stopwatch WriteStopWatch; + // Filenames come from the remote oplog, which may be compromised or untrusted. + // Reject anything that could escape the mirror root via an absolute path, drive + // letter / UNC / device path prefix, or '..' component before it is joined to + // RootPath. Returns nullptr when the filename is safe. + auto UnsafeFileNameReason = [](const std::filesystem::path& FileName) -> const char* { + if (FileName.empty()) + { + return "filename is empty"; + } + if (FileName.has_root_name()) + { + return "filename has a root name (drive letter, UNC share, or device path)"; + } + if (FileName.has_root_directory()) + { + return "filename is absolute"; + } + for (const std::filesystem::path& Component : FileName) + { + const std::u8string C = Component.u8string(); + if (C.empty() || C == u8"..") + { + return "filename contains a '..' or empty component"; + } + } + return nullptr; + }; + auto EmitFilesForDataArray = [&](CbArrayView DataArray) { for (auto DataIter : DataArray) { if (CbObjectView Data = DataIter.AsObjectView()) { std::filesystem::path FileName(Data["filename"sv].AsU8String()); + if (const char* Reason = UnsafeFileNameReason(FileName)) + { + ZEN_CONSOLE_ERROR("Rejecting unsafe filename '{}' from remote oplog: {}", FileName.string(), Reason); + AbortFlag.store(true); + break; + } if (!m_FilenameFilter.empty()) { std::string FileNameLowerCase = ToLower(FileName.string()); @@ -2162,7 +2191,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg IoBuffer ChunkData = m_Decompress ? TryDecompress(ChunkResponse.ResponsePayload) : ChunkResponse.ResponsePayload; - if (!MoveToFile(TargetPath, ChunkData)) + if (std::error_code MoveEc = MoveToFile(TargetPath, ChunkData); MoveEc) { WriteFile(TargetPath, ChunkData); } @@ -2196,17 +2225,18 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg ZEN_CONSOLE("Fetched oplog in {}", NiceTimeSpanMs(uint64_t(Response.ElapsedSeconds * 1000.0))); if (CbObject ResponseObject = Response.AsObject()) { - std::unique_ptr<ProgressBar> EmitProgressBar; + std::unique_ptr<ProgressBase> ProgressOwner2(CreateConsoleProgress(ConsoleProgressMode::Pretty)); + std::unique_ptr<ProgressBase::ProgressBar> EmitProgressBar; { - ProgressBar ParseProgressBar(ProgressBar::Mode::Pretty, ""); - CbArrayView Entries = ResponseObject["entries"sv].AsArrayView(); - uint64_t Remaining = Entries.Num(); + std::unique_ptr<ProgressBase::ProgressBar> ParseProgressBar = ProgressOwner2->CreateProgressBar(""); + CbArrayView Entries = ResponseObject["entries"sv].AsArrayView(); + uint64_t Remaining = Entries.Num(); for (auto EntryIter : Entries) { if (!AbortFlag) { CbObjectView Entry = EntryIter.AsObjectView(); - ParseProgressBar.UpdateState( + ParseProgressBar->UpdateState( {.Task = "Parsing oplog", .Details = "", .TotalCount = Entries.Num(), .RemainingCount = Remaining}, false); Remaining--; @@ -2219,7 +2249,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } if (!EmitProgressBar) { - EmitProgressBar = std::make_unique<ProgressBar>(ProgressBar::Mode::Pretty, ""sv); + EmitProgressBar = ProgressOwner2->CreateProgressBar(""sv); WriteStopWatch.Reset(); } @@ -2229,7 +2259,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg ++OplogEntryCount; } } - ParseProgressBar.Finish(); + ParseProgressBar->Finish(); } WorkRemaining.CountDown(); @@ -2262,7 +2292,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg if (AbortFlag) { - throw std::runtime_error("Failed top mirror oplog"); + throw std::runtime_error("Failed to mirror oplog"); } } else @@ -2306,14 +2336,8 @@ OplogValidateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -2472,7 +2496,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a }; ParseSystemOptions(); - ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; auto ParseOutputOptions = [&]() { if (m_Verbose && m_Quiet) @@ -2494,23 +2518,19 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a if (m_LogProgress) { - ProgressMode = ProgressBar::Mode::Log; + ProgressMode = ConsoleProgressMode::Log; } else if (m_PlainProgress) { - ProgressMode = ProgressBar::Mode::Plain; - } - else if (m_Verbose) - { - ProgressMode = ProgressBar::Mode::Plain; + ProgressMode = ConsoleProgressMode::Plain; } else if (m_Quiet) { - ProgressMode = ProgressBar::Mode::Quiet; + ProgressMode = ConsoleProgressMode::Quiet; } else { - ProgressMode = ProgressBar::Mode::Pretty; + ProgressMode = ConsoleProgressMode::Pretty; } }; ParseOutputOptions(); @@ -2565,7 +2585,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a m_BoostWorkerMemory = true; } - std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode)); TransferThreadWorkers Workers(m_BoostWorkerCount, /*SingleThreaded*/ false); if (!m_Quiet) @@ -2594,7 +2614,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a /*Hidden*/ false, m_Verbose); - BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*OperationLogOutput, + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(), ClientSettings, m_Host, m_OverrideHost, @@ -2629,6 +2649,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a .MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u}, [&AbortFlag]() { return AbortFlag.load(); }); Storage.CacheHost = ResolveRes.Cache; + + Storage.SetupCacheSession(ResolveRes.Cache.Address, Name, GetSessionId()); } if (!m_Quiet) @@ -2677,7 +2699,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } ProjectStoreOperationOplogState State( - *OperationLogOutput, + ConsoleLog(), Storage, BuildId, {.IsQuiet = m_Quiet, .IsVerbose = m_Verbose, .ForceDownload = m_ForceDownload, .TempFolderPath = StorageTempPath}); @@ -2704,7 +2726,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } std::atomic<bool> PauseFlag; - ProjectStoreOperationDownloadAttachments Op(*OperationLogOutput, + ProjectStoreOperationDownloadAttachments Op(ConsoleLog(), + *Progress, Storage, AbortFlag, PauseFlag, diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index 1ba98b39e..41db36139 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -217,7 +217,7 @@ class SnapshotOplogCommand : public ProjectStoreCommand { public: static constexpr char Name[] = "oplog-snapshot"; - static constexpr char Description[] = "Snapshot project store oplog"; + static constexpr char Description[] = "Copy oplog's loose files on disk into zenserver"; SnapshotOplogCommand(); ~SnapshotOplogCommand(); diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp deleted file mode 100644 index 3bf81a9df..000000000 --- a/src/zen/cmds/rpcreplay_cmd.cpp +++ /dev/null @@ -1,486 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "rpcreplay_cmd.h" - -#include <zencore/compactbinarybuilder.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/process.h> -#include <zencore/scopeguard.h> -#include <zencore/session.h> -#include <zencore/stream.h> -#include <zencore/timer.h> -#include <zencore/workthreadpool.h> -#include <zenhttp/formatters.h> -#include <zenhttp/httpclient.h> -#include <zenhttp/httpcommon.h> -#include <zenhttp/packageformat.h> -#include <zenutil/rpcrecording.h> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <fmt/format.h> -#include <gsl/gsl-lite.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - -#include <memory> - -namespace zen { - -using namespace std::literals; - -RpcStartRecordingCommand::RpcStartRecordingCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), "<path>"); - - m_Options.parse_positional("path"); -} - -RpcStartRecordingCommand::~RpcStartRecordingCommand() = default; - -void -RpcStartRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - if (m_RecordingPath.empty()) - { - throw OptionParseException("'--path' is required", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = - Http.Post("/z$/exec$/start-recording"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap({{"path", m_RecordingPath}}))) - { - ZEN_CONSOLE("{}", Response.ToText()); - } - else - { - Response.ThrowError("Failed to start recording"); - } -} - -//////////////////////////////////////////////////// - -RpcStopRecordingCommand::RpcStopRecordingCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); -} - -RpcStopRecordingCommand::~RpcStopRecordingCommand() = default; - -void -RpcStopRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = Http.Post("/z$/exec$/stop-recording"sv)) - { - ZEN_CONSOLE("{}", Response.ToText()); - } - else - { - Response.ThrowError("Failed to stop recording"); - } -} - -//////////////////////////////////////////////////// - -RpcReplayCommand::RpcReplayCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), "<path>"); - m_Options.add_option("", "", "dry", "Do a dry run", cxxopts::value(m_DryRun), "<enable>"); - m_Options.add_option("", - "w", - "numthreads", - "Number of worker threads per process", - cxxopts::value(m_ThreadCount)->default_value(fmt::format("{}", GetHardwareConcurrency())), - "<count>"); - m_Options.add_option("", "", "onhost", "Replay on host, bypassing http/network layer", cxxopts::value(m_OnHost), "<onhost>"); - m_Options.add_option("", - "", - "showmethodstats", - "Show statistics of which RPC methods are used", - cxxopts::value(m_ShowMethodStats), - "<showmethodstats>"); - m_Options.add_option("", - "", - "offset", - "Offset into request recording to start replay", - cxxopts::value(m_Offset)->default_value("0"), - "<offset>"); - m_Options.add_option("", - "", - "stride", - "Stride for request recording when replaying requests", - cxxopts::value(m_Stride)->default_value("1"), - "<stride>"); - m_Options.add_option("", "", "numproc", "Number of worker processes", cxxopts::value(m_ProcessCount)->default_value("1"), "<count>"); - m_Options.add_option("", - "", - "forceallowlocalrefs", - "Force enable local refs in requests", - cxxopts::value(m_ForceAllowLocalRefs), - "<enable>"); - m_Options - .add_option("", "", "disablelocalrefs", "Force disable local refs in requests", cxxopts::value(m_DisableLocalRefs), "<enable>"); - m_Options.add_option("", - "", - "forceallowlocalhandlerefs", - "Force enable local refs as handles in requests", - cxxopts::value(m_ForceAllowLocalHandleRef), - "<enable>"); - m_Options.add_option("", - "", - "disablelocalhandlerefs", - "Force disable local refs as handles in requests", - cxxopts::value(m_DisableLocalHandleRefs), - "<enable>"); - m_Options.add_option("", - "", - "forceallowpartiallocalrefs", - "Force enable local refs for all sizes", - cxxopts::value(m_ForceAllowPartialLocalRefs), - "<enable>"); - m_Options.add_option("", - "", - "disablepartiallocalrefs", - "Force disable local refs for all sizes", - cxxopts::value(m_DisablePartialLocalRefs), - "<enable>"); - - m_Options.parse_positional("path"); -} - -RpcReplayCommand::~RpcReplayCommand() = default; - -void -RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - if (m_RecordingPath.empty()) - { - throw OptionParseException("'--path' is required", m_Options.help()); - } - - if (!IsDir(m_RecordingPath)) - { - throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath)); - } - - m_ThreadCount = Max(m_ThreadCount, 1); - - ZEN_CONSOLE("Replay '{}' (start offset {}, stride {}) to '{}', {} threads", - m_RecordingPath, - m_Offset, - m_Stride, - m_HostName, - m_ThreadCount); - - Stopwatch TotalTimer; - - if (m_OnHost) - { - HttpClient Http = CreateHttpClient(m_HostName); - if (HttpClient::Response Response = - Http.Post("/z$/exec$/replay-recording"sv, - HttpClient::KeyValueMap{}, - HttpClient::KeyValueMap({{"path", m_RecordingPath}, {"thread-count", fmt::format("{}", m_ThreadCount)}}))) - { - ZEN_CONSOLE("{}", Response.ToText()); - - return; - } - else - { - Response.ThrowError("Failed to start replay"); - } - } - - std::unique_ptr<cache::IRpcRequestReplayer> Replayer = cache::MakeDiskRequestReplayer(m_RecordingPath, true); - uint64_t EntryCount = Replayer->GetRequestCount(); - - std::atomic_uint64_t EntryOffset = m_Offset; - std::atomic_uint64_t BytesSent = 0; - std::atomic_uint64_t BytesReceived = 0; - - Stopwatch Timer; - - if (m_ProcessCount > 1) - { - std::vector<std::unique_ptr<ProcessHandle>> WorkerProcesses; - WorkerProcesses.resize(m_ProcessCount); - - ProcessMonitor Monitor; - for (int ProcessIndex = 0; ProcessIndex < m_ProcessCount; ++ProcessIndex) - { - std::string CommandLine = - fmt::format("{} rpc-record-replay --hosturl {} --path \"{}\" --offset {} --stride {} --numthreads {} --numproc {}"sv, - argv[0], - m_HostName, - m_RecordingPath, - m_Stride == 1 ? 0 : m_Offset + ProcessIndex, - m_Stride, - m_ThreadCount, - 1); - CreateProcResult Result(CreateProc(std::filesystem::path(std::string(argv[0])), CommandLine)); - WorkerProcesses[ProcessIndex] = std::make_unique<ProcessHandle>(); - WorkerProcesses[ProcessIndex]->Initialize(Result); - Monitor.AddPid(WorkerProcesses[ProcessIndex]->Pid()); - } - while (Monitor.IsRunning()) - { - ZEN_CONSOLE("Waiting for worker processes..."); - Sleep(1000); - } - return; - } - else - { - std::map<std::string, size_t> MethodTypes; - RwLock MethodTypesLock; - - WorkerThreadPool WorkerPool(m_ThreadCount); - - Latch WorkLatch(m_ThreadCount); - for (int WorkerIndex = 0; WorkerIndex < m_ThreadCount; ++WorkerIndex) - { - WorkerPool.ScheduleWork( - [this, &WorkLatch, EntryCount, &EntryOffset, &Replayer, &BytesSent, &BytesReceived, &MethodTypes, &MethodTypesLock]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - - std::map<std::string, size_t> LocalMethodTypes; - - auto ReduceTypes = MakeGuard([&] { - RwLock::ExclusiveLockScope __(MethodTypesLock); - - for (auto& Entry : LocalMethodTypes) - { - MethodTypes[Entry.first] += Entry.second; - } - }); - - HttpClient Http = CreateHttpClient(m_HostName); - - uint64_t EntryIndex = EntryOffset.fetch_add(m_Stride); - while (EntryIndex < EntryCount) - { - IoBuffer Payload; - const zen::cache::RecordedRequestInfo RequestInfo = Replayer->GetRequest(EntryIndex, /* out */ Payload); - - if (RequestInfo != zen::cache::RecordedRequestInfo::NullRequest) - { - CbPackage RequestPackage; - CbObject Request; - - switch (RequestInfo.ContentType) - { - case ZenContentType::kCbPackage: - { - if (ParsePackageMessageWithLegacyFallback(Payload, RequestPackage)) - { - Request = RequestPackage.GetObject(); - } - } - break; - case ZenContentType::kCbObject: - { - Request = LoadCompactBinaryObject(Payload); - } - break; - } - - RpcAcceptOptions OriginalAcceptOptions = static_cast<RpcAcceptOptions>(Request["AcceptFlags"sv].AsUInt16(0u)); - int OriginalProcessPid = Request["Pid"sv].AsInt32(0); - - int AdjustedPid = 0; - RpcAcceptOptions AdjustedAcceptOptions = RpcAcceptOptions::kNone; - - if (!m_DisableLocalRefs) - { - if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowLocalReferences) || - m_ForceAllowLocalRefs) - { - AdjustedAcceptOptions |= RpcAcceptOptions::kAllowLocalReferences; - if (!m_DisablePartialLocalRefs) - { - if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowPartialLocalReferences) || - m_ForceAllowPartialLocalRefs) - { - AdjustedAcceptOptions |= RpcAcceptOptions::kAllowPartialLocalReferences; - } - } - if (!m_DisableLocalHandleRefs) - { - if (OriginalProcessPid != 0 || m_ForceAllowLocalHandleRef) - { - AdjustedPid = GetCurrentProcessId(); - } - } - } - } - - if (m_ShowMethodStats) - { - std::string MethodName = std::string(Request["Method"sv].AsString()); - if (auto It = LocalMethodTypes.find(MethodName); It != LocalMethodTypes.end()) - { - It->second++; - } - else - { - LocalMethodTypes[MethodName] = 1; - } - } - - if (OriginalAcceptOptions != AdjustedAcceptOptions || OriginalProcessPid != AdjustedPid) - { - CbObjectWriter RequestCopyWriter; - for (const CbFieldView& Field : Request) - { - if (!Field.HasName()) - { - RequestCopyWriter.AddField(Field); - continue; - } - std::string_view FieldName = Field.GetName(); - if (FieldName == "Pid"sv) - { - continue; - } - if (FieldName == "AcceptFlags"sv) - { - continue; - } - RequestCopyWriter.AddField(FieldName, Field); - } - if (AdjustedPid != 0) - { - RequestCopyWriter.AddInteger("Pid"sv, AdjustedPid); - } - if (AdjustedAcceptOptions != RpcAcceptOptions::kNone) - { - RequestCopyWriter.AddInteger("AcceptFlags"sv, static_cast<uint16_t>(AdjustedAcceptOptions)); - } - - if (RequestInfo.ContentType == ZenContentType::kCbPackage) - { - RequestPackage.SetObject(RequestCopyWriter.Save()); - std::vector<IoBuffer> Buffers = FormatPackageMessage(RequestPackage); - std::vector<SharedBuffer> SharedBuffers(Buffers.begin(), Buffers.end()); - Payload = CompositeBuffer(std::move(SharedBuffers)).Flatten().AsIoBuffer(); - } - else - { - RequestCopyWriter.Finalize(); - Payload = IoBuffer(RequestCopyWriter.GetSaveSize()); - RequestCopyWriter.Save(Payload.GetMutableView()); - } - } - - if (!m_DryRun) - { - Http.SetSessionId(RequestInfo.SessionId); - Payload.SetContentType(RequestInfo.ContentType); - - HttpClient::Response Response = - Http.Post("/z$/$rpc", Payload, {HttpClient::Accept(RequestInfo.AcceptType)}); - - BytesSent.fetch_add(Payload.GetSize()); - if (!Response) - { - ZEN_CONSOLE_ERROR("{}", Response); - break; - } - BytesReceived.fetch_add(Response.DownloadedBytes); - } - } - - EntryIndex = EntryOffset.fetch_add(m_Stride); - } - }, - WorkerThreadPool::EMode::EnableBacklog); - } - - while (!WorkLatch.Wait(1000)) - { - const uint64_t RequestsTotal = (EntryCount - m_Offset) / m_Stride; - const uint64_t RequestsRemaining = (EntryCount - EntryOffset.load()) / m_Stride; - - ZEN_CONSOLE("[{:3}%] [{}] {} requests, {} remaining (sent {}, received {})", - (RequestsTotal - RequestsRemaining) * 100 / RequestsTotal, - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - RequestsTotal, - RequestsRemaining, - NiceBytes(BytesSent.load()), - NiceBytes(BytesReceived.load())); - } - - if (m_ShowMethodStats) - { - for (const auto& It : MethodTypes) - { - ZEN_CONSOLE("{:18}: {:10}", It.first, It.second); - } - } - } - - const uint64_t RequestsSent = (EntryOffset.load() - m_Offset) / m_Stride; - const uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); - const uint64_t Sent = BytesSent.load(); - const uint64_t Received = BytesReceived.load(); - - ZEN_CONSOLE("Processed requests: {} ({}), payloads sent {} ({}), payloads received {} ({}) in {}.\nTotal runtime: {}", - RequestsSent, - NiceRate(RequestsSent, ElapsedMS, "req"), - NiceBytes(Sent), - NiceByteRate(Sent, ElapsedMS), - NiceBytes(Received), - NiceByteRate(Received, ElapsedMS), - NiceTimeSpanMs(ElapsedMS), - NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs())); -} - -} // namespace zen diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h deleted file mode 100644 index 332a3126c..000000000 --- a/src/zen/cmds/rpcreplay_cmd.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class RpcStartRecordingCommand : public CacheStoreCommand -{ -public: - static constexpr char Name[] = "rpc-record-start"; - static constexpr char Description[] = "Starts recording of cache rpc requests on a host"; - - RpcStartRecordingCommand(); - ~RpcStartRecordingCommand(); - - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_RecordingPath; -}; - -class RpcStopRecordingCommand : public CacheStoreCommand -{ -public: - static constexpr char Name[] = "rpc-record-stop"; - static constexpr char Description[] = "Stops recording of cache rpc requests on a host"; - - RpcStopRecordingCommand(); - ~RpcStopRecordingCommand(); - - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; -}; - -class RpcReplayCommand : public CacheStoreCommand -{ -public: - static constexpr char Name[] = "rpc-record-replay"; - static constexpr char Description[] = "Replays a previously recorded session of rpc requests"; - - RpcReplayCommand(); - ~RpcReplayCommand(); - - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_RecordingPath; - bool m_OnHost = false; - bool m_ShowMethodStats = false; - int m_ProcessCount = 1; - int m_ThreadCount = 0; - uint64_t m_Offset = 0; - uint64_t m_Stride = 1; - bool m_ForceAllowLocalRefs = false; - bool m_DisableLocalRefs = false; - bool m_ForceAllowLocalHandleRef = false; - bool m_DisableLocalHandleRefs = false; - bool m_ForceAllowPartialLocalRefs = false; - bool m_DisablePartialLocalRefs = false; - bool m_DryRun = false; -}; - -} // namespace zen diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp deleted file mode 100644 index ee47eb9f3..000000000 --- a/src/zen/cmds/run_cmd.cpp +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "run_cmd.h" - -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/process.h> -#include <zencore/string.h> -#include <zencore/timer.h> - -using namespace std::literals; - -#define ZEN_COLOR_BLACK "\033[0;30m" -#define ZEN_COLOR_RED "\033[0;31m" -#define ZEN_COLOR_GREEN "\033[0;32m" -#define ZEN_COLOR_YELLOW "\033[0;33m" -#define ZEN_COLOR_BLUE "\033[0;34m" -#define ZEN_COLOR_MAGENTA "\033[0;35m" -#define ZEN_COLOR_CYAN "\033[0;36m" -#define ZEN_COLOR_WHITE "\033[0;37m" - -#define ZEN_BRIGHT_COLOR_BLACK "\033[1;30m" -#define ZEN_BRIGHT_COLOR_RED "\033[1;31m" -#define ZEN_BRIGHT_COLOR_GREEN "\033[1;32m" -#define ZEN_BRIGHT_COLOR_YELLOW "\033[1;33m" -#define ZEN_BRIGHT_COLOR_BLUE "\033[1;34m" -#define ZEN_BRIGHT_COLOR_MAGENTA "\033[1;35m" -#define ZEN_BRIGHT_COLOR_CYAN "\033[1;36m" -#define ZEN_BRIGHT_COLOR_WHITE "\033[1;37m" - -#define ZEN_COLOR_RESET "\033[0m" - -namespace zen { - -RunCommand::RunCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "n", "count", "Number of times to run command", cxxopts::value(m_RunCount), "<count>"); - m_Options.add_option("", "t", "time", "How long to run command(s) for", cxxopts::value(m_RunTime), "<seconds>"); - m_Options.add_option("", - "", - "basepath", - "Where to run command. Each run will execute in its own numbered subdirectory below this directory. Additionally, " - "stdout will be redirected to a file in the provided directory", - cxxopts::value(m_BaseDirectory), - "<path>"); - m_Options.add_option("", - "", - "max-dirs", - "Number of base directories to retain on rotation", - cxxopts::value(m_MaxBaseDirectoryCount), - "<count>"); -} - -RunCommand::~RunCommand() -{ -} - -void -RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - if (!ParseOptions(argc, argv)) - { - return; - } - - // Validate arguments - - if (GlobalOptions.PassthroughArgV.empty() || GlobalOptions.PassthroughArgV[0].empty()) - throw OptionParseException("No command specified. The command to run is passed in after a double dash ('--') on the command line", - m_Options.help()); - - if (m_RunCount < 0) - throw OptionParseException(fmt::format("'--count' ('{}') is invalid", m_RunCount), m_Options.help()); - - if (m_RunTime < -1 || m_RunTime == 0) - throw OptionParseException(fmt::format("'--time' ('{}') is invalid", m_RunTime), m_Options.help()); - - if (m_MaxBaseDirectoryCount < 0) - throw OptionParseException(fmt::format("'--max-dirs' ('{}') is invalid", m_MaxBaseDirectoryCount), m_Options.help()); - - if (m_RunTime > 0 && m_RunCount > 0) - throw OptionParseException(fmt::format("'--time' ('{}') conflicts with '--count' ('{}') ", m_RunTime, m_RunCount), - m_Options.help()); - - if (m_RunCount == 0) - m_RunCount = 1; - - std::filesystem::path BaseDirectory; - - if (!m_BaseDirectory.empty()) - { - BaseDirectory = m_BaseDirectory; - - if (m_MaxBaseDirectoryCount) - { - RotateDirectories(BaseDirectory, m_MaxBaseDirectoryCount); - CreateDirectories(BaseDirectory); - } - else - { - CleanDirectory(BaseDirectory, /*ForceRemoveReadOnlyFiles*/ false); - } - } - - bool TimedRun = false; - auto CommandDeadlineTime = std::chrono::system_clock::now(); - - if (m_RunTime > 0) - { - m_RunCount = 1'000'000; - TimedRun = true; - - CommandDeadlineTime += std::chrono::seconds(m_RunTime); - } - - struct RunResults - { - int ExitCode = 0; - std::chrono::duration<long, std::milli> Duration{}; - }; - - std::vector<RunResults> Results; - int ErrorCount = 0; - - std::filesystem::path ExecutablePath = SearchPathForExecutable(GlobalOptions.PassthroughArgV[0]); - std::string CommandArguments = GlobalOptions.PassthroughArgs; - - for (int i = 0; i < m_RunCount; ++i) - { - std::filesystem::path RunDir; - if (!BaseDirectory.empty()) - { - RunDir = BaseDirectory / IntNum(i + 1).c_str(); - CreateDirectories(RunDir); - } - - Stopwatch Timer; - - CreateProcOptions ProcOptions; - - if (!RunDir.empty()) - { - ProcOptions.WorkingDirectory = &RunDir; - ProcOptions.StdoutFile = RunDir / "stdout.txt"; - } - - fmt::print(ZEN_BRIGHT_COLOR_WHITE "run #{}" ZEN_COLOR_RESET ": {}\n", i + 1, GlobalOptions.PassthroughCommandLine); - - ProcessHandle Proc; - Proc.Initialize(CreateProc(ExecutablePath, GlobalOptions.PassthroughCommandLine, ProcOptions)); - if (!Proc.IsValid()) - { - throw std::runtime_error(fmt::format("failed to launch '{}'", ExecutablePath)); - } - - int ExitCode = Proc.WaitExitCode(); - - auto EndTime = std::chrono::system_clock::now(); - - if (ExitCode) - ++ErrorCount; - - Results.emplace_back(RunResults{.ExitCode = ExitCode, .Duration = std::chrono::milliseconds(Timer.GetElapsedTimeMs())}); - - if (TimedRun) - { - if (EndTime >= CommandDeadlineTime) - { - m_RunCount = i + 1; - break; - } - } - } - - fmt::print("{:>5} {:>3} {:>6}\n", "run", "rc", "time"); - int i = 0; - for (const auto& Entry : Results) - { - fmt::print("{:5} {:3} {:>6}\n", ++i, Entry.ExitCode, NiceTimeSpanMs(Entry.Duration.count())); - } - - if (ErrorCount) - { - fmt::print("run complete ({}/{} failed)\n", ErrorCount, m_RunCount); - } - else - { - fmt::print("run complete, no error exit code\n", m_RunCount); - } -} - -} // namespace zen diff --git a/src/zen/cmds/run_cmd.h b/src/zen/cmds/run_cmd.h deleted file mode 100644 index 300c08c5b..000000000 --- a/src/zen/cmds/run_cmd.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class RunCommand : public ZenCmdBase -{ -public: - static constexpr char Name[] = "run"; - static constexpr char Description[] = "Run command with special options"; - - RunCommand(); - ~RunCommand(); - - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } - -private: - cxxopts::Options m_Options{Name, Description}; - int m_RunCount = 0; - int m_RunTime = -1; - std::string m_BaseDirectory; - int m_MaxBaseDirectoryCount = 10; -}; - -} // namespace zen diff --git a/src/zen/cmds/service_cmd.cpp b/src/zen/cmds/service_cmd.cpp index 37baf5483..c43c4e614 100644 --- a/src/zen/cmds/service_cmd.cpp +++ b/src/zen/cmds/service_cmd.cpp @@ -320,7 +320,7 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("'verb' option is required", m_Options.help()); } - // Parse subcommand permissively — forward unrecognised options to the parent parser. + // Parse subcommand permissively - forward unrecognised options to the parent parser. std::vector<std::string> SubUnmatched; if (!ParseOptionsPermissive(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data(), SubUnmatched)) { @@ -500,9 +500,9 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { std::filesystem::path Destination = m_InstallPath / File.filename(); - if (!CopyFile(File, Destination, {.EnableClone = false})) + if (std::error_code CopyEc = CopyFile(File, Destination, {.EnableClone = false}); CopyEc) { - throw std::runtime_error(fmt::format("Failed to copy '{}' to '{}'", File, Destination)); + throw std::system_error(CopyEc, fmt::format("Failed to copy '{}' to '{}'", File, Destination)); } ZEN_INFO("Copied '{}' to '{}'", File, Destination); diff --git a/src/zen/cmds/trace_cmd.cpp b/src/zen/cmds/trace_cmd.cpp deleted file mode 100644 index 54c0f080d..000000000 --- a/src/zen/cmds/trace_cmd.cpp +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "trace_cmd.h" -#include <zencore/logging.h> -#include <zenhttp/httpclient.h> -#include <zenhttp/httpcommon.h> - -using namespace std::literals; - -namespace zen { - -TraceCommand::TraceCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "s", "stop", "Stop tracing", cxxopts::value(m_Stop)->default_value("false"), "<stop>"); - m_Options.add_option("", "", "host", "Start tracing to host", cxxopts::value(m_TraceHost), "<hostip>"); - m_Options.add_option("", "", "file", "Start tracing to file", cxxopts::value(m_TraceFile), "<filepath>"); -} - -TraceCommand::~TraceCommand() = default; - -void -TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } - - zen::HttpClient Http = CreateHttpClient(m_HostName); - - if (m_Stop) - { - if (zen::HttpClient::Response Response = Http.Post("/admin/trace/stop"sv)) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - } - else - { - Response.ThrowError("Trace stop failed"); - } - return; - } - - std::string StartArg; - if (!m_TraceHost.empty()) - { - StartArg = fmt::format("host={}", m_TraceHost); - } - else if (!m_TraceFile.empty()) - { - StartArg = fmt::format("file={}", m_TraceFile); - } - - if (!StartArg.empty()) - { - if (zen::HttpClient::Response Response = Http.Post(fmt::format("/admin/trace/start?{}"sv, StartArg))) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - } - else - { - Response.ThrowError("Trace start failed"); - } - } - else - { - if (zen::HttpClient::Response Response = Http.Get("/admin/trace"sv)) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - } - else - { - Response.ThrowError("Trace status failed"); - } - } -} - -} // namespace zen diff --git a/src/zen/cmds/ui_cmd.cpp b/src/zen/cmds/ui_cmd.cpp index 4846b4d18..53dbb22da 100644 --- a/src/zen/cmds/ui_cmd.cpp +++ b/src/zen/cmds/ui_cmd.cpp @@ -2,6 +2,9 @@ #include "ui_cmd.h" +#include "browser_launcher.h" +#include "zenserviceclient.h" + #include <zencore/except_fmt.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> @@ -9,11 +12,6 @@ #include <zenutil/consoletui.h> #include <zenutil/zenserverprocess.h> -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -# include <shellapi.h> -#endif - namespace zen { namespace { @@ -83,40 +81,10 @@ UiCommand::OpenBrowser(std::string_view HostName) } } - bool Success = false; - ExtendableStringBuilder<256> FullUrl; FullUrl << HostName << m_DashboardPath; -#if ZEN_PLATFORM_WINDOWS - HINSTANCE Result = ShellExecuteA(nullptr, "open", FullUrl.c_str(), nullptr, nullptr, SW_SHOWNORMAL); - Success = reinterpret_cast<intptr_t>(Result) > 32; -#else - // Validate URL doesn't contain shell metacharacters that could lead to command injection - std::string_view FullUrlView = FullUrl; - constexpr std::string_view DangerousChars = ";|&$`\\\"'<>(){}[]!#*?~\n\r"; - if (FullUrlView.find_first_of(DangerousChars) != std::string_view::npos) - { - throw OptionParseException(fmt::format("URL contains invalid characters: '{}'", FullUrl), m_Options.help()); - } - -# if ZEN_PLATFORM_MAC - std::string Command = fmt::format("open \"{}\"", FullUrl); -# elif ZEN_PLATFORM_LINUX - std::string Command = fmt::format("xdg-open \"{}\"", FullUrl); -# else - ZEN_NOT_IMPLEMENTED("Browser launching not implemented on this platform"); -# endif - - Success = system(Command.c_str()) == 0; -#endif - - if (!Success) - { - throw zen::runtime_error("Failed to launch browser for '{}'", FullUrl); - } - - ZEN_CONSOLE("Web browser launched for '{}' successfully", FullUrl); + LaunchBrowser(std::string_view(FullUrl)); } void @@ -162,7 +130,7 @@ UiCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Labels.push_back(fmt::format("(all {} instances)", Servers.size())); const int32_t Cols = static_cast<int32_t>(TuiConsoleColumns()); - constexpr int32_t kIndicator = 3; // " ▶ " or " " prefix + constexpr int32_t kIndicator = 3; // " > " or " " prefix constexpr int32_t kSeparator = 2; // " " before cmdline constexpr int32_t kEllipsis = 3; // "..." @@ -225,17 +193,14 @@ UiCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_HostName = ResolveTargetHostSpec(m_HostName, ServerPort); } - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - } + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); - if (IsUnixSocketSpec(m_HostName)) + if (IsUnixSocketSpec(Service.HostSpec())) { throw std::runtime_error("Cannot open browser for a Unix domain socket connection"); } - OpenBrowser(m_HostName); + OpenBrowser(Service.HostSpec()); } } // namespace zen diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index 809a41bb6..1f23e6819 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -2,6 +2,7 @@ #include "up_cmd.h" +#include <zencore/basicfile.h> #include <zencore/compactbinary.h> #include <zencore/compactbinaryutil.h> #include <zencore/filesystem.h> @@ -12,6 +13,57 @@ namespace zen { +namespace { + + bool TryShutdownByPid(ZenServerState& Instance, uint32_t Pid, const std::filesystem::path& ProgramBaseDir, bool ForceTerminate) + { + Instance.Sweep(); + + uint16_t DesiredPort = 0; + Instance.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + if (Entry.Pid.load() == Pid) + { + DesiredPort = Entry.DesiredListenPort.load(); + } + }); + + ZenServerState::ZenServerEntry* Entry = (DesiredPort != 0) ? Instance.Lookup(DesiredPort) : nullptr; + if (Entry && Entry->Pid.load() != Pid) + { + Entry = nullptr; + } + + if (Entry) + { + if (ShutdownZenServer(ConsoleLog(), Instance, Entry, ProgramBaseDir)) + { + return true; + } + } + + std::error_code Ec; + ProcessHandle Proc; + Proc.Initialize(int(Pid), Ec); + if (!Ec && Proc.IsValid() && !Proc.IsRunning()) + { + return true; + } + + if (ForceTerminate && !Ec && Proc.IsValid() && Proc.IsRunning()) + { + ZEN_CONSOLE_WARN("Hard terminating zen process with pid ({})", Pid); + if (Proc.Terminate(0)) + { + ZEN_CONSOLE("Terminate complete"); + return true; + } + } + + return false; + } + +} // namespace + UpCommand::UpCommand() { m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("0"), "<hostport>"); @@ -40,12 +92,19 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("'--show-console' conflicts with '--show-log'", m_Options.help()); } + if (m_ProgramBaseDir.empty()) + { + m_ProgramBaseDir = GetRunningExecutablePath().parent_path(); + } + MakeSafeAbsolutePathInPlace(m_ProgramBaseDir); + std::optional<int> StartResult = StartupZenServer(ConsoleLog(), - {.ProgramBaseDir = m_ProgramBaseDir, - .Port = m_Port, - .OpenConsole = m_ShowConsole, - .ShowLog = m_ShowLog, - .ExtraArgs = GlobalOptions.PassthroughCommandLine}); + {.ProgramBaseDir = m_ProgramBaseDir, + .Port = m_Port, + .OpenConsole = m_ShowConsole, + .ShowLog = m_ShowLog, + .ExtraArgs = GlobalOptions.PassthroughCommandLine, + .EnableExecutionHistory = GlobalOptions.EnableExecutionHistory}); if (!StartResult.has_value()) { ZEN_CONSOLE("Zen server already running"); @@ -80,6 +139,11 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } + if (!m_DataDir.empty()) + { + MakeSafeAbsolutePathInPlace(m_DataDir); + } + ZenServerState Instance; Instance.Initialize(); Instance.Sweep(); @@ -87,9 +151,9 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_DataDir.empty()) { - if (!IsFile(m_DataDir / ".lock")) + if (!LockFile::IsHeldLive(m_DataDir / ".lock", /*AttemptCleanup*/ false)) { - throw std::runtime_error(fmt::format("Lock file does not exist in directory '{}'", m_DataDir)); + throw std::runtime_error(fmt::format("No live zen server holding lock file in directory '{}'", m_DataDir)); } CbValidateError ValidateResult = CbValidateError::None; if (CbObject LockFileObject = @@ -134,6 +198,13 @@ DownCommand::DownCommand() m_Options.add_option("", "f", "force", "Force terminate if graceful shutdown fails", cxxopts::value(m_ForceTerminate), "<force>"); m_Options.add_option("", "b", "base-dir", "Parent folder of server executable", cxxopts::value(m_ProgramBaseDir), "<directory>"); m_Options.add_option("", "", "data-dir", "Path to data directory to inspect for running server", cxxopts::value(m_DataDir), "<file>"); + m_Options.add_option("", "", "pid", "Shut down zen server process by PID", cxxopts::value(m_Pid)->default_value("0"), "<pid>"); + m_Options.add_option("", + "", + "executable", + "Shut down all zen server processes matching executable path", + cxxopts::value(m_ExecutablePath), + "<path>"); } DownCommand::~DownCommand() = default; @@ -148,16 +219,77 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } + const bool HasPid = m_Pid != 0; + const bool HasExecutable = !m_ExecutablePath.empty(); + const bool HasDataDir = !m_DataDir.empty(); + const int SelectorCount = int(m_All) + int(HasPid) + int(HasExecutable) + int(HasDataDir); + if (SelectorCount > 1) + { + throw OptionParseException("--all, --pid, --executable, and --data-dir are mutually exclusive", m_Options.help()); + } + if (m_ProgramBaseDir.empty()) { std::filesystem::path ExePath = GetRunningExecutablePath(); m_ProgramBaseDir = ExePath.parent_path(); } + MakeSafeAbsolutePathInPlace(m_ProgramBaseDir); + if (!m_DataDir.empty()) + { + MakeSafeAbsolutePathInPlace(m_DataDir); + } + if (!m_ExecutablePath.empty() && m_ExecutablePath.has_parent_path()) + { + MakeSafeAbsolutePathInPlace(m_ExecutablePath); + } // Discover executing instances ZenServerState Instance; Instance.Initialize(); + if (HasPid) + { + if (!TryShutdownByPid(Instance, m_Pid, m_ProgramBaseDir, m_ForceTerminate)) + { + throw std::runtime_error(fmt::format("Failed to shut down zen process with pid {}, use --force to hard terminate", m_Pid)); + } + ZEN_CONSOLE("Zen server with pid {} is down", m_Pid); + return; + } + + if (HasExecutable) + { + int ShutdownCount = 0; + while (true) + { + ProcessHandle Proc; + std::error_code Ec = FindProcess(m_ExecutablePath, Proc, /*IncludeSelf*/ false); + if (Ec) + { + throw std::system_error(Ec, fmt::format("FindProcess failed for '{}'", m_ExecutablePath)); + } + if (!Proc.IsValid()) + { + break; + } + const uint32_t Pid = uint32_t(Proc.Pid()); + if (!TryShutdownByPid(Instance, Pid, m_ProgramBaseDir, m_ForceTerminate)) + { + throw std::runtime_error(fmt::format("Failed to shut down zen process with pid {}, use --force to hard terminate", Pid)); + } + ++ShutdownCount; + } + if (ShutdownCount == 0) + { + ZEN_CONSOLE("No zen server processes matching executable '{}'", m_ExecutablePath); + } + else + { + ZEN_CONSOLE("Shut down {} zen server instance(s) matching executable '{}'", ShutdownCount, m_ExecutablePath); + } + return; + } + if (m_All) { struct EntryInfo @@ -184,15 +316,10 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) int FailCount = 0; for (const EntryInfo& Info : Entries) { - Instance.Sweep(); - ZenServerState::ZenServerEntry* Entry = Instance.Lookup(Info.Port); - if (Entry && Entry->Pid.load() == Info.Pid) + if (!TryShutdownByPid(Instance, Info.Pid, m_ProgramBaseDir, m_ForceTerminate)) { - if (!ShutdownZenServer(ConsoleLog(), Instance, Entry, m_ProgramBaseDir)) - { - ZEN_CONSOLE_WARN("Failed to shutdown server on port {} (pid {})", Info.Port, Info.Pid); - ++FailCount; - } + ZEN_CONSOLE_WARN("Failed to shutdown server on port {} (pid {})", Info.Port, Info.Pid); + ++FailCount; } } @@ -207,9 +334,10 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_DataDir.empty()) { - if (!IsFile(m_DataDir / ".lock")) + if (!LockFile::IsHeldLive(m_DataDir / ".lock", /*AttemptCleanup*/ true)) { - throw std::runtime_error(fmt::format("Lock file does not exist in directory '{}'", m_DataDir)); + ZEN_CONSOLE("No live zen server holding lock file in '{}', nothing to do", m_DataDir); + return; } CbValidateError ValidateResult = CbValidateError::None; if (CbObject LockFileObject = diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h index f904fe0d9..bcc7d7da4 100644 --- a/src/zen/cmds/up_cmd.h +++ b/src/zen/cmds/up_cmd.h @@ -64,10 +64,12 @@ public: private: cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; + uint32_t m_Pid = 0; bool m_All = false; bool m_ForceTerminate = false; std::filesystem::path m_ProgramBaseDir; std::filesystem::path m_DataDir; + std::filesystem::path m_ExecutablePath; }; } // namespace zen diff --git a/src/zen/cmds/version_cmd.cpp b/src/zen/cmds/version_cmd.cpp index 0948de1bb..19951f862 100644 --- a/src/zen/cmds/version_cmd.cpp +++ b/src/zen/cmds/version_cmd.cpp @@ -2,6 +2,8 @@ #include "version_cmd.h" +#include "zenserviceclient.h" + #include <zencore/basicfile.h> #include <zencore/config.h> #include <zencore/filesystem.h> @@ -57,7 +59,9 @@ VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Querying host {}", m_HostName); } - HttpClient Client = CreateHttpClient(m_HostName, {.Timeout = std::chrono::milliseconds(5000)}); + ZenServiceClient Service( + {.HostSpec = m_HostName, .CommandName = Name, .HttpSettings = {.Timeout = std::chrono::milliseconds(5000)}}); + HttpClient& Client = Service.Http(); HttpClient::KeyValueMap Parameters; if (m_DetailedVersion) diff --git a/src/zen/cmds/vfs_cmd.cpp b/src/zen/cmds/vfs_cmd.cpp index 29ad8dc7c..c07526789 100644 --- a/src/zen/cmds/vfs_cmd.cpp +++ b/src/zen/cmds/vfs_cmd.cpp @@ -2,6 +2,8 @@ #include "vfs_cmd.h" +#include "zenserviceclient.h" + #include <zencore/compactbinarybuilder.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> @@ -40,12 +42,8 @@ VfsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // Validate arguments - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - throw OptionParseException("Unable to resolve server specification", m_Options.help()); - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (m_Verb == "mount"sv) { diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index 10f5ad8e1..d5344fb01 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -4,6 +4,7 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/iohash.h> #include <zencore/logging.h> #include <zencore/parallelwork.h> #include <zencore/string.h> @@ -11,7 +12,7 @@ #include <zencore/trace.h> #include <zenutil/workerpools.h> -#include "../progressbar.h" +#include "consoleprogress.h" #include <signal.h> @@ -34,13 +35,13 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { namespace wipe_impl { - static std::atomic<bool> AbortFlag = false; - static std::atomic<bool> PauseFlag = false; - static bool IsVerbose = false; - static bool Quiet = false; - static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; - const bool SingleThreaded = false; - bool BoostWorkerThreads = true; + static std::atomic<bool> AbortFlag = false; + static std::atomic<bool> PauseFlag = false; + static bool IsVerbose = false; + static bool Quiet = false; + static ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; + const bool SingleThreaded = false; + bool BoostWorkerThreads = true; WorkerThreadPool& GetIOWorkerPool() { @@ -167,7 +168,8 @@ namespace wipe_impl { ZEN_TRACE_CPU("CleanDirectory"); Stopwatch Timer; - ProgressBar Progress(ProgressMode, "Clean Folder"); + std::unique_ptr<ProgressBase> ProgressOwner(CreateConsoleProgress(ProgressMode)); + std::unique_ptr<ProgressBase::ProgressBar> Progress = ProgressOwner->CreateProgressBar("Clean Folder"); std::atomic<bool> CleanWipe = true; std::atomic<uint64_t> DiscoveredItemCount = 0; @@ -413,7 +415,7 @@ namespace wipe_impl { GetIOWorkerPool(), Work.PendingWork()); - Work.Wait(ProgressMode == ProgressBar::Mode::Pretty ? 200 : 5000, [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { + Work.Wait(ProgressOwner->GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { ZEN_UNUSED(PendingWork); if (Quiet) { @@ -424,12 +426,12 @@ namespace wipe_impl { uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); - Progress.UpdateState({.Task = "Removing files ", - .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), - .TotalCount = Discovered, - .RemainingCount = Discovered - Deleted, - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); + Progress->UpdateState({.Task = "Removing files ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted, + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); }); std::vector<std::filesystem::path> DirectoriesToDelete; @@ -473,22 +475,22 @@ namespace wipe_impl { } uint64_t NowMs = Timer.GetElapsedTimeMs(); - if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode)) + if ((NowMs - LastUpdateTimeMs) >= ProgressOwner->GetProgressUpdateDelayMS()) { LastUpdateTimeMs = NowMs; uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); - Progress.UpdateState({.Task = "Removing folders", - .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), - .TotalCount = DirectoriesToDelete.size(), - .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex}, - false); + Progress->UpdateState({.Task = "Removing folders", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = DirectoriesToDelete.size(), + .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex}, + false); } } - Progress.Finish(); + Progress->Finish(); uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); if (!Quiet) @@ -536,10 +538,10 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) using namespace wipe_impl; ZEN_UNUSED(GlobalOptions); - signal(SIGINT, SignalCallbackHandler); + ScopedSignalHandler SigIntGuard(SIGINT, SignalCallbackHandler); #if ZEN_PLATFORM_WINDOWS - signal(SIGBREAK, SignalCallbackHandler); -#endif // ZEN_PLATFORM_WINDOWS + ScopedSignalHandler SigBreakGuard(SIGBREAK, SignalCallbackHandler); +#endif if (!ParseOptions(argc, argv)) { @@ -548,7 +550,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Quiet = m_Quiet; IsVerbose = m_Verbose; - ProgressMode = (IsVerbose || m_PlainProgress) ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty; + ProgressMode = m_PlainProgress ? ConsoleProgressMode::Plain : ConsoleProgressMode::Pretty; BoostWorkerThreads = m_BoostWorkerThreads; MakeSafeAbsolutePathInPlace(m_Directory); diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 9e49b464e..3ab3b9e04 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -2,6 +2,8 @@ #include "workspaces_cmd.h" +#include "zenserviceclient.h" + #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> @@ -137,7 +139,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("'verb' option is required", m_Options.help()); } - // Parse subcommand permissively — forward unrecognised options to the parent parser. + // Parse subcommand permissively - forward unrecognised options to the parent parser. std::vector<std::string> SubUnmatched; if (!ParseOptionsPermissive(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data(), SubUnmatched)) { @@ -159,8 +161,6 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - if (m_SystemRootDir.empty()) { m_SystemRootDir = PickDefaultSystemRootDirectory(); @@ -199,7 +199,8 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!m_HostName.empty()) { - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); @@ -271,7 +272,8 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!m_HostName.empty()) { - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); @@ -403,7 +405,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw OptionParseException("'verb' option is required", m_Options.help()); } - // Parse subcommand permissively — forward unrecognised options to the parent parser. + // Parse subcommand permissively - forward unrecognised options to the parent parser. std::vector<std::string> SubUnmatched; if (!ParseOptionsPermissive(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data(), SubUnmatched)) { @@ -425,8 +427,6 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** return; } - m_HostName = ResolveTargetHostSpec(m_HostName); - if (m_SystemRootDir.empty()) { m_SystemRootDir = PickDefaultSystemRootDirectory(); @@ -509,7 +509,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** { if (!m_HostName.empty()) { - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); @@ -626,7 +627,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** { if (!m_HostName.empty()) { - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); @@ -674,12 +676,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** Params.Entries.insert_or_assign("refresh", ToString(m_Refresh)); } - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", SubOption->help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/files", GetShareIdentityUrl(m_FilesOptions)), {}, Params)) { ZEN_CONSOLE("{}: {}", Result, Result.ToText()); @@ -707,12 +705,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** Params.Entries.insert_or_assign("refresh", ToString(m_Refresh)); } - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", SubOption->help()); - } - - HttpClient Http = CreateHttpClient(m_HostName); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/entries", GetShareIdentityUrl(m_EntriesOptions)), {}, Params)) { ZEN_CONSOLE("{}: {}", Result, Result.ToText()); @@ -777,18 +771,14 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (SubOption == &m_GetChunkOptions) { - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", SubOption->help()); - } - if (m_ChunkId.empty()) { throw OptionParseException("'--chunk' is required", SubOption->help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ChunkId = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, std::vector<std::string>{m_ChunkId})[0]; + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ChunkId = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, std::vector<std::string>{m_ChunkId})[0]; HttpClient::KeyValueMap Params; if (m_Offset != 0) @@ -813,11 +803,6 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (SubOption == &m_GetChunkBatchOptions) { - if (m_HostName.empty()) - { - throw OptionParseException("Unable to resolve server specification", SubOption->help()); - } - if (m_ShareId.empty()) { throw OptionParseException("'--share' is required", SubOption->help()); @@ -828,8 +813,9 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw OptionParseException("'--chunks' is required", SubOption->help()); } - HttpClient Http = CreateHttpClient(m_HostName); - m_ChunkIds = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, m_ChunkIds); + ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name}); + HttpClient& Http = Service.Http(); + m_ChunkIds = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, m_ChunkIds); std::vector<RequestChunkEntry> ChunkRequests; ChunkRequests.resize(m_ChunkIds.size()); |