aboutsummaryrefslogtreecommitdiff
path: root/src/zen/cmds
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2026-04-23 18:16:57 +0200
committerStefan Boberg <[email protected]>2026-04-23 18:16:57 +0200
commit0232b991cd7d8e3a2114ea30e4591dd3e7b65c36 (patch)
tree94730e7594fd09ae1fa820391ce311f6daf13905 /src/zen/cmds
parentFix forward declaration order for s_GotSigWinch and SigWinchHandler (diff)
parenttrace: declare Region event name fields as AnsiString (#1012) (diff)
downloadarchived-zen-sb/zen-help.tar.xz
archived-zen-sb/zen-help.zip
Merge branch 'main' into sb/zen-helpsb/zen-help
- Combine HelpCommand (this branch) with HistoryCommand (main) in zen CLI dispatcher - Keep filter-aware TuiPickOne rewrite; adopt main's ASCII arrow glyphs in doc comment
Diffstat (limited to 'src/zen/cmds')
-rw-r--r--src/zen/cmds/admin_cmd.cpp86
-rw-r--r--src/zen/cmds/bench_cmd.cpp2
-rw-r--r--src/zen/cmds/builds_cmd.cpp3871
-rw-r--r--src/zen/cmds/builds_cmd.h405
-rw-r--r--src/zen/cmds/cache_cmd.cpp884
-rw-r--r--src/zen/cmds/cache_cmd.h330
-rw-r--r--src/zen/cmds/compute_cmd.cpp88
-rw-r--r--src/zen/cmds/compute_cmd.h53
-rw-r--r--src/zen/cmds/copy_cmd.cpp207
-rw-r--r--src/zen/cmds/copy_cmd.h32
-rw-r--r--src/zen/cmds/dedup_cmd.cpp13
-rw-r--r--src/zen/cmds/exec_cmd.cpp89
-rw-r--r--src/zen/cmds/history_cmd.cpp228
-rw-r--r--src/zen/cmds/history_cmd.h (renamed from src/zen/cmds/trace_cmd.h)17
-rw-r--r--src/zen/cmds/hub_cmd.cpp43
-rw-r--r--src/zen/cmds/info_cmd.cpp14
-rw-r--r--src/zen/cmds/projectstore_cmd.cpp301
-rw-r--r--src/zen/cmds/projectstore_cmd.h2
-rw-r--r--src/zen/cmds/rpcreplay_cmd.cpp486
-rw-r--r--src/zen/cmds/rpcreplay_cmd.h75
-rw-r--r--src/zen/cmds/run_cmd.cpp194
-rw-r--r--src/zen/cmds/run_cmd.h30
-rw-r--r--src/zen/cmds/service_cmd.cpp6
-rw-r--r--src/zen/cmds/trace_cmd.cpp89
-rw-r--r--src/zen/cmds/ui_cmd.cpp51
-rw-r--r--src/zen/cmds/up_cmd.cpp162
-rw-r--r--src/zen/cmds/up_cmd.h2
-rw-r--r--src/zen/cmds/version_cmd.cpp6
-rw-r--r--src/zen/cmds/vfs_cmd.cpp10
-rw-r--r--src/zen/cmds/wipe_cmd.cpp56
-rw-r--r--src/zen/cmds/workspaces_cmd.cpp58
31 files changed, 3093 insertions, 4797 deletions
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index 034d430fd..2580517fa 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -2,6 +2,8 @@
#include "admin_cmd.h"
+#include "zenserviceclient.h"
+
#include <zencore/basicfile.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
@@ -41,14 +43,8 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
HttpClient::KeyValueMap Params{{"skipdelete", ToString(m_DryRun)},
{"skipgc", ToString(m_NoGc)},
@@ -168,12 +164,7 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
HttpClient::KeyValueMap Params;
Params.Entries.insert({"smallobjects", m_SmallObjects ? "true" : "false"});
@@ -258,7 +249,7 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
Params.Entries.insert({"enablevalidation", m_EnableValidation ? "true" : "false"});
- HttpClient Http = CreateHttpClient(m_HostName);
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Response = Http.Post("/admin/gc"sv, HttpClient::Accept(HttpContentType::kJSON), Params))
{
ZEN_CONSOLE("OK: {}", Response.ToText());
@@ -290,14 +281,8 @@ GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Response = Http.Get("/admin/gc"sv, HttpClient::Accept(HttpContentType::kJSON)))
{
ZEN_CONSOLE("OK: {}", Response.ToText());
@@ -328,14 +313,8 @@ GcStopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Response = Http.Post("/admin/gc-stop"sv, HttpClient::Accept(HttpContentType::kJSON)))
{
if (Response.StatusCode == HttpResponseCode::Accepted)
@@ -377,14 +356,8 @@ JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (m_Cancel)
{
@@ -460,14 +433,8 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
HttpClient::KeyValueMap Parameters;
@@ -522,9 +489,10 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
throw std::runtime_error(fmt::format("Failed to retrieve {} log path", SourceName));
}
- if (!CopyFile(SourcePath, TargetPath, {}))
+ if (std::error_code Ec = CopyFile(SourcePath, TargetPath, {}); Ec)
{
- throw std::runtime_error(
+ throw std::system_error(
+ Ec,
fmt::format("Failed to copy {} log file {} to output file '{}'", SourceName, SourcePath, TargetPath));
}
};
@@ -579,16 +547,10 @@ FlushCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- zen::HttpClient Http = CreateHttpClient(m_HostName);
-
- if (zen::HttpClient::Response Response = Http.Post("/admin/flush"sv))
+ if (HttpClient::Response Response = Http.Post("/admin/flush"sv))
{
ZEN_CONSOLE("OK: {}", Response.ToText());
@@ -624,7 +586,10 @@ Copy(const std::filesystem::path& Source, const std::filesystem::path& Target)
CreateDirectories(Target.parent_path());
CopyFileOptions Options;
- CopyFile(Source, Target, Options);
+ if (std::error_code Ec = CopyFile(Source, Target, Options); Ec)
+ {
+ throw std::system_error(Ec, fmt::format("Failed to copy '{}' to '{}'", Source, Target));
+ }
}
static bool
@@ -638,7 +603,8 @@ TryCopy(const std::filesystem::path& Source, const std::filesystem::path& Target
CreateDirectories(Target.parent_path());
CopyFileOptions Options;
- return CopyFile(Source, Target, Options);
+ std::error_code Ec = CopyFile(Source, Target, Options);
+ return !Ec;
}
void
diff --git a/src/zen/cmds/bench_cmd.cpp b/src/zen/cmds/bench_cmd.cpp
index b1639105a..c935179e2 100644
--- a/src/zen/cmds/bench_cmd.cpp
+++ b/src/zen/cmds/bench_cmd.cpp
@@ -1661,7 +1661,7 @@ BenchDiskSubCmd::RunClone(const std::filesystem::path& Dir)
try
{
std::filesystem::path DstPath = Dir / fmt::format("bench_clone_{}.tmp", FileIndex);
- if (TryCloneFile(SrcPath, DstPath))
+ if (std::error_code CloneEc = TryCloneFile(SrcPath, DstPath); !CloneEc)
{
CloneCount.fetch_add(1, std::memory_order_relaxed);
}
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index cc8315e0b..84d8424aa 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -4,68 +4,34 @@
#include <zencore/basicfile.h>
#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinaryfile.h>
-#include <zencore/compactbinaryfmt.h>
-#include <zencore/compactbinaryvalue.h>
-#include <zencore/compress.h>
-#include <zencore/except.h>
+#include <zencore/except_fmt.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/parallelwork.h>
+#include <zencore/process.h>
#include <zencore/scopeguard.h>
#include <zencore/session.h>
-#include <zencore/stream.h>
#include <zencore/string.h>
-#include <zencore/trace.h>
-#include <zencore/uid.h>
+#include <zenhttp/auth/authmgr.h>
#include <zenhttp/formatters.h>
-#include <zenhttp/httpclient.h>
-#include <zenhttp/httpclientauth.h>
#include <zenhttp/httpcommon.h>
-#include <zenremotestore/builds/buildcontent.h>
-#include <zenremotestore/builds/buildmanifest.h>
-#include <zenremotestore/builds/buildsavedstate.h>
-#include <zenremotestore/builds/buildstoragecache.h>
-#include <zenremotestore/builds/buildstorageoperations.h>
-#include <zenremotestore/builds/buildstorageutil.h>
+#include <zenremotestore/builds/buildinspect.h>
+#include <zenremotestore/builds/buildprimecache.h>
+#include <zenremotestore/builds/buildupdatefolder.h>
+#include <zenremotestore/builds/builduploadfolder.h>
+#include <zenremotestore/builds/buildvalidatebuildpart.h>
#include <zenremotestore/builds/filebuildstorage.h>
#include <zenremotestore/builds/jupiterbuildstorage.h>
-#include <zenremotestore/chunking/chunkblock.h>
-#include <zenremotestore/chunking/chunkedcontent.h>
-#include <zenremotestore/chunking/chunkedfile.h>
#include <zenremotestore/chunking/chunkingcache.h>
#include <zenremotestore/chunking/chunkingcontroller.h>
-#include <zenremotestore/filesystemutils.h>
-#include <zenremotestore/jupiter/jupiterhost.h>
-#include <zenremotestore/operationlogoutput.h>
-#include <zenremotestore/transferthreadworkers.h>
+#include <zenutil/filesystemutils.h>
+#include <zenutil/progress.h>
#include <zenutil/wildcard.h>
#include <zenutil/workerpools.h>
-#include <zenutil/zenserverprocess.h>
-
-#include "../progressbar.h"
#include <signal.h>
#include <memory>
-#include <numeric>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <tsl/robin_map.h>
-#include <tsl/robin_set.h>
-#include <json11.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
-
-#if ZEN_PLATFORM_WINDOWS
-# include <zencore/windows.h>
-#else
-# include <fcntl.h>
-# include <sys/file.h>
-# include <sys/stat.h>
-# include <unistd.h>
-#endif
-
-static const bool DoExtraContentVerify = false;
namespace zen {
@@ -115,14 +81,6 @@ namespace builds_impl {
}
};
- struct MemMap
- {
- void* Handle = nullptr;
- void* Data = nullptr;
- size_t Size = 0;
- std::string Name;
- };
-
class ZenState
{
public:
@@ -236,1869 +194,103 @@ namespace builds_impl {
}
}
- const std::string ZenFolderName = ".zen";
- std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; }
- // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json";
- // }
-
- std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path)
- {
- const std::u8string LocalPathString = Path.generic_u8string();
- IoHash PathHash = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length());
- return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash);
- }
-
- const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt";
-
- const std::string UnsyncFolderName = ".unsync";
-
- const std::string UGSFolderName = ".ugs";
- const std::string LegacyZenTempFolderName = ".zen-tmp";
-
- const std::vector<std::string> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName});
- const std::vector<std::string> DefaultExcludeExtensions({});
-
+ // Debugging knobs for file build storage - always 0 in shipped builds
const double DefaultLatency = 0; // .0010;
const double DefaultDelayPerKBSec = 0; // 0.00005;
- const bool SingleThreaded = false;
- bool UseSparseFiles = false;
-
- static bool IsVerbose = false;
- static bool IsQuiet = false;
- static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
-
-#undef ZEN_CONSOLE_VERBOSE
-#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \
- if (IsVerbose) \
- { \
- ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \
- }
-
- const std::string DefaultAccessTokenEnvVariableName(
-#if ZEN_PLATFORM_WINDOWS
- "UE-CloudDataCacheAccessToken"sv
-#endif
-#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
- "UE_CloudDataCacheAccessToken"sv
-#endif
-
- );
-
- static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory)
- {
- return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u;
- }
-
- class FilteredRate
- {
- public:
- FilteredRate() {}
-
- void Start()
- {
- if (StartTimeUS == (uint64_t)-1)
- {
- uint64_t Expected = (uint64_t)-1;
- if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()))
- {
- LastTimeUS = StartTimeUS.load();
- }
- }
- }
- void Stop()
- {
- if (EndTimeUS == (uint64_t)-1)
- {
- uint64_t Expected = (uint64_t)-1;
- EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs());
- }
- }
-
- void Update(uint64_t Count)
- {
- if (LastTimeUS == (uint64_t)-1)
- {
- return;
- }
- uint64_t TimeUS = Timer.GetElapsedTimeUs();
- uint64_t TimeDeltaUS = TimeUS - LastTimeUS;
- if (TimeDeltaUS >= 2000000)
- {
- uint64_t Delta = Count - LastCount;
- uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS;
-
- LastPerSecond = PerSecond;
-
- LastCount = Count;
-
- FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8;
-
- LastTimeUS = TimeUS;
- }
- }
-
- uint64_t GetCurrent() const // If Stopped - return total count / total time
- {
- if (LastTimeUS == (uint64_t)-1)
- {
- return 0;
- }
- return FilteredPerSecond;
- }
-
- uint64_t GetElapsedTimeUS() const
- {
- if (StartTimeUS == (uint64_t)-1)
- {
- return 0;
- }
- if (EndTimeUS == (uint64_t)-1)
- {
- return 0;
- }
- uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS;
- return TimeDeltaUS;
- }
-
- bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); }
-
- private:
- Stopwatch Timer;
- std::atomic<uint64_t> StartTimeUS = (uint64_t)-1;
- std::atomic<uint64_t> EndTimeUS = (uint64_t)-1;
- std::atomic<uint64_t> LastTimeUS = (uint64_t)-1;
- uint64_t LastCount = 0;
- uint64_t LastPerSecond = 0;
- uint64_t FilteredPerSecond = 0;
- };
-
- uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count)
+ void WriteResultObject(const std::filesystem::path& Path, const CbObject& Response)
{
- if (ElapsedWallTimeUS == 0)
- {
- return 0;
- }
- return Count * 1000000 / ElapsedWallTimeUS;
- }
-
- bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, const std::filesystem::path& Directory)
- {
- return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory);
- }
-
- void ValidateBuildPart(OperationLogOutput& Output,
- TransferThreadWorkers& Workers,
- BuildStorageBase& Storage,
- const Oid& BuildId,
- Oid BuildPartId,
- const std::string_view BuildPartName)
- {
- ZEN_TRACE_CPU("ValidateBuildPart");
-
- ProgressBar::SetLogOperationName(ProgressMode, "Validate Part");
-
- BuildsOperationValidateBuildPart ValidateOp(Output,
- Storage,
- AbortFlag,
- PauseFlag,
- Workers.GetIOWorkerPool(),
- Workers.GetNetworkPool(),
- BuildId,
- BuildPartId,
- BuildPartName,
- BuildsOperationValidateBuildPart::Options{.IsQuiet = IsQuiet, .IsVerbose = IsVerbose});
-
- ValidateOp.Execute();
-
- const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount;
- const uint64_t DownloadedByteCount =
- ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount;
- ZEN_CONSOLE("Verified: {:>8} ({}), {}B/sec, {}",
- DownloadedCount,
- NiceBytes(DownloadedByteCount),
- NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)),
- NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000));
- }
-
- struct UploadFolderOptions
- {
- std::filesystem::path TempDir;
- uint64_t FindBlockMaxCount;
- uint8_t BlockReuseMinPercentLimit;
- bool AllowMultiparts;
- bool CreateBuild;
- bool IgnoreExistingBlocks;
- bool UploadToZenCache;
- const std::vector<std::string>& ExcludeFolders = DefaultExcludeFolders;
- const std::vector<std::string>& ExcludeExtensions = DefaultExcludeExtensions;
- };
-
- std::vector<std::pair<Oid, std::string>> UploadFolder(OperationLogOutput& Output,
- TransferThreadWorkers& Workers,
- StorageInstance& Storage,
- const Oid& BuildId,
- const Oid& BuildPartId,
- const std::string_view BuildPartName,
- const std::filesystem::path& Path,
- const std::filesystem::path& ManifestPath,
- const CbObject& MetaData,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache,
- const UploadFolderOptions& Options)
- {
- ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder");
-
- Stopwatch UploadTimer;
-
- BuildsOperationUploadFolder UploadOp(
- Output,
- Storage,
- AbortFlag,
- PauseFlag,
- Workers.GetIOWorkerPool(),
- Workers.GetNetworkPool(),
- BuildId,
- Path,
- Options.CreateBuild,
- std::move(MetaData),
- BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet,
- .IsVerbose = IsVerbose,
- .DoExtraContentValidation = DoExtraContentVerify,
- .FindBlockMaxCount = Options.FindBlockMaxCount,
- .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit,
- .AllowMultiparts = Options.AllowMultiparts,
- .IgnoreExistingBlocks = Options.IgnoreExistingBlocks,
- .TempDir = Options.TempDir,
- .ExcludeFolders = Options.ExcludeFolders,
- .ExcludeExtensions = Options.ExcludeExtensions,
- .ZenExcludeManifestName = ZenExcludeManifestName,
- .NonCompressableExtensions = DefaultSplitOnlyExtensions,
- .PopulateCache = Options.UploadToZenCache});
-
- std::vector<std::pair<Oid, std::string>> UploadedParts =
- UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache);
- if (AbortFlag)
- {
- return {};
- }
-
- ZEN_CONSOLE_VERBOSE(
- "Folder scanning stats:"
- "\n FoundFileCount: {}"
- "\n FoundFileByteCount: {}"
- "\n AcceptedFileCount: {}"
- "\n AcceptedFileByteCount: {}"
- "\n ElapsedWallTimeUS: {}",
- UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
- NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
- UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(),
- NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()),
- NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Chunking stats:"
- "\n FilesProcessed: {}"
- "\n FilesChunked: {}"
- "\n BytesHashed: {}"
- "\n UniqueChunksFound: {}"
- "\n UniqueSequencesFound: {}"
- "\n UniqueBytesFound: {}"
- "\n FilesFoundInCache: {}"
- "\n ChunksFoundInCache: {}"
- "\n FilesStoredInCache: {}"
- "\n ChunksStoredInCache: {}"
- "\n ElapsedWallTimeUS: {}",
- UploadOp.m_ChunkingStats.FilesProcessed.load(),
- UploadOp.m_ChunkingStats.FilesChunked.load(),
- NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()),
- UploadOp.m_ChunkingStats.UniqueChunksFound.load(),
- UploadOp.m_ChunkingStats.UniqueSequencesFound.load(),
- NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()),
- UploadOp.m_ChunkingStats.FilesFoundInCache.load(),
- UploadOp.m_ChunkingStats.ChunksFoundInCache.load(),
- NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()),
- UploadOp.m_ChunkingStats.FilesStoredInCache.load(),
- UploadOp.m_ChunkingStats.ChunksStoredInCache.load(),
- NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()),
- NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Find block stats:"
- "\n FindBlockTimeMS: {}"
- "\n PotentialChunkCount: {}"
- "\n PotentialChunkByteCount: {}"
- "\n FoundBlockCount: {}"
- "\n FoundBlockChunkCount: {}"
- "\n FoundBlockByteCount: {}"
- "\n AcceptedBlockCount: {}"
- "\n NewBlocksCount: {}"
- "\n NewBlocksChunkCount: {}"
- "\n NewBlocksChunkByteCount: {}",
- NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS),
- UploadOp.m_FindBlocksStats.PotentialChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount),
- UploadOp.m_FindBlocksStats.FoundBlockCount,
- UploadOp.m_FindBlocksStats.FoundBlockChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount),
- UploadOp.m_FindBlocksStats.AcceptedBlockCount,
- UploadOp.m_FindBlocksStats.NewBlocksCount,
- UploadOp.m_FindBlocksStats.NewBlocksChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount));
-
- ZEN_CONSOLE_VERBOSE(
- "Reuse block stats:"
- "\n AcceptedChunkCount: {}"
- "\n AcceptedByteCount: {}"
- "\n AcceptedRawByteCount: {}"
- "\n RejectedBlockCount: {}"
- "\n RejectedChunkCount: {}"
- "\n RejectedByteCount: {}"
- "\n AcceptedReduntantChunkCount: {}"
- "\n AcceptedReduntantByteCount: {}",
- UploadOp.m_ReuseBlocksStats.AcceptedChunkCount,
- NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount),
- NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount),
- UploadOp.m_ReuseBlocksStats.RejectedBlockCount,
- UploadOp.m_ReuseBlocksStats.RejectedChunkCount,
- NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount),
- UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount,
- NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount));
-
- ZEN_CONSOLE_VERBOSE(
- "Generate blocks stats:"
- "\n GeneratedBlockByteCount: {}"
- "\n GeneratedBlockCount: {}"
- "\n GenerateBlocksElapsedWallTimeUS: {}",
- NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
- UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Generate blocks stats:"
- "\n ChunkCount: {}"
- "\n ChunkByteCount: {}"
- "\n CompressedChunkCount: {}"
- "\n CompressChunksElapsedWallTimeUS: {}",
- UploadOp.m_LooseChunksStats.ChunkCount,
- NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount),
- UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
- NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Disk stats:"
- "\n OpenReadCount: {}"
- "\n OpenWriteCount: {}"
- "\n ReadCount: {}"
- "\n ReadByteCount: {}"
- "\n WriteCount: {} ({} cloned)"
- "\n WriteByteCount: {} ({} cloned)"
- "\n CurrentOpenFileCount: {}",
- UploadOp.m_DiskStats.OpenReadCount.load(),
- UploadOp.m_DiskStats.OpenWriteCount.load(),
- UploadOp.m_DiskStats.ReadCount.load(),
- NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()),
- UploadOp.m_DiskStats.WriteCount.load(),
- UploadOp.m_DiskStats.CloneCount.load(),
- NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()),
- NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()),
- UploadOp.m_DiskStats.CurrentOpenFileCount.load());
-
- ZEN_CONSOLE_VERBOSE(
- "Upload stats:"
- "\n BlockCount: {}"
- "\n BlocksBytes: {}"
- "\n ChunkCount: {}"
- "\n ChunksBytes: {}"
- "\n ReadFromDiskBytes: {}"
- "\n MultipartAttachmentCount: {}"
- "\n ElapsedWallTimeUS: {}",
- UploadOp.m_UploadStats.BlockCount.load(),
- NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
- UploadOp.m_UploadStats.ChunkCount.load(),
- NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
- NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()),
- UploadOp.m_UploadStats.MultipartAttachmentCount.load(),
- NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000));
-
- const double DeltaByteCountPercent =
- UploadOp.m_ChunkingStats.BytesHashed > 0
- ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) /
- (UploadOp.m_ChunkingStats.BytesHashed)
- : 0.0;
-
- const std::string MultipartAttachmentStats =
- Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : "";
-
- if (!IsQuiet)
- {
- ZEN_CONSOLE(
- "Uploaded part {} ('{}') to build {}, {}\n"
- " Scanned files: {:>8} ({}), {}B/sec, {}\n"
- " New data: {:>8} ({}) {:.1f}%\n"
- " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
- " Blocks: {:>8} ({})\n"
- " Chunks: {:>8} ({}){}",
- BuildPartId,
- BuildPartName,
- BuildId,
- NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()),
-
- UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
- NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
- NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)),
- NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000),
-
- UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes),
- DeltaByteCountPercent,
-
- UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount),
- NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
- NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
- UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
-
- UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes),
- NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS,
- UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)),
- NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
-
- UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(),
- NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS,
- (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)),
- NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000),
-
- UploadOp.m_UploadStats.BlockCount.load(),
- NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
-
- UploadOp.m_UploadStats.ChunkCount.load(),
- NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
- MultipartAttachmentStats);
- }
- return UploadedParts;
- }
-
- struct VerifyFolderStatistics
- {
- std::atomic<uint64_t> FilesVerified = 0;
- std::atomic<uint64_t> FilesFailed = 0;
- std::atomic<uint64_t> ReadBytes = 0;
- uint64_t VerifyElapsedWallTimeUs = 0;
- };
-
- void VerifyFolder(TransferThreadWorkers& Workers,
- const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- const std::filesystem::path& Path,
- const std::vector<std::string>& ExcludeFolders,
- bool VerifyFileHash,
- VerifyFolderStatistics& VerifyFolderStats)
- {
- ZEN_TRACE_CPU("VerifyFolder");
-
- Stopwatch Timer;
-
- ProgressBar ProgressBar(ProgressMode, "Verify Files");
-
- WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool();
-
- ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size());
-
- RwLock ErrorLock;
- std::vector<std::string> Errors;
-
- auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool {
- for (const std::string& ExcludeFolder : ExcludeFolders)
- {
- if (RelativePath.starts_with(ExcludeFolder))
- {
- if (RelativePath.length() == ExcludeFolder.length())
- {
- return false;
- }
- else if (RelativePath[ExcludeFolder.length()] == '/')
- {
- return false;
- }
- }
- }
- return true;
- };
-
- for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++)
- {
- if (Work.IsAborted())
- {
- break;
- }
-
- Work.ScheduleWork(
- VerifyPool,
- [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex](
- std::atomic<bool>&) {
- if (!AbortFlag)
- {
- ZEN_TRACE_CPU("VerifyFile_work");
-
- // TODO: Convert ScheduleWork body to function
-
- const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred();
- if (IsAcceptedFolder(TargetPath.parent_path().generic_string()))
- {
- const uint64_t ExpectedSize = Content.RawSizes[PathIndex];
- if (!IsFile(TargetPath))
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize));
- });
- VerifyFolderStats.FilesFailed++;
- }
- else
- {
- std::error_code Ec;
- uint64_t SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec));
- if (Ec)
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(
- fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value()));
- });
- VerifyFolderStats.FilesFailed++;
- }
- else if (SizeOnDisk < ExpectedSize)
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}",
- TargetPath,
- ExpectedSize,
- SizeOnDisk));
- });
- VerifyFolderStats.FilesFailed++;
- }
- else if (SizeOnDisk > ExpectedSize)
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}",
- TargetPath,
- ExpectedSize,
- SizeOnDisk));
- });
- VerifyFolderStats.FilesFailed++;
- }
- else if (SizeOnDisk > 0 && VerifyFileHash)
- {
- const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex];
- IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath);
- IoHash RawHash = IoHash::HashBuffer(Buffer);
- if (RawHash != ExpectedRawHash)
- {
- uint64_t FileOffset = 0;
- const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash);
- const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex];
- for (uint32_t OrderIndex = OrderOffset;
- OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex];
- OrderIndex++)
- {
- uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex];
- uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
- IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex];
- IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize);
- if (IoHash::HashBuffer(FileChunk) != ChunkHash)
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format(
- "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. "
- "Mismatch at chunk {}",
- TargetPath,
- ExpectedRawHash,
- RawHash,
- OrderIndex - OrderOffset));
- });
- break;
- }
- FileOffset += ChunkSize;
- }
- VerifyFolderStats.FilesFailed++;
- }
- VerifyFolderStats.ReadBytes += SizeOnDisk;
- }
- }
- }
- VerifyFolderStats.FilesVerified++;
- }
- },
- [&, PathIndex](std::exception_ptr Ex, std::atomic<bool>&) {
- std::string Description;
- try
- {
- std::rethrow_exception(Ex);
- }
- catch (const std::exception& Ex)
- {
- Description = Ex.what();
- }
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}",
- (Path / Content.Paths[PathIndex]).make_preferred(),
- Description));
- });
- VerifyFolderStats.FilesFailed++;
- });
- }
-
- Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
- std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}",
- VerifyFolderStats.FilesVerified.load(),
- PathCount,
- NiceBytes(VerifyFolderStats.ReadBytes.load()),
- VerifyFolderStats.FilesFailed.load());
- ProgressBar.UpdateState({.Task = "Verifying files ",
- .Details = Details,
- .TotalCount = gsl::narrow<uint64_t>(PathCount),
- .RemainingCount = gsl::narrow<uint64_t>(PathCount - VerifyFolderStats.FilesVerified.load()),
- .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
- VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs();
-
- ProgressBar.Finish();
- if (AbortFlag)
- {
- return;
- }
-
- for (const std::string& Error : Errors)
- {
- ZEN_CONSOLE_ERROR("{}", Error);
- }
- if (!Errors.empty())
- {
- throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size()));
- }
- }
-
- CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId)
- {
- Stopwatch GetBuildTimer;
- CbObject BuildObject = Storage.GetBuild(BuildId);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
- NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
- BuildObject["name"sv].AsString(),
- NiceBytes(BuildObject.GetSize()));
-
- ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv));
- }
- return BuildObject;
- }
-
- std::vector<std::filesystem::path> GetNewPaths(const std::span<const std::filesystem::path> KnownPaths,
- const std::span<const std::filesystem::path> Paths)
- {
- tsl::robin_set<std::string> KnownPathsSet;
- KnownPathsSet.reserve(KnownPaths.size());
- for (const std::filesystem::path& LocalPath : KnownPaths)
- {
- KnownPathsSet.insert(LocalPath.generic_string());
- }
-
- std::vector<std::filesystem::path> NewPaths;
- for (const std::filesystem::path& UntrackedPath : Paths)
- {
- if (!KnownPathsSet.contains(UntrackedPath.generic_string()))
- {
- NewPaths.push_back(UntrackedPath);
- }
- }
- return NewPaths;
- }
-
- BuildSaveState GetLocalStateFromPaths(TransferThreadWorkers& Workers,
- GetFolderContentStatistics& LocalFolderScanStats,
- ChunkingStatistics& ChunkingStats,
- const std::filesystem::path& Path,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache,
- std::span<const std::filesystem::path> PathsToCheck)
- {
- FolderContent FolderState;
- ChunkedFolderContent ChunkedContent;
- {
- ProgressBar ProgressBar(ProgressMode, "Check Files");
- FolderState = GetValidFolderContent(
- Workers.GetIOWorkerPool(),
- LocalFolderScanStats,
- Path,
- PathsToCheck,
- [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) {
- std::string Details =
- fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load());
- ProgressBar.UpdateState({.Task = "Checking files ",
- .Details = Details,
- .TotalCount = PathCount,
- .RemainingCount = PathCount - CompletedPathCount,
- .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
- false);
- },
- GetUpdateDelayMS(ProgressMode),
- AbortFlag,
- PauseFlag);
- ProgressBar.Finish();
- }
-
- if (FolderState.Paths.size() > 0)
- {
- uint64_t ByteCountToScan = 0;
- for (const uint64_t RawSize : FolderState.RawSizes)
- {
- ByteCountToScan += RawSize;
- }
- ProgressBar ProgressBar(ProgressMode, "Scan Files");
- FilteredRate FilteredBytesHashed;
- FilteredBytesHashed.Start();
- ChunkingStatistics LocalChunkingStats;
- ChunkedContent = ChunkFolderContent(
- LocalChunkingStats,
- Workers.GetIOWorkerPool(),
- Path,
- FolderState,
- ChunkController,
- ChunkCache,
- GetUpdateDelayMS(ProgressMode),
- [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
- FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load());
- std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
- LocalChunkingStats.FilesProcessed.load(),
- FolderState.Paths.size(),
- NiceBytes(LocalChunkingStats.BytesHashed.load()),
- NiceBytes(ByteCountToScan),
- NiceNum(FilteredBytesHashed.GetCurrent()),
- LocalChunkingStats.UniqueChunksFound.load(),
- NiceBytes(LocalChunkingStats.UniqueBytesFound.load()));
- ProgressBar.UpdateState({.Task = "Scanning files ",
- .Details = Details,
- .TotalCount = ByteCountToScan,
- .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(),
- .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- },
- AbortFlag,
- PauseFlag);
- ChunkingStats += LocalChunkingStats;
- FilteredBytesHashed.Stop();
- ProgressBar.Finish();
- }
-
- return BuildSaveState{.State = BuildState{.ChunkedContent = std::move(ChunkedContent)},
- .FolderState = FolderState,
- .LocalPath = Path};
- }
-
- BuildSaveState GetLocalContent(TransferThreadWorkers& Workers,
- GetFolderContentStatistics& LocalFolderScanStats,
- ChunkingStatistics& ChunkingStats,
- const std::filesystem::path& Path,
- const std::filesystem::path& StateFilePath,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache)
- {
- Stopwatch ReadStateTimer;
- bool FileExists = IsFile(StateFilePath);
- if (!FileExists)
- {
- ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path);
- return {};
- }
-
- BuildSaveState SavedLocalState;
- try
- {
- SavedLocalState = ReadBuildSaveStateFile(StateFilePath);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
- }
- }
- catch (const std::exception& Ex)
+ const MemoryView ResponseView = Response.GetView();
+ if (ToLower(Path.extension().string()) == ".cbo")
{
- ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what());
- return {};
- }
-
- FolderContent CurrentLocalFolderState;
- {
- ProgressBar ProgressBar(ProgressMode, "Check Known Files");
- CurrentLocalFolderState = GetValidFolderContent(
- Workers.GetIOWorkerPool(),
- LocalFolderScanStats,
- Path,
- SavedLocalState.FolderState.Paths,
- [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) {
- std::string Details =
- fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load());
- ProgressBar.UpdateState({.Task = "Checking files ",
- .Details = Details,
- .TotalCount = PathCount,
- .RemainingCount = PathCount - CompletedPathCount,
- .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
- false);
- },
- GetUpdateDelayMS(ProgressMode),
- AbortFlag,
- PauseFlag);
- ProgressBar.Finish();
- }
- if (AbortFlag)
- {
- return {};
- }
-
- if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState))
- {
- const size_t LocalStatePathCount = SavedLocalState.FolderState.Paths.size();
- std::vector<std::filesystem::path> DeletedPaths;
- FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths);
- if (!DeletedPaths.empty())
- {
- SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths);
- }
-
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
- DeletedPaths.size(),
- UpdatedContent.Paths.size(),
- LocalStatePathCount);
- }
- if (UpdatedContent.Paths.size() > 0)
- {
- uint64_t ByteCountToScan = 0;
- for (const uint64_t RawSize : UpdatedContent.RawSizes)
- {
- ByteCountToScan += RawSize;
- }
- ProgressBar ProgressBar(ProgressMode, "Scan Known Files");
- FilteredRate FilteredBytesHashed;
- FilteredBytesHashed.Start();
- ChunkingStatistics LocalChunkingStats;
- ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent(
- LocalChunkingStats,
- Workers.GetIOWorkerPool(),
- Path,
- UpdatedContent,
- ChunkController,
- ChunkCache,
- GetUpdateDelayMS(ProgressMode),
- [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
- FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load());
- std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
- LocalChunkingStats.FilesProcessed.load(),
- UpdatedContent.Paths.size(),
- NiceBytes(LocalChunkingStats.BytesHashed.load()),
- NiceBytes(ByteCountToScan),
- NiceNum(FilteredBytesHashed.GetCurrent()),
- LocalChunkingStats.UniqueChunksFound.load(),
- NiceBytes(LocalChunkingStats.UniqueBytesFound.load()));
- ProgressBar.UpdateState({.Task = "Scanning files ",
- .Details = Details,
- .TotalCount = ByteCountToScan,
- .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(),
- .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- },
- AbortFlag,
- PauseFlag);
-
- ChunkingStats += LocalChunkingStats;
-
- FilteredBytesHashed.Stop();
- ProgressBar.Finish();
- if (AbortFlag)
- {
- return {};
- }
- SavedLocalState.State.ChunkedContent =
- MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}});
- }
- }
- else
- {
- // Remove files from LocalContent no longer in LocalFolderState
- tsl::robin_set<std::string> LocalFolderPaths;
- LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size());
- for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths)
- {
- LocalFolderPaths.insert(LocalFolderPath.generic_string());
- }
- std::vector<std::filesystem::path> DeletedPaths;
- for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths)
- {
- if (!LocalFolderPaths.contains(LocalContentPath.generic_string()))
- {
- DeletedPaths.push_back(LocalContentPath);
- }
- }
- if (!DeletedPaths.empty())
- {
- SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths);
- }
- }
-
- SavedLocalState.FolderState = CurrentLocalFolderState;
-
- return SavedLocalState;
- }
-
- ChunkedFolderContent ScanAndChunkFolder(
- TransferThreadWorkers& Workers,
- GetFolderContentStatistics& GetFolderContentStats,
- ChunkingStatistics& ChunkingStats,
- const std::filesystem::path& Path,
- std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder,
- std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache)
- {
- Stopwatch Timer;
-
- ZEN_TRACE_CPU("ScanAndChunkFolder");
-
- FolderContent Content = GetFolderContent(
- GetFolderContentStats,
- Path,
- std::move(IsAcceptedFolder),
- std::move(IsAcceptedFile),
- Workers.GetIOWorkerPool(),
- GetUpdateDelayMS(ProgressMode),
- [](bool, std::ptrdiff_t) {},
- AbortFlag);
- if (AbortFlag)
- {
- return {};
- }
-
- BuildState LocalContent = GetLocalContent(Workers,
- GetFolderContentStats,
- ChunkingStats,
- Path,
- ZenStateFilePath(Path / ZenFolderName),
- ChunkController,
- ChunkCache)
- .State;
-
- std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths);
-
- BuildState UntrackedLocalContent =
- GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, ChunkCache, UntrackedPaths).State;
-
- ChunkedFolderContent Result = MergeChunkedFolderContents(LocalContent.ChunkedContent,
- std::vector<ChunkedFolderContent>{UntrackedLocalContent.ChunkedContent});
-
- const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0));
- const uint64_t ChunkedRawSize =
- std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));
-
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- Result.Paths.size(),
- NiceBytes(TotalRawSize),
- Result.ChunkedContent.ChunkHashes.size(),
- NiceBytes(ChunkedRawSize),
- Path,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
- }
- return Result;
- };
-
- struct DownloadOptions
- {
- std::filesystem::path SystemRootDir;
- std::filesystem::path ZenFolderPath;
- bool AllowMultiparts = true;
- EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed;
- bool CleanTargetFolder = false;
- bool PostDownloadVerify = false;
- bool PrimeCacheOnly = false;
- bool EnableOtherDownloadsScavenging = true;
- bool EnableTargetFolderScavenging = true;
- bool AllowFileClone = true;
- std::vector<std::string> IncludeWildcards;
- std::vector<std::string> ExcludeWildcards;
- uint64_t MaximumInMemoryPayloadSize = 512u * 1024u;
- bool PopulateCache = true;
- bool AppendNewContent = false;
- std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
- };
-
- void DownloadFolder(OperationLogOutput& Output,
- TransferThreadWorkers& Workers,
- StorageInstance& Storage,
- const BuildStorageCache::Statistics& StorageCacheStats,
- const Oid& BuildId,
- const std::vector<Oid>& BuildPartIds,
- std::span<const std::string> BuildPartNames,
- const std::filesystem::path& DownloadSpecPath,
- const std::filesystem::path& Path,
- const DownloadOptions& Options)
- {
- ZEN_TRACE_CPU("DownloadFolder");
-
- ProgressBar::SetLogOperationName(ProgressMode, "Download Folder");
-
- enum TaskSteps : uint32_t
- {
- CheckState,
- CompareState,
- Download,
- Verify,
- Cleanup,
- StepCount
- };
-
- auto EndProgress =
- MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
-
- ZEN_ASSERT((!Options.PrimeCacheOnly) ||
- (Options.PrimeCacheOnly && (Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off)));
-
- Stopwatch DownloadTimer;
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount);
-
- const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath);
- CreateDirectories(ZenTempFolder);
-
- std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
-
- CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);
-
- std::vector<std::pair<Oid, std::string>> AllBuildParts =
- ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
-
- BuildManifest Manifest;
- if (!DownloadSpecPath.empty())
- {
- const std::filesystem::path AbsoluteDownloadSpecPath =
- DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath);
- Manifest = ParseBuildManifest(DownloadSpecPath);
- }
-
- std::vector<ChunkedFolderContent> PartContents;
-
- std::unique_ptr<ChunkingController> ChunkController;
-
- std::vector<ChunkBlockDescription> BlockDescriptions;
- std::vector<IoHash> LooseChunkHashes;
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount);
-
- ChunkedFolderContent RemoteContent = GetRemoteContent(Output,
- Storage,
- BuildId,
- AllBuildParts,
- Manifest,
- Options.IncludeWildcards,
- Options.ExcludeWildcards,
- ChunkController,
- PartContents,
- BlockDescriptions,
- LooseChunkHashes,
- IsQuiet,
- IsVerbose,
- DoExtraContentVerify);
-
- const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
- GetFolderContentStatistics LocalFolderScanStats;
- ChunkingStatistics ChunkingStats;
-
- BuildSaveState LocalState;
-
- if (!Options.PrimeCacheOnly)
- {
- if (IsDir(Path))
- {
- if (!ChunkController && !IsQuiet)
- {
- ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default");
- ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- }
- std::unique_ptr<ChunkingCache> ChunkCache(CreateNullChunkingCache());
-
- LocalState = GetLocalContent(Workers,
- LocalFolderScanStats,
- ChunkingStats,
- Path,
- ZenStateFilePath(Path / ZenFolderName),
- *ChunkController,
- *ChunkCache);
-
- std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths);
-
- BuildSaveState UntrackedLocalContent = GetLocalStateFromPaths(Workers,
- LocalFolderScanStats,
- ChunkingStats,
- Path,
- *ChunkController,
- *ChunkCache,
- UntrackedPaths);
-
- if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty())
- {
- LocalState.State.ChunkedContent =
- MergeChunkedFolderContents(LocalState.State.ChunkedContent,
- std::vector<ChunkedFolderContent>{UntrackedLocalContent.State.ChunkedContent});
-
- // TODO: Helper
- LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(),
- UntrackedLocalContent.FolderState.Paths.begin(),
- UntrackedLocalContent.FolderState.Paths.end());
- LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(),
- UntrackedLocalContent.FolderState.RawSizes.begin(),
- UntrackedLocalContent.FolderState.RawSizes.end());
- LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(),
- UntrackedLocalContent.FolderState.Attributes.begin(),
- UntrackedLocalContent.FolderState.Attributes.end());
- LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(),
- UntrackedLocalContent.FolderState.ModificationTicks.begin(),
- UntrackedLocalContent.FolderState.ModificationTicks.end());
- }
-
- if (Options.AppendNewContent)
- {
- RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent,
- RemoteContent,
- Options.IncludeWildcards,
- Options.ExcludeWildcards);
- }
-#if ZEN_BUILD_DEBUG
- ValidateChunkedFolderContent(RemoteContent,
- BlockDescriptions,
- LooseChunkHashes,
- Options.IncludeWildcards,
- Options.ExcludeWildcards);
-#endif // ZEN_BUILD_DEBUG
- }
- else
- {
- CreateDirectories(Path);
- }
- }
- if (AbortFlag)
- {
- return;
- }
-
- LocalState.LocalPath = Path;
-
- {
- BuildsSelection::Build RemoteBuildState = {.Id = BuildId,
- .IncludeWildcards = Options.IncludeWildcards,
- .ExcludeWildcards = Options.ExcludeWildcards};
- RemoteBuildState.Parts.reserve(BuildPartIds.size());
- for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++)
- {
- RemoteBuildState.Parts.push_back(
- {BuildsSelection::BuildPart{.Id = BuildPartIds[PartIndex],
- .Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}});
- }
-
- if (Options.AppendNewContent)
- {
- LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState));
- }
- else
- {
- LocalState.State.Selection.Builds = std::vector<BuildsSelection::Build>{std::move(RemoteBuildState)};
- }
- }
-
- if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder &&
- CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent))
- {
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
- NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
- }
-
- Stopwatch WriteStateTimer;
-
- CbObject StateObject = CreateBuildSaveStateObject(LocalState);
- CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
- TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
- }
-
- AddDownloadedPath(Options.SystemRootDir,
- BuildsDownloadInfo{.Selection = LocalState.State.Selection,
- .LocalPath = Path,
- .StateFilePath = ZenStateFilePath(Options.ZenFolderPath),
- .Iso8601Date = DateTime::Now().ToIso8601()});
+ WriteFile(Path, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
}
else
{
- ExtendableStringBuilder<128> BuildPartString;
- for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts)
- {
- BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first));
- }
-
- uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0));
-
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
- }
-
- Stopwatch IndexTimer;
-
- const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent);
- const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
-
- if (!IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(Output, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
- }
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount);
-
- BuildsOperationUpdateFolder Updater(
- Output,
- Storage,
- AbortFlag,
- PauseFlag,
- Workers.GetIOWorkerPool(),
- Workers.GetNetworkPool(),
- BuildId,
- Path,
- LocalState.State.ChunkedContent,
- LocalLookup,
- RemoteContent,
- RemoteLookup,
- BlockDescriptions,
- LooseChunkHashes,
- BuildsOperationUpdateFolder::Options{
- .IsQuiet = IsQuiet,
- .IsVerbose = IsVerbose,
- .AllowFileClone = Options.AllowFileClone,
- .UseSparseFiles = UseSparseFiles,
- .SystemRootDir = Options.SystemRootDir,
- .ZenFolderPath = Options.ZenFolderPath,
- .LargeAttachmentSize = LargeAttachmentSize,
- .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
- .PartialBlockRequestMode = Options.PartialBlockRequestMode,
- .WipeTargetFolder = Options.CleanTargetFolder,
- .PrimeCacheOnly = Options.PrimeCacheOnly,
- .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging,
- .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent,
- .ValidateCompletedSequences = Options.PostDownloadVerify,
- .ExcludeFolders = Options.ExcludeFolders,
- .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize,
- .PopulateCache = Options.PopulateCache});
- {
- ProgressBar::PushLogOperation(ProgressMode, "Download");
- auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); });
- FolderContent UpdatedLocalFolderState;
- Updater.Execute(UpdatedLocalFolderState);
-
- LocalState.State.ChunkedContent = RemoteContent;
- LocalState.FolderState = std::move(UpdatedLocalFolderState);
- }
-
- VerifyFolderStatistics VerifyFolderStats;
- if (!AbortFlag)
- {
- if (!Options.PrimeCacheOnly)
- {
- AddDownloadedPath(Options.SystemRootDir,
- BuildsDownloadInfo{.Selection = LocalState.State.Selection,
- .LocalPath = Path,
- .StateFilePath = ZenStateFilePath(Options.ZenFolderPath),
- .Iso8601Date = DateTime::Now().ToIso8601()});
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount);
-
- VerifyFolder(Workers,
- RemoteContent,
- RemoteLookup,
- Path,
- Options.ExcludeFolders,
- Options.PostDownloadVerify,
- VerifyFolderStats);
-
- Stopwatch WriteStateTimer;
- CbObject StateObject = CreateBuildSaveStateObject(LocalState);
-
- CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
- TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
- }
-
-#if 0
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(StateObject, SB);
- WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
-#endif // 0
- }
- const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() +
- Updater.m_DownloadStats.DownloadedBlockCount.load() +
- Updater.m_DownloadStats.DownloadedPartialBlockCount.load();
- const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() +
- Updater.m_DownloadStats.DownloadedBlockByteCount.load() +
- Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load();
- const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs();
-
- if (!IsQuiet)
- {
- std::string CloneInfo;
- if (Updater.m_DiskStats.CloneByteCount > 0)
- {
- CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load()));
- }
-
- std::string DownloadDetails;
- {
- ExtendableStringBuilder<128> SB;
- BuildStorageBase::ExtendedStatistics ExtendedDownloadStats;
- if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats))
- {
- if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty())
- {
- for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource)
- {
- if (SB.Size() > 0)
- {
- SB.Append(", "sv);
- }
- SB.Append(It.first);
- SB.Append(": "sv);
- SB.Append(NiceBytes(It.second));
- }
- }
- }
- if (Storage.CacheStorage)
- {
- if (SB.Size() > 0)
- {
- SB.Append(", "sv);
- }
- SB.Append("Cache: ");
- SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load()));
- }
- if (SB.Size() > 0)
- {
- DownloadDetails = fmt::format(" ({})", SB.ToView());
- }
- }
-
- ZEN_CONSOLE(
- "Downloaded build {}, parts:{} in {}\n"
- " Scavenge: {} (Target: {}, Cache: {}, Others: {})\n"
- " Download: {} ({}) {}bits/s{}\n"
- " Write: {} ({}) {}B/s{}\n"
- " Clean: {}\n"
- " Finalize: {}\n"
- " Verify: {}",
- BuildId,
- BuildPartString.ToView(),
- NiceTimeSpanMs(DownloadTimeMs),
-
- NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs +
- Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs +
- Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) /
- 1000),
- NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000),
-
- DownloadCount,
- NiceBytes(DownloadByteCount),
- NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
- DownloadDetails,
-
- Updater.m_DiskStats.WriteCount.load(),
- NiceBytes(Updater.m_WrittenChunkByteCount.load()),
- NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())),
- CloneInfo,
-
- NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
-
- NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
-
- NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
- }
- }
- }
- if (Options.PrimeCacheOnly)
- {
- if (Storage.CacheStorage)
- {
- Storage.CacheStorage->Flush(5000, [](intptr_t Remaining) {
- if (!IsQuiet)
- {
- if (Remaining == 0)
- {
- ZEN_CONSOLE("Build cache upload complete");
- }
- else
- {
- ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining);
- }
- }
- return !AbortFlag;
- });
- }
- }
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
-
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), ZenTempFolder);
- }
-
- void ListBuild(StorageInstance& Storage,
- const Oid& BuildId,
- const std::vector<Oid>& BuildPartIds,
- std::span<const std::string> BuildPartNames,
- std::span<const std::string> IncludeWildcards,
- std::span<const std::string> ExcludeWildcards,
- CbObjectWriter* OptionalStructuredOutput)
- {
- std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
-
- CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId);
- OptionalStructuredOutput->AddObject("build"sv, BuildObject);
- }
-
- std::vector<std::pair<Oid, std::string>> AllBuildParts =
- ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
-
- if (!AllBuildParts.empty())
- {
- Stopwatch GetBuildPartTimer;
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->BeginArray("parts"sv);
- }
-
- for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++)
- {
- const Oid BuildPartId = AllBuildParts[BuildPartIndex].first;
- const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second;
- CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->BeginObject();
- OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId);
- OptionalStructuredOutput->AddString("partName"sv, BuildPartName);
- }
- {
- if (OptionalStructuredOutput != nullptr)
- {
- }
- else if (!IsQuiet)
- {
- ZEN_CONSOLE("{}Part: {} ('{}'):\n",
- BuildPartIndex > 0 ? "\n" : "",
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(BuildPartManifest.GetSize()));
- }
-
- std::vector<std::filesystem::path> Paths;
- std::vector<IoHash> RawHashes;
- std::vector<uint64_t> RawSizes;
- std::vector<uint32_t> Attributes;
-
- SourcePlatform Platform;
- std::vector<IoHash> SequenceRawHashes;
- std::vector<uint32_t> ChunkCounts;
- std::vector<uint32_t> AbsoluteChunkOrders;
- std::vector<IoHash> LooseChunkHashes;
- std::vector<uint64_t> LooseChunkRawSizes;
- std::vector<IoHash> BlockRawHashes;
-
- ReadBuildContentFromCompactBinary(BuildPartManifest,
- Platform,
- Paths,
- RawHashes,
- RawSizes,
- Attributes,
- SequenceRawHashes,
- ChunkCounts,
- AbsoluteChunkOrders,
- LooseChunkHashes,
- LooseChunkRawSizes,
- BlockRawHashes);
-
- std::vector<size_t> Order(Paths.size());
- std::iota(Order.begin(), Order.end(), 0);
-
- std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) {
- const std::filesystem::path& LhsPath = Paths[Lhs];
- const std::filesystem::path& RhsPath = Paths[Rhs];
- return LhsPath < RhsPath;
- });
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->BeginArray("files"sv);
- }
- {
- for (size_t Index : Order)
- {
- const std::filesystem::path& Path = Paths[Index];
- if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true))
- {
- const IoHash& RawHash = RawHashes[Index];
- const uint64_t RawSize = RawSizes[Index];
- const uint32_t Attribute = Attributes[Index];
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->BeginObject();
- {
- OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path));
- OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize);
- OptionalStructuredOutput->AddHash("rawHash"sv, RawHash);
- switch (Platform)
- {
- case SourcePlatform::Windows:
- OptionalStructuredOutput->AddInteger("attributes"sv, Attribute);
- break;
- case SourcePlatform::MacOS:
- case SourcePlatform::Linux:
- OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute));
- break;
- default:
- throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform));
- }
- }
- OptionalStructuredOutput->EndObject();
- }
- else
- {
- ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash);
- }
- }
- }
- }
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->EndArray(); // "files"
- }
- }
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->EndObject();
- }
- }
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->EndArray(); // parts
- }
- }
- }
-
- void DiffFolders(TransferThreadWorkers& Workers,
- const std::filesystem::path& BasePath,
- const std::filesystem::path& ComparePath,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache,
- const std::vector<std::string>& ExcludeFolders,
- const std::vector<std::string>& ExcludeExtensions)
- {
- ZEN_TRACE_CPU("DiffFolders");
-
- ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders");
-
- enum TaskSteps : uint32_t
- {
- CheckBase,
- CheckCompare,
- Diff,
- Cleanup,
- StepCount
- };
-
- auto EndProgress =
- MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
-
- ChunkedFolderContent BaseFolderContent;
- ChunkedFolderContent CompareFolderContent;
-
- {
- auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool {
- for (const std::string& ExcludeFolder : ExcludeFolders)
- {
- if (RelativePath.starts_with(ExcludeFolder))
- {
- if (RelativePath.length() == ExcludeFolder.length())
- {
- return false;
- }
- else if (RelativePath[ExcludeFolder.length()] == '/')
- {
- return false;
- }
- }
- }
- return true;
- };
-
- auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool {
- for (const std::string& ExcludeExtension : ExcludeExtensions)
- {
- if (RelativePath.ends_with(ExcludeExtension))
- {
- return false;
- }
- }
- return true;
- };
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount);
-
- GetFolderContentStatistics BaseGetFolderContentStats;
- ChunkingStatistics BaseChunkingStats;
- BaseFolderContent = ScanAndChunkFolder(Workers,
- BaseGetFolderContentStats,
- BaseChunkingStats,
- BasePath,
- IsAcceptedFolder,
- IsAcceptedFile,
- ChunkController,
- ChunkCache);
- if (AbortFlag)
- {
- return;
- }
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount);
-
- GetFolderContentStatistics CompareGetFolderContentStats;
- ChunkingStatistics CompareChunkingStats;
- CompareFolderContent = ScanAndChunkFolder(Workers,
- CompareGetFolderContentStats,
- CompareChunkingStats,
- ComparePath,
- IsAcceptedFolder,
- IsAcceptedFile,
- ChunkController,
- ChunkCache);
-
- if (AbortFlag)
- {
- return;
- }
- }
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount);
-
- std::vector<IoHash> AddedHashes;
- std::vector<IoHash> RemovedHashes;
- uint64_t RemovedSize = 0;
- uint64_t AddedSize = 0;
-
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BaseRawHashLookup;
- for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++)
- {
- const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex];
- BaseRawHashLookup.insert_or_assign(RawHash, PathIndex);
- }
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CompareRawHashLookup;
- for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++)
- {
- const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex];
- if (!BaseRawHashLookup.contains(RawHash))
- {
- AddedHashes.push_back(RawHash);
- AddedSize += CompareFolderContent.RawSizes[PathIndex];
- }
- CompareRawHashLookup.insert_or_assign(RawHash, PathIndex);
- }
- for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++)
- {
- const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex];
- if (!CompareRawHashLookup.contains(RawHash))
- {
- RemovedHashes.push_back(RawHash);
- RemovedSize += BaseFolderContent.RawSizes[PathIndex];
- }
- }
-
- uint64_t BaseTotalRawSize = 0;
- for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++)
- {
- BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex];
- }
-
- double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0;
-
- ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept",
- RemovedHashes.size(),
- NiceBytes(RemovedSize),
- AddedHashes.size(),
- NiceBytes(AddedSize),
- BaseFolderContent.Paths.size() - RemovedHashes.size(),
- NiceBytes(BaseTotalRawSize - RemovedSize),
- KeptPercent);
-
- uint64_t CompareTotalRawSize = 0;
-
- uint64_t FoundChunkCount = 0;
- uint64_t FoundChunkSize = 0;
- uint64_t NewChunkCount = 0;
- uint64_t NewChunkSize = 0;
- const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent);
- for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++)
- {
- const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex];
- if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash))
- {
- FoundChunkCount++;
- FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- }
- else
- {
- NewChunkCount++;
- NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- }
- CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(ResponseView, SB);
+ WriteFile(Path, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
}
-
- double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0;
- double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0;
-
- ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
- FoundChunkCount,
- NiceBytes(FoundChunkSize),
- FoundPercent,
- CompareFolderContent.ChunkedContent.ChunkHashes.size(),
- NiceBytes(CompareTotalRawSize),
- BaseFolderContent.ChunkedContent.ChunkHashes.size(),
- NiceBytes(BaseTotalRawSize),
- NewChunkCount,
- NiceBytes(NewChunkSize),
- NewPercent);
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
}
} // namespace builds_impl
-//////////////////////////////////////////////////////////////////////////////////////////////////////
-// BuildsCommand — Option-adding helpers
-//
+//////////////////////////////////////////////////////////////////////////
void
-BuildsCommand::AddSystemOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddSystemOptions(cxxopts::Options& Ops)
{
- Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>");
+ Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(SystemRootDir), "<systemdir>");
Ops.add_option("",
"",
"use-sparse-files",
"Enable use of sparse files when writing large files. Defaults to true.",
- cxxopts::value(m_UseSparseFiles),
+ cxxopts::value(UseSparseFiles),
"<usesparsefiles>");
}
void
-BuildsCommand::AddCloudOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddCloudOptions(cxxopts::Options& Ops)
{
- m_AuthOptions.AddOptions(Ops);
+ AuthOptions.AddOptions(Ops);
- Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>");
- Ops.add_option("cloud build",
- "",
- "url",
- "Cloud Builds host url (legacy - use --override-host)",
- cxxopts::value(m_OverrideHost),
- "<url>");
- Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>");
- Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>");
+ Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(OverrideHost), "<override-host>");
+ Ops.add_option("cloud build", "", "url", "Cloud Builds host url (legacy - use --override-host)", cxxopts::value(OverrideHost), "<url>");
+ Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(Url), "<cloud-url>");
+ Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(Host), "<host>");
Ops.add_option("cloud build",
"",
"assume-http2",
"Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake",
- cxxopts::value(m_AssumeHttp2),
+ cxxopts::value(AssumeHttp2),
"<assumehttp2>");
Ops.add_option("cloud build",
"",
"verbose-http",
"Enable verbose option for http client",
- cxxopts::value(m_VerboseHttp),
+ cxxopts::value(VerboseHttp),
"<verbosehttp>");
- Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>");
- Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>");
- Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(m_AllowRedirect), "<allow-redirect>");
+ Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(Namespace), "<namespace>");
+ Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(Bucket), "<bucket>");
+ Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(AllowRedirect), "<allow-redirect>");
}
void
-BuildsCommand::AddFileOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddFileOptions(cxxopts::Options& Ops)
{
- Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>");
+ Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(StoragePath), "<storagepath>");
Ops.add_option("filestorage",
"",
"json-metadata",
"Write build, part and block metadata as .json files in addition to .cb files",
- cxxopts::value(m_WriteMetadataAsJson),
+ cxxopts::value(WriteMetadataAsJson),
"<jsonmetadata>");
}
void
-BuildsCommand::AddCacheOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddCacheOptions(cxxopts::Options& Ops)
{
- Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>");
+ Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(ZenCacheHost), "<zenhost>");
}
void
-BuildsCommand::AddOutputOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddOutputOptions(cxxopts::Options& Ops)
{
- Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<plainprogress>");
- Ops.add_option("output",
- "",
- "log-progress",
- "Write @progress style progress to output",
- cxxopts::value(m_LogProgress),
- "<logprogress>");
- Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
- Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>");
+ Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(PlainProgress), "<plainprogress>");
+ Ops.add_option("output", "", "log-progress", "Write @progress style progress to output", cxxopts::value(LogProgress), "<logprogress>");
+ Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(Verbose), "<verbose>");
+ Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(Quiet), "<quiet>");
}
void
-BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddWorkerOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"boost-worker-count",
"Increase the number of worker threads - may cause computer to be less responsive",
- cxxopts::value(m_BoostWorkerCount),
+ cxxopts::value(BoostWorkerCount),
"<boostworkercount>");
Ops.add_option("",
@@ -2106,47 +298,47 @@ BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops)
"boost-worker-memory",
"Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to "
"be less responsive due to high memory usage",
- cxxopts::value(m_BoostWorkerMemory),
+ cxxopts::value(BoostWorkerMemory),
"<boostworkermemory>");
Ops.add_option("",
"",
"boost-workers",
"Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive",
- cxxopts::value(m_BoostWorkers),
+ cxxopts::value(BoostWorkers),
"<boostworkermemory>");
}
void
-BuildsCommand::AddZenFolderOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddZenFolderOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"zen-folder-path",
- fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", builds_impl::ZenFolderName),
- cxxopts::value(m_ZenFolderPath),
- "<boostworkers>");
+ "Path to the zen state and temp folder used by this command. Overrides the command-specific default.",
+ cxxopts::value(ZenFolderPath),
+ "<path>");
}
void
-BuildsCommand::AddChunkingCacheOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddChunkingCacheOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"chunking-cache-path",
"Path to cache for chunking information of scanned files. Default is empty resulting in no caching",
- cxxopts::value(m_ChunkingCachePath),
+ cxxopts::value(ChunkingCachePath),
"<chunkingcachepath>");
}
void
-BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddWildcardOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"wildcard",
"Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;",
- cxxopts::value(m_IncludeWildcard),
+ cxxopts::value(IncludeWildcard),
"<wildcard>");
Ops.add_option("",
@@ -2154,46 +346,46 @@ BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops)
"exclude-wildcard",
"Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard "
"include filter",
- cxxopts::value(m_ExcludeWildcard),
+ cxxopts::value(ExcludeWildcard),
"<excludewildcard>");
}
void
-BuildsCommand::AddExcludeFolderOption(cxxopts::Options& Ops)
+BuildsConfiguration::AddExcludeFolderOption(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"exclude-folders",
"Names of folders to exclude, separated by ;",
- cxxopts::value(m_ExcludeFolders),
+ cxxopts::value(ExcludeFolders),
"<excludefolders>");
}
void
-BuildsCommand::AddExcludeExtensionsOption(cxxopts::Options& Ops)
+BuildsConfiguration::AddExcludeExtensionsOption(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"exclude-extensions",
"Extensions to exclude, separated by ;"
"include filter",
- cxxopts::value(m_ExcludeExtensions),
+ cxxopts::value(ExcludeExtensions),
"<excludeextensions>");
}
void
-BuildsCommand::AddMultipartOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddMultipartOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"allow-multipart",
"Allow large attachments to be transfered using multipart protocol. Defaults to true.",
- cxxopts::value(m_AllowMultiparts),
+ cxxopts::value(AllowMultiparts),
"<allowmultipart>");
}
void
-BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddPartialBlockRequestOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
@@ -2206,12 +398,12 @@ BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops)
"allowed to host\n"
" true = multiple partial block ranges requests per block allowed to zen cache and host\n"
"Defaults to 'mixed'.",
- cxxopts::value(m_AllowPartialBlockRequests),
+ cxxopts::value(AllowPartialBlockRequests),
"<allowpartialblockrequests>");
}
void
-BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddAppendNewContentOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
@@ -2220,26 +412,26 @@ BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops)
" false = the local content will be replaced by the remote content\n"
" true = the remote data will be overlayed on top of local data\n"
"Defaults to false.",
- cxxopts::value(m_AppendNewContent),
+ cxxopts::value(AppendNewContent),
"<append>");
}
BuildsCommand::BuildsCommand()
-: m_ListNamespacesSubCmd(*this)
-, m_ListSubCmd(*this)
-, m_ListBlocksSubCmd(*this)
-, m_UploadSubCmd(*this)
-, m_DownloadSubCmd(*this)
-, m_LsSubCmd(*this)
-, m_DiffSubCmd(*this)
-, m_FetchBlobSubCmd(*this)
-, m_PrimeCacheSubCmd(*this)
-, m_PauseSubCmd(*this)
-, m_ResumeSubCmd(*this)
-, m_AbortSubCmd(*this)
-, m_ValidatePartSubCmd(*this)
-, m_TestSubCmd(*this)
-, m_MultiTestDownloadSubCmd(*this)
+: m_ListNamespacesSubCmd(m_Configuration)
+, m_ListSubCmd(m_Configuration)
+, m_ListBlocksSubCmd(m_Configuration)
+, m_UploadSubCmd(m_Configuration)
+, m_DownloadSubCmd(m_Configuration)
+, m_LsSubCmd(m_Configuration)
+, m_DiffSubCmd(m_Configuration)
+, m_FetchBlobSubCmd(m_Configuration)
+, m_PrimeCacheSubCmd(m_Configuration)
+, m_PauseSubCmd(m_Configuration)
+, m_ResumeSubCmd(m_Configuration)
+, m_AbortSubCmd(m_Configuration)
+, m_ValidatePartSubCmd(m_Configuration)
+, m_TestSubCmd(m_Configuration)
+, m_MultiTestDownloadSubCmd(m_Configuration)
{
m_Options.add_options()("h,help", "Print help");
m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), "");
@@ -2269,174 +461,250 @@ BuildsCommand::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/)
{
using namespace builds_impl;
- signal(SIGINT, SignalCallbackHandler);
+ // Held in member variables so the handlers stay installed through the
+ // subcommand's Run() and are restored when BuildsCommand is destroyed.
+ m_SigIntGuard.emplace(SIGINT, SignalCallbackHandler);
#if ZEN_PLATFORM_WINDOWS
- signal(SIGBREAK, SignalCallbackHandler);
-#endif // ZEN_PLATFORM_WINDOWS
+ m_SigBreakGuard.emplace(SIGBREAK, SignalCallbackHandler);
+#endif
// Validate output options
- if (m_Verbose && m_Quiet)
+ if (m_Configuration.Verbose && m_Configuration.Quiet)
{
throw OptionParseException("'--verbose' conflicts with '--quiet'", {});
}
- if (m_LogProgress && m_PlainProgress)
+ if (m_Configuration.LogProgress && m_Configuration.PlainProgress)
{
throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", {});
}
- if (m_LogProgress && m_Quiet)
+ if (m_Configuration.LogProgress && m_Configuration.Quiet)
{
throw OptionParseException("'--quiet' conflicts with '--log-progress'", {});
}
- if (m_PlainProgress && m_Quiet)
+ if (m_Configuration.PlainProgress && m_Configuration.Quiet)
{
throw OptionParseException("'--quiet' conflicts with '--plain-progress'", {});
}
- IsVerbose = m_Verbose;
- IsQuiet = m_Quiet;
- if (m_LogProgress)
- {
- ProgressMode = ProgressBar::Mode::Log;
- }
- else if (m_PlainProgress)
+ if (m_Configuration.LogProgress)
{
- ProgressMode = ProgressBar::Mode::Plain;
+ m_Configuration.ProgressMode = ConsoleProgressMode::Log;
}
- else if (m_Verbose)
+ else if (m_Configuration.PlainProgress)
{
- ProgressMode = ProgressBar::Mode::Plain;
+ m_Configuration.ProgressMode = ConsoleProgressMode::Plain;
}
- else if (IsQuiet)
+ else if (m_Configuration.Quiet)
{
- ProgressMode = ProgressBar::Mode::Quiet;
+ m_Configuration.ProgressMode = ConsoleProgressMode::Quiet;
}
else
{
- ProgressMode = ProgressBar::Mode::Pretty;
+ m_Configuration.ProgressMode = ConsoleProgressMode::Pretty;
}
- if (m_BoostWorkers)
+ if (m_Configuration.BoostWorkers)
{
- m_BoostWorkerCount = true;
- m_BoostWorkerMemory = true;
+ m_Configuration.BoostWorkerCount = true;
+ m_Configuration.BoostWorkerMemory = true;
}
// Parse system options
- if (m_SystemRootDir.empty())
+ if (m_Configuration.SystemRootDir.empty())
{
- m_SystemRootDir = PickDefaultSystemRootDirectory();
+ m_Configuration.SystemRootDir = PickDefaultSystemRootDirectory();
}
- MakeSafeAbsolutePathInPlace(m_SystemRootDir);
-
- UseSparseFiles = m_UseSparseFiles;
+ MakeSafeAbsolutePathInPlace(m_Configuration.SystemRootDir);
+ MakeSafeAbsolutePathInPlace(m_Configuration.ChunkingCachePath);
return true;
}
+std::atomic<bool>&
+BuildsSubCmdBase::AbortFlag() const
+{
+ return builds_impl::AbortFlag;
+}
+
+std::atomic<bool>&
+BuildsSubCmdBase::PauseFlag() const
+{
+ return builds_impl::PauseFlag;
+}
+
+std::unique_ptr<ProgressBase>
+BuildsSubCmdBase::CreateProgress() const
+{
+ return std::unique_ptr<ProgressBase>(CreateConsoleProgress(m_Config.ProgressMode));
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+BuildsSubCmdBase::LogBanner()
+{
+ if (!m_Config.Quiet)
+ {
+ ZenCmdBase::LogExecutableVersionAndPid();
+ }
+}
+
void
-BuildsCommand::ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::LogWorkersInfo(const TransferThreadWorkers& Workers)
{
- if (!m_Url.empty())
+ if (!m_Config.Quiet)
{
- if (!m_Host.empty())
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
+ }
+}
+
+void
+BuildsSubCmdBase::EnsureZenFolderExists()
+{
+ m_CreatedZenFolder = CreateDirectories(GetZenFolderPath());
+}
+
+void
+BuildsSubCmdBase::CleanZenFolder()
+{
+ using namespace builds_impl;
+ WorkerThreadPool& Pool = GetSmallWorkerPool(EWorkloadType::Burst);
+ if (m_CreatedZenFolder)
+ {
+ CleanAndRemoveDirectory(Pool, AbortFlag(), PauseFlag(), GetZenFolderPath());
+ }
+ else
+ {
+ CleanAndRemoveDirectory(Pool, AbortFlag(), PauseFlag(), ZenTempFolderPath(GetZenFolderPath()));
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+BuildsSubCmdBase::ResolvedStorage
+BuildsSubCmdBase::ParseStorageOptions(std::string& BuildId,
+ const CreateBuildStorageOptions& Options,
+ cxxopts::Options& SubOpts,
+ const std::filesystem::path& SystemRootDirOverride,
+ const std::filesystem::path& StoragePathOverride)
+{
+ ResolvedStorage Resolved{.SystemRootDir = SystemRootDirOverride.empty() ? m_Config.SystemRootDir : SystemRootDirOverride,
+ .Host = m_Config.Host,
+ .Namespace = m_Config.Namespace,
+ .Bucket = m_Config.Bucket,
+ .StoragePath = StoragePathOverride.empty() ? m_Config.StoragePath : StoragePathOverride};
+
+ if (!m_Config.Url.empty())
+ {
+ if (!Resolved.Host.empty())
{
- throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOpts.help());
+ throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", Resolved.Host, m_Config.Url),
+ SubOpts.help());
}
- if (!m_Bucket.empty())
+ if (!Resolved.Bucket.empty())
{
- throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), SubOpts.help());
+ throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", Resolved.Bucket, m_Config.Url),
+ SubOpts.help());
}
if (!BuildId.empty())
{
- throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Url), SubOpts.help());
+ throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Config.Url),
+ SubOpts.help());
}
- if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, BuildId))
+ if (!ParseBuildStorageUrl(m_Config.Url, Resolved.Host, Resolved.Namespace, Resolved.Bucket, BuildId))
{
- throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOpts.help());
+ throw OptionParseException(
+ fmt::format("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", m_Config.Url),
+ SubOpts.help());
}
}
- if (!m_OverrideHost.empty() || !m_Host.empty())
+ if (!m_Config.OverrideHost.empty() || !Resolved.Host.empty())
{
- if (!m_StoragePath.empty())
+ if (!Resolved.StoragePath.empty())
{
throw OptionParseException(
- fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath),
+ fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", Resolved.StoragePath),
SubOpts.help());
}
- if (RequireNamespace && m_Namespace.empty())
+ if (Options.RequireNamespace && Resolved.Namespace.empty())
{
throw OptionParseException("'--namespace' is required", SubOpts.help());
}
- if (RequireBucket && m_Bucket.empty())
+ if (Options.RequireBucket && Resolved.Bucket.empty())
{
throw OptionParseException("'--bucket' is required", SubOpts.help());
}
}
- else if (m_StoragePath.empty())
+ else if (Resolved.StoragePath.empty())
{
throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help());
}
- MakeSafeAbsolutePathInPlace(m_StoragePath);
+ MakeSafeAbsolutePathInPlace(Resolved.StoragePath);
+ return Resolved;
}
StorageInstance
-BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
- BuildStorageCache::Statistics& StorageCacheStats,
- const std::filesystem::path& TempPath,
- std::string& BuildId,
- bool RequireNamespace,
- bool RequireBucket,
- bool BoostCacheBackgroundWorkerPool,
- std::unique_ptr<AuthMgr>& Auth,
- cxxopts::Options& SubOpts)
+BuildsSubCmdBase::CreateBuildStorage(const std::filesystem::path& ZenFolder,
+ const CreateBuildStorageOptions& Options,
+ cxxopts::Options& SubOpts,
+ BuildStorageBase::Statistics& StorageStats,
+ BuildStorageCache::Statistics& StorageCacheStats,
+ std::unique_ptr<AuthMgr>& Auth,
+ const ResolvedStorage& Resolved)
{
using namespace builds_impl;
- ParseStorageOptions(BuildId, RequireNamespace, RequireBucket, SubOpts);
+ // Empty ZenFolder => operate without a temp directory and keep all downloads in memory.
+ const bool HasZenFolder = !ZenFolder.empty();
+ const std::filesystem::path StorageTempPath = HasZenFolder ? ZenTempFolderPath(ZenFolder) / "storage" : std::filesystem::path{};
+ const std::filesystem::path CacheTempPath = HasZenFolder ? ZenTempFolderPath(ZenFolder) / "zencache" : std::filesystem::path{};
- HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
- .AssumeHttp2 = m_AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 2,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)};
+ HttpClientSettings ClientSettings{
+ .LogCategory = "httpbuildsclient",
+ .AssumeHttp2 = m_Config.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 2,
+ .Verbose = m_Config.VerboseHttp,
+ .MaximumInMemoryDownloadSize = HasZenFolder ? GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)
+ : std::numeric_limits<uint64_t>::max()};
std::string StorageDescription;
std::string CacheDescription;
StorageInstance Result;
- if (!m_Host.empty() || !m_OverrideHost.empty())
+ if (!Resolved.Host.empty() || !m_Config.OverrideHost.empty())
{
- m_AuthOptions.ParseOptions(SubOpts,
- m_SystemRootDir,
- ClientSettings,
- m_Host.empty() ? m_OverrideHost : m_Host,
- Auth,
- IsQuiet,
- /*Hidden*/ false,
- m_Verbose);
-
- BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*CreateConsoleLogOutput(ProgressMode),
+ AuthCommandLineOptions AuthOpts = m_Config.AuthOptions;
+ AuthOpts.ParseOptions(SubOpts,
+ Resolved.SystemRootDir,
+ ClientSettings,
+ Resolved.Host.empty() ? m_Config.OverrideHost : Resolved.Host,
+ Auth,
+ m_Config.Quiet,
+ /*Hidden*/ false,
+ m_Config.Verbose);
+
+ BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(),
ClientSettings,
- m_Host,
- m_OverrideHost,
- m_ZenCacheHost,
+ Resolved.Host,
+ m_Config.OverrideHost,
+ m_Config.ZenCacheHost,
ZenCacheResolveMode::All,
- m_Verbose);
+ m_Config.Verbose);
if (!ResolveRes.Cloud.Address.empty())
{
ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2;
Result.BuildStorageHttp =
- std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); });
+ std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, [this]() { return AbortFlag().load(); });
Result.BuildStorage = CreateJupiterBuildStorage(Log(),
*Result.BuildStorageHttp,
StorageStats,
- m_Namespace,
- m_Bucket,
- m_AllowRedirect,
- TempPath / "storage");
+ Resolved.Namespace,
+ Resolved.Bucket,
+ m_Config.AllowRedirect,
+ StorageTempPath);
Result.BuildStorageHost = ResolveRes.Cloud;
uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0;
@@ -2446,8 +714,8 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
ResolveRes.Cloud.Name,
(ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address),
Result.BuildStorageHttp->GetSessionId(),
- m_Namespace,
- m_Bucket,
+ Resolved.Namespace,
+ Resolved.Bucket,
NiceLatencyNs(HostLatencyNs));
if (!ResolveRes.Cache.Address.empty())
@@ -2461,19 +729,21 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
.AssumeHttp2 = ResolveRes.Cache.AssumeHttp2,
.AllowResume = true,
.RetryCount = 0,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
- []() { return AbortFlag.load(); });
+ .Verbose = m_Config.VerboseHttp,
+ .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)},
+ [this]() { return AbortFlag().load(); });
Result.CacheStorage =
CreateZenBuildStorageCache(*Result.CacheHttp,
StorageCacheStats,
- m_Namespace,
- m_Bucket,
- TempPath / "zencache",
- BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
- : GetTinyWorkerPool(EWorkloadType::Background));
+ Resolved.Namespace,
+ Resolved.Bucket,
+ CacheTempPath,
+ Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background)
+ : GetTinyWorkerPool(EWorkloadType::Background));
Result.CacheHost = ResolveRes.Cache;
+ Result.SetupCacheSession(ResolveRes.Cache.Address, fmt::format("builds {}", m_SubOptions.program()), GetSessionId());
+
uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0;
CacheDescription =
@@ -2483,69 +753,71 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
Result.CacheHttp->GetSessionId(),
NiceLatencyNs(CacheLatencyNs));
- if (!m_Namespace.empty())
+ if (!Resolved.Namespace.empty())
{
- CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
+ CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace);
}
- if (!m_Bucket.empty())
+ if (!Resolved.Bucket.empty())
{
- CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
+ CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket);
}
}
}
}
- else if (!m_StoragePath.empty())
+ else if (!Resolved.StoragePath.empty())
{
- StorageDescription = fmt::format("folder {}", m_StoragePath);
- Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
+ StorageDescription = fmt::format("folder {}", Resolved.StoragePath);
+ Result.BuildStorage = CreateFileBuildStorage(Resolved.StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
- Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(),
+ Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = Resolved.StoragePath.generic_string(),
.Name = "Disk",
.LatencySec = 1.0 / 100000, // 1 us
.Caps = {.MaxRangeCountPerRequest = 2048u}};
- if (!m_ZenCacheHost.empty())
+ if (!m_Config.ZenCacheHost.empty())
{
- ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp);
+ ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_Config.ZenCacheHost, m_Config.AssumeHttp2, m_Config.VerboseHttp);
if (TestResult.Success)
{
Result.CacheHttp = std::make_unique<HttpClient>(
- m_ZenCacheHost,
+ m_Config.ZenCacheHost,
HttpClientSettings{
.LogCategory = "httpcacheclient",
.ConnectTimeout = std::chrono::milliseconds{3000},
.Timeout = std::chrono::milliseconds{30000},
- .AssumeHttp2 = m_AssumeHttp2,
+ .AssumeHttp2 = m_Config.AssumeHttp2,
.AllowResume = true,
.RetryCount = 0,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
- []() { return AbortFlag.load(); });
+ .Verbose = m_Config.VerboseHttp,
+ .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)},
+ [this]() { return AbortFlag().load(); });
Result.CacheStorage =
CreateZenBuildStorageCache(*Result.CacheHttp,
StorageCacheStats,
- m_Namespace,
- m_Bucket,
- TempPath / "zencache",
- BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
- : GetTinyWorkerPool(EWorkloadType::Background));
- Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_ZenCacheHost,
- .Name = m_ZenCacheHost,
- .AssumeHttp2 = m_AssumeHttp2,
+ Resolved.Namespace,
+ Resolved.Bucket,
+ CacheTempPath,
+ Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background)
+ : GetTinyWorkerPool(EWorkloadType::Background));
+ Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_Config.ZenCacheHost,
+ .Name = m_Config.ZenCacheHost,
+ .AssumeHttp2 = m_Config.AssumeHttp2,
.LatencySec = TestResult.LatencySeconds,
.Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}};
+ Result.SetupCacheSession(m_Config.ZenCacheHost, fmt::format("builds {}", m_SubOptions.program()), GetSessionId());
+
CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId());
- if (!m_Namespace.empty())
+ if (!Resolved.Namespace.empty())
{
- CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
+ CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace);
}
- if (!m_Bucket.empty())
+ if (!Resolved.Bucket.empty())
{
- CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
+ CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket);
}
}
}
@@ -2555,7 +827,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help());
}
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("Remote: {}", StorageDescription);
if (!Result.CacheHost.Name.empty())
@@ -2567,45 +839,33 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
}
Oid
-BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts)
{
- if (BuildIdStr.length() != Oid::StringLength)
+ const Oid BuildId = Oid::TryFromHexString(BuildIdStr);
+ if (BuildId == Oid::Zero)
{
throw OptionParseException(
- fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildIdStr, Oid::StringLength),
+ fmt::format("'--build-id' ('{}') is malformed, expected {} hex characters", BuildIdStr, Oid::StringLength),
SubOpts.help());
}
- else if (Oid BuildId = Oid::FromHexString(BuildIdStr); BuildId == Oid::Zero)
- {
- throw OptionParseException(fmt::format("'--build-id' ('{}') is invalid", BuildIdStr), SubOpts.help());
- }
- else
- {
- return BuildId;
- }
+ return BuildId;
}
Oid
-BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts)
{
- if (BuildPartIdStr.length() != Oid::StringLength)
+ const Oid BuildPartId = Oid::TryFromHexString(BuildPartIdStr);
+ if (BuildPartId == Oid::Zero)
{
throw OptionParseException(
- fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildPartIdStr, Oid::StringLength),
+ fmt::format("'--build-part-id' ('{}') is malformed, expected {} hex characters", BuildPartIdStr, Oid::StringLength),
SubOpts.help());
}
- else if (Oid BuildPartId = Oid::FromHexString(BuildPartIdStr); BuildPartId == Oid::Zero)
- {
- throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildPartIdStr), SubOpts.help());
- }
- else
- {
- return BuildPartId;
- }
+ return BuildPartId;
}
std::vector<Oid>
-BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts)
{
std::vector<Oid> BuildPartIds;
for (const std::string& BuildPartId : BuildPartIdStrs)
@@ -2620,7 +880,7 @@ BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs
}
std::vector<std::string>
-BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts)
{
std::vector<std::string> BuildPartNames;
for (const std::string& BuildPartName : BuildPartNameStrs)
@@ -2635,10 +895,10 @@ BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartName
}
CbObject
-BuildsCommand::ParseBuildMetadata(bool CreateBuild,
- std::filesystem::path& BuildMetadataPath,
- const std::string& BuildMetadata,
- cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildMetadata(bool CreateBuild,
+ std::filesystem::path& BuildMetadataPath,
+ const std::string& BuildMetadata,
+ cxxopts::Options& SubOpts)
{
if (CreateBuild)
{
@@ -2697,7 +957,7 @@ BuildsCommand::ParseBuildMetadata(bool CreateBuild,
}
void
-BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts)
{
if (Path.empty())
{
@@ -2707,7 +967,7 @@ BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts)
}
IoHash
-BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts)
{
if (BlobHashStr.empty())
{
@@ -2731,7 +991,7 @@ BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& S
}
void
-BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards)
+BuildsSubCmdBase::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards)
{
auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) {
ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) {
@@ -2757,12 +1017,13 @@ BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, s
});
};
- SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards);
- SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards);
+ SplitAndAppendWildcard(m_Config.IncludeWildcard, OutIncludeWildcards);
+ SplitAndAppendWildcard(m_Config.ExcludeWildcard, OutExcludeWildcards);
}
void
-BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions)
+BuildsSubCmdBase::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders,
+ std::vector<std::string>& OutExcludeExtensions)
{
auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) {
ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) {
@@ -2779,38 +1040,50 @@ BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExclu
});
};
- SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders);
- SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions);
+ SplitAndAppendExclusion(m_Config.ExcludeFolders, OutExcludeFolders);
+ SplitAndAppendExclusion(m_Config.ExcludeExtensions, OutExcludeExtensions);
+}
+
+void
+BuildsSubCmdBase::SetZenFolderPath(const std::filesystem::path& Fallback)
+{
+ m_ResolvedZenFolderPath = m_Config.ZenFolderPath.empty() ? Fallback : m_Config.ZenFolderPath;
+ MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath);
}
void
-BuildsCommand::ResolveZenFolderPath(const std::filesystem::path& DefaultPath)
+BuildsSubCmdBase::ResolveZenFolderPath(const std::filesystem::path& LocalPath)
{
- if (m_ZenFolderPath.empty())
+ using namespace builds_impl;
+ if (!m_Config.ZenFolderPath.empty())
+ {
+ m_ResolvedZenFolderPath = m_Config.ZenFolderPath;
+ }
+ else if (!LocalPath.empty())
+ {
+ m_ResolvedZenFolderPath = LocalPath / ZenFolderName;
+ }
+ else
{
- m_ZenFolderPath = DefaultPath;
+ m_ResolvedZenFolderPath = std::filesystem::current_path() / ZenFolderName;
}
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath);
}
EPartialBlockRequestMode
-BuildsCommand::ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts)
{
- if (PrimeCacheOnly)
- {
- return EPartialBlockRequestMode::Off;
- }
- EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests);
+ EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_Config.AllowPartialBlockRequests);
if (Mode == EPartialBlockRequestMode::Invalid)
{
- throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests),
+ throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_Config.AllowPartialBlockRequests),
SubOpts.help());
}
return Mode;
}
void
-BuildsCommand::ParseZenProcessId(int& ZenProcessId)
+BuildsSubCmdBase::ParseZenProcessId(int& ZenProcessId)
{
if (ZenProcessId == -1)
{
@@ -2831,20 +1104,15 @@ BuildsCommand::ParseZenProcessId(int& ZenProcessId)
//////////////////////////////////////////////////////////////////////////
-// ---------------------------------------------------------------------------
-// Subcommand implementations
-// ---------------------------------------------------------------------------
-
-BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("list-namespaces", "List all namespaces and optionally their buckets")
-, m_Parent(Parent)
+BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "list-namespaces", "List all namespaces and optionally their buckets")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "recursive", "Enable fetch of buckets within namespaces also", cxxopts::value(m_Recursive), "<recursive>");
Opts.add_option("",
"",
@@ -2859,36 +1127,20 @@ BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent)
void
BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
if (!m_ResultPath.empty())
{
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ LogBanner();
}
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
-
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); });
+ cxxopts::Options& Opts = SubOptions();
- std::unique_ptr<AuthMgr> Auth;
- std::string DummyBuildId;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- DummyBuildId,
- /*RequireNamespace*/ false,
- /*RequireBucket*/ false,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
+ std::string DummyBuildId;
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireNamespace = false, .RequireBucket = false}, Opts);
+ // Read-only listing: no local zen folder needed; downloads stay in memory.
+ StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
CbObject Response = Storage.BuildStorage->ListNamespaces(m_Recursive);
ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
@@ -2900,29 +1152,20 @@ BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
}
else
{
- std::filesystem::path ResultPath = MakeSafeAbsolutePath(m_ResultPath);
- if (ToLower(ResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ builds_impl::WriteResultObject(MakeSafeAbsolutePath(m_ResultPath), Response);
}
}
-BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list", "List builds matching a query"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsListSubCmd::BuildsListSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "list", "List builds matching a query")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("",
"",
"query-path",
@@ -2942,19 +1185,16 @@ BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list"
void
BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ if (!m_ResultPath.empty())
+ {
+ LogBanner();
+ }
+
+ cxxopts::Options& Opts = SubOptions();
MakeSafeAbsolutePathInPlace(m_QueryPath);
MakeSafeAbsolutePathInPlace(m_ResultPath);
- if (!m_ResultPath.empty())
- {
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
- }
std::string JsonQuery;
if (m_QueryPath.empty())
{
@@ -2991,25 +1231,13 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
}
}
+ std::string DummyBuildId;
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
-
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- std::string DummyBuildId;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- DummyBuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ false,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireBucket = false}, Opts);
+ // Read-only listing: no local zen folder needed; downloads stay in memory.
+ StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery);
ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
@@ -3021,28 +1249,20 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
}
else
{
- if (ToLower(m_ResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ builds_impl::WriteResultObject(m_ResultPath, Response);
}
}
-BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("list-blocks", "List blocks for a build")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "list-blocks", "List blocks for a build")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
Opts.add_option("",
"",
@@ -3058,50 +1278,34 @@ BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent)
void
BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
- MakeSafeAbsolutePathInPlace(m_ResultPath);
-
if (!m_ResultPath.empty())
{
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ LogBanner();
}
+ cxxopts::Options& Opts = SubOptions();
+
+ MakeSafeAbsolutePathInPlace(m_ResultPath);
+
if (m_MaxCount == 0)
{
throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_MaxCount), Opts.help());
}
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ // Read-only listing: no local zen folder needed; downloads stay in memory.
+ StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_MaxCount);
ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response);
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size());
}
@@ -3116,35 +1320,25 @@ BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
}
else
{
- if (ToLower(m_ResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ builds_impl::WriteResultObject(m_ResultPath, Response);
}
}
-BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("upload", "Upload a folder to build storage")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "upload", "Upload a folder to build storage")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
- Parent.AddExcludeFolderOption(Opts);
- Parent.AddExcludeExtensionsOption(Opts);
- Parent.AddChunkingCacheOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
+ Config.AddExcludeFolderOption(Opts);
+ Config.AddExcludeExtensionsOption(Opts);
+ Config.AddChunkingCacheOptions(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
Opts.add_option("",
"",
@@ -3192,7 +1386,7 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent)
cxxopts::value(m_UploadToZenCache),
"<uploadtozencache>");
- Parent.AddMultipartOptions(Opts);
+ Config.AddMultipartOptions(Opts);
Opts.add_option("",
"",
@@ -3217,50 +1411,31 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent)
void
BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
- ZenState InstanceState;
+ ParsePath(m_Path, Opts);
- m_Parent.ParsePath(m_Path, Opts);
+ builds_impl::ZenState InstanceState;
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
- MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath);
-
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ ResolveZenFolderPath({});
+ StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ EnsureZenFolderExists();
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
if (m_BuildPartName.empty() && m_ManifestPath.empty())
{
m_BuildPartName = m_Path.filename().string();
}
- const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : m_Parent.ParseBuildId(m_BuildId, Opts);
+ const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(m_BuildId, Opts);
if (m_BuildId.empty())
{
m_BuildId = BuildId.ToString();
@@ -3269,28 +1444,31 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
Oid BuildPartId;
if (!m_BuildPartId.empty())
{
- BuildPartId = m_Parent.ParseBuildPartId(m_BuildPartId, Opts);
+ BuildPartId = ParseBuildPartId(m_BuildPartId, Opts);
}
- CbObject MetaData = m_Parent.ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts);
+ CbObject MetaData = ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts);
- const std::filesystem::path TempDir = ZenTempFolderPath(m_Parent.GetZenFolderPath());
+ const std::filesystem::path TempDir = ZenTempFolderPath(GetZenFolderPath());
std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
- m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+ ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty()
+ std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty()
? CreateNullChunkingCache()
- : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+ : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u);
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
std::vector<std::pair<Oid, std::string>> UploadedParts =
- UploadFolder(*Output,
+ UploadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
BuildId,
BuildPartId,
m_BuildPartName,
@@ -3302,89 +1480,89 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
UploadFolderOptions{.TempDir = TempDir,
.FindBlockMaxCount = m_FindBlockMaxCount,
.BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.CreateBuild = m_CreateBuild,
.IgnoreExistingBlocks = m_Clean,
.UploadToZenCache = m_UploadToZenCache,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
.ExcludeFolders = ExcludeFolders,
.ExcludeExtensions = ExcludeExtensions});
- if (!AbortFlag)
+ if (!AbortFlag())
{
if (m_PostUploadVerify)
{
for (const auto& Part : UploadedParts)
{
- ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second);
+ ValidateBuildPart(ConsoleLog(),
+ *Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ m_Config.Verbose,
+ Workers,
+ *Storage.BuildStorage,
+ ZenTempFolderPath(GetZenFolderPath()) / "validate",
+ BuildId,
+ Part.first,
+ Part.second);
}
}
}
- if (true)
+ if (!m_Config.Quiet)
{
- if (!IsQuiet)
- {
- ZEN_CONSOLE(
- "{}:\n"
- "Read: {}\n"
- "Write: {}\n"
- "Requests: {}\n"
- "Avg Request Time: {}\n"
- "Avg I/O Time: {}",
- Storage.BuildStorageHost.Name,
- NiceBytes(StorageStats.TotalBytesRead.load()),
- NiceBytes(StorageStats.TotalBytesWritten.load()),
- StorageStats.TotalRequestCount.load(),
- StorageStats.TotalExecutionTimeUs.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0,
- StorageStats.TotalRequestCount.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0);
- }
+ const BuildStorageResolveResult::Host& Host = Storage.BuildStorageHost;
+ const BuildStorageBase::Statistics& Stats = StorageStats;
+ ZEN_CONSOLE(
+ "{}:\n"
+ "Read: {}\n"
+ "Write: {}\n"
+ "Requests: {}\n"
+ "Avg Request Time: {}\n"
+ "Avg I/O Time: {}",
+ Host.Name,
+ NiceBytes(Stats.TotalBytesRead.load()),
+ NiceBytes(Stats.TotalBytesWritten.load()),
+ Stats.TotalRequestCount.load(),
+ Stats.TotalExecutionTimeUs.load() > 0
+ ? NiceTimeSpanMs(Stats.TotalExecutionTimeUs.load() / 1000 / Stats.TotalRequestCount.load())
+ : 0,
+ Stats.TotalRequestCount.load() > 0 ? NiceTimeSpanMs(Stats.TotalRequestTimeUs.load() / 1000 / Stats.TotalRequestCount.load())
+ : 0);
}
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Upload aborted");
}
}
-BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("download", "Download a build to a local folder")
-, m_Parent(Parent)
-{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddWildcardOptions(Opts);
- Parent.AddAppendNewContentOptions(Opts);
- Parent.AddExcludeFolderOption(Opts);
+//////////////////////////////////////////////////////////////////////////
- Opts.add_option("cache",
- "",
- "cache-prime-only",
- "Only download blobs missing in cache and upload to cache",
- cxxopts::value(m_PrimeCacheOnly),
- "<cacheprimeonly>");
+BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "download", "Download a build to a local folder")
+{
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddWildcardOptions(Opts);
+ Config.AddAppendNewContentOptions(Opts);
+ Config.AddExcludeFolderOption(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- Opts.add_option("",
- "",
- "build-part-id",
- "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
- cxxopts::value(m_BuildPartIds),
- "<id>");
+ Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>");
Opts.add_option("",
"",
"build-part-name",
- "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
- "all parts will be downloaded",
+ "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, "
+ "the part named 'default' is selected. Use '*' (alone) to select all parts.",
cxxopts::value(m_BuildPartNames),
"<name>");
Opts.add_option("",
@@ -3405,9 +1583,9 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent)
"Upload data downloaded from remote host to zen cache",
cxxopts::value(m_UploadToZenCache),
"<uploadtozencache>");
- Parent.AddMultipartOptions(Opts);
+ Config.AddMultipartOptions(Opts);
- Parent.AddPartialBlockRequestOptions(Opts);
+ Config.AddPartialBlockRequestOptions(Opts);
Opts.add_option(
"",
@@ -3437,139 +1615,107 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent)
void
BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
- ZenState InstanceState;
-
- m_Parent.ParsePath(m_Path, Opts);
+ ParsePath(m_Path, Opts);
std::vector<std::string> IncludeWildcards;
std::vector<std::string> ExcludeWildcards;
- m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards);
+ ParseFileFilters(IncludeWildcards, ExcludeWildcards);
- m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName);
+ builds_impl::ZenState InstanceState;
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ m_PrimeCacheOnly,
- Auth,
- Opts);
-
- const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
-
- if (m_PostDownloadVerify && m_PrimeCacheOnly)
- {
- throw OptionParseException("'--cache-prime-only' conflicts with '--verify'", Opts.help());
- }
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ // Preserve zen folder on exit - <zen>/current_state.cbo tracks what has been downloaded into
+ // this folder and is consulted by later download operations against the same target.
+ ResolveZenFolderPath(m_Path);
+ StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved);
- if (m_Clean && m_PrimeCacheOnly)
- {
- ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled");
- }
-
- if (m_Force && m_PrimeCacheOnly)
- {
- ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled");
- }
-
- if (m_Parent.m_AllowPartialBlockRequests != "false" && m_PrimeCacheOnly)
- {
- ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled");
- }
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
- std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts);
- std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts);
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts);
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts);
+ NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help());
- EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(m_PrimeCacheOnly, Opts);
+ EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts);
- if (m_Parent.m_AppendNewContent && m_Clean)
+ if (m_Config.AppendNewContent && m_Clean)
{
throw OptionParseException("'--append' conflicts with '--clean'", Opts.help());
}
std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
- m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+ ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
DownloadFolder(
- *Output,
+ ConsoleLog(),
+ *Progress,
Workers,
Storage,
- StorageCacheStats,
+ AbortFlag(),
+ PauseFlag(),
+ CacheStats,
BuildId,
BuildPartIds,
BuildPartNames,
m_DownloadSpecPath,
m_Path,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
- .ZenFolderPath = m_Parent.GetZenFolderPath(),
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ DownloadOptions{.SystemRootDir = m_Config.SystemRootDir,
+ .ZenFolderPath = GetZenFolderPath(),
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = m_Clean,
.PostDownloadVerify = m_PostDownloadVerify,
- .PrimeCacheOnly = m_PrimeCacheOnly,
.EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force,
.EnableTargetFolderScavenging = !m_Force,
.AllowFileClone = m_AllowFileClone,
.IncludeWildcards = IncludeWildcards,
.ExcludeWildcards = ExcludeWildcards,
- .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Parent.m_BoostWorkerMemory),
+ .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory),
.PopulateCache = m_UploadToZenCache,
- .AppendNewContent = m_Parent.m_AppendNewContent,
+ .AppendNewContent = m_Config.AppendNewContent,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles,
.ExcludeFolders = ExcludeFolders});
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Download aborted");
}
}
-BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "List files in a build"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsLsSubCmd::BuildsLsSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "ls", "List files in a build")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddWildcardOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddWildcardOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- Opts.add_option("",
- "",
- "build-part-id",
- "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
- cxxopts::value(m_BuildPartIds),
- "<id>");
+ Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>");
Opts.add_option("",
"",
"build-part-name",
- "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
- "all parts will be downloaded",
+ "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, "
+ "the part named 'default' is selected. Use '*' (alone) to select all parts.",
cxxopts::value(m_BuildPartNames),
"<name>");
@@ -3594,43 +1740,31 @@ BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "Lis
void
BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
if (!m_ResultPath.empty())
{
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ LogBanner();
}
- ZenState InstanceState;
+ cxxopts::Options& Opts = SubOptions();
+
+ builds_impl::ZenState InstanceState;
std::vector<std::string> IncludeWildcards;
std::vector<std::string> ExcludeWildcards;
- m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards);
-
- m_Parent.ResolveZenFolderPath(m_Parent.m_StoragePath); // ls uses storage path context
+ ParseFileFilters(IncludeWildcards, ExcludeWildcards);
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ // Read-only listing: no local zen folder needed; downloads stay in memory.
+ StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
- std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts);
- std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts);
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts);
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts);
+ NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help());
std::unique_ptr<CbObjectWriter> StructuredOutput;
if (!m_ResultPath.empty())
@@ -3639,38 +1773,30 @@ BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
StructuredOutput = std::make_unique<CbObjectWriter>();
}
- ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get());
+ ListBuild(m_Config.Quiet, Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get());
if (StructuredOutput)
{
CbObject Response = StructuredOutput->Save();
- if (ToLower(m_ResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ builds_impl::WriteResultObject(m_ResultPath, Response);
}
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("List build aborted");
}
}
-BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff", "Diff two local folders"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "diff", "Diff two local folders")
{
- auto& Opts = SubOptions();
- Parent.AddOutputOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddExcludeFolderOption(Opts);
- Parent.AddExcludeExtensionsOption(Opts);
- Parent.AddChunkingCacheOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddOutputOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddExcludeFolderOption(Opts);
+ Config.AddExcludeExtensionsOption(Opts);
+ Config.AddChunkingCacheOptions(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
Opts.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>");
Opts.add_option("",
@@ -3686,38 +1812,28 @@ BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff"
void
BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
- m_Parent.ParsePath(m_Path, Opts);
+ ParsePath(m_Path, Opts);
if (m_DiffPath.empty())
{
throw OptionParseException("'--compare-path' is required", Opts.help());
}
MakeSafeAbsolutePathInPlace(m_DiffPath);
- MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath);
-
std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
- m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+ ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
StandardChunkingControllerSettings ChunkingSettings;
std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings);
- std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty()
+ std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty()
? CreateNullChunkingCache()
- : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+ : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u);
if (m_OnlyChunked)
{
@@ -3729,24 +1845,37 @@ BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
ChunkingSettings.SplitAndCompressExtensions.end());
}
- DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions);
- if (AbortFlag)
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
+
+ DiffFolders(*Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ Workers,
+ m_Path,
+ m_DiffPath,
+ *ChunkController,
+ *ChunkCache,
+ ExcludeFolders,
+ ExcludeExtensions);
+ if (AbortFlag())
{
throw std::runtime_error("Diff folders aborted");
}
}
-BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("fetch-blob", "Fetch and validate a specific blob")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "fetch-blob", "Fetch and validate a specific blob")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
Opts.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>");
Opts.parse_positional({"build-id", "blob-hash"});
@@ -3756,80 +1885,55 @@ BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent)
void
BuildsFetchBlobSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
-
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- IoHash BlobHash = m_Parent.ParseBlobHash(m_BlobHash, Opts);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ ResolveZenFolderPath({});
+ StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ EnsureZenFolderExists();
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
- const Oid BuildId = Oid::FromHexString(m_BuildId);
+ IoHash BlobHash = ParseBlobHash(m_BlobHash, Opts);
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
uint64_t CompressedSize;
uint64_t DecompressedSize;
- ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
- if (AbortFlag)
+ ValidateBlob(AbortFlag(), *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
+ if (AbortFlag())
{
throw std::runtime_error("Fetch blob aborted");
}
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize);
}
}
-BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("prime-cache", "Prime the zen cache with build data")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "prime-cache", "Prime the zen cache with build data")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
- Opts.add_option("",
- "",
- "build-part-id",
- "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded",
- cxxopts::value(m_BuildPartIds),
- "<id>");
+ Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>");
Opts.add_option("",
"",
"build-part-name",
- "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given "
- "all parts will be downloaded",
+ "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, "
+ "the part named 'default' is selected. Use '*' (alone) to select all parts.",
cxxopts::value(m_BuildPartNames),
"<name>");
Opts.add_option("",
@@ -3845,47 +1949,31 @@ BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent)
void
BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ ResolveZenFolderPath({});
+ StorageInstance Storage =
+ CreateBuildStorage(GetZenFolderPath(), {.BoostCacheBackgroundWorkers = true}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ EnsureZenFolderExists();
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ true,
- Auth,
- Opts);
-
- const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
-
- std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts);
- std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts);
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts);
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts);
+ NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help());
std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
- CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);
+ CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, m_Config.Quiet);
std::vector<std::pair<Oid, std::string>> AllBuildParts =
ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
@@ -3897,100 +1985,101 @@ BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
AllBuildPartIds.push_back(BuildPart.first);
}
- ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache");
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
+ Progress->SetLogOperationName("Prime Cache");
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
-
- BuildsOperationPrimeCache PrimeOp(*Output,
+ BuildsOperationPrimeCache PrimeOp(ConsoleLog(),
+ *Progress,
Storage,
- AbortFlag,
- PauseFlag,
+ AbortFlag(),
+ PauseFlag(),
Workers.GetNetworkPool(),
BuildId,
AllBuildPartIds,
- BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet,
- .IsVerbose = IsVerbose,
- .ZenFolderPath = m_Parent.GetZenFolderPath(),
+ BuildsOperationPrimeCache::Options{.IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .ZenFolderPath = GetZenFolderPath(),
.LargeAttachmentSize = PreferredMultipartChunkSize * 4u,
.PreferredMultipartChunkSize = PreferredMultipartChunkSize,
.ForceUpload = m_Force},
- StorageCacheStats);
+ CacheStats);
PrimeOp.Execute();
- if (!IsQuiet)
+ if (!m_Config.Quiet && Storage.CacheStorage)
{
- if (Storage.CacheStorage)
- {
- ZEN_CONSOLE("Uploaded {} ({}) blobs to {}",
- StorageCacheStats.PutBlobCount.load(),
- NiceBytes(StorageCacheStats.PutBlobByteCount),
- Storage.CacheHost.Name);
- }
+ ZEN_CONSOLE("Uploaded {} ({}) blobs to {}",
+ CacheStats.PutBlobCount.load(),
+ NiceBytes(CacheStats.PutBlobByteCount),
+ Storage.CacheHost.Name);
}
}
-BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("pause", "Pause a running zen process"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+namespace {
+ void AddProcessIdOption(cxxopts::Options& Opts, int& ZenProcessId)
+ {
+ Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(ZenProcessId), "<pid>");
+ Opts.parse_positional({"process-id"});
+ Opts.positional_help("process-id");
+ }
+} // namespace
+
+BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "pause", "Pause a running zen process")
{
- auto& Opts = SubOptions();
- Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
- Opts.parse_positional({"process-id"});
- Opts.positional_help("process-id");
+ AddProcessIdOption(SubOptions(), m_ZenProcessId);
}
void
BuildsPauseSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- using namespace builds_impl;
- m_Parent.ParseZenProcessId(m_ZenProcessId);
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Pause.store(true);
+ ParseZenProcessId(m_ZenProcessId);
+ builds_impl::ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Pause.store(1);
}
-BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("resume", "Resume a paused zen process"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "resume", "Resume a paused zen process")
{
- auto& Opts = SubOptions();
- Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
- Opts.parse_positional({"process-id"});
- Opts.positional_help("process-id");
+ AddProcessIdOption(SubOptions(), m_ZenProcessId);
}
void
BuildsResumeSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- using namespace builds_impl;
- m_Parent.ParseZenProcessId(m_ZenProcessId);
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Pause.store(false);
+ ParseZenProcessId(m_ZenProcessId);
+ builds_impl::ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Pause.store(0);
}
-BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("abort", "Abort a running zen process"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "abort", "Abort a running zen process")
{
- auto& Opts = SubOptions();
- Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
- Opts.parse_positional({"process-id"});
- Opts.positional_help("process-id");
+ AddProcessIdOption(SubOptions(), m_ZenProcessId);
}
void
BuildsAbortSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- using namespace builds_impl;
- m_Parent.ParseZenProcessId(m_ZenProcessId);
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Abort.store(true);
+ ParseZenProcessId(m_ZenProcessId);
+ builds_impl::ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Abort.store(1);
}
-BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("validate-part", "Validate a build part")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "validate-part", "Validate a build part")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
Opts.add_option("",
"",
@@ -4011,42 +2100,24 @@ BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent)
void
BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- ZenState InstanceState;
+ cxxopts::Options& Opts = SubOptions();
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ ResolveZenFolderPath({});
+ StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ EnsureZenFolderExists();
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
+ builds_impl::ZenState InstanceState;
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+ Oid BuildId = ParseBuildId(m_BuildId, Opts);
if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
{
@@ -4055,33 +2126,46 @@ BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
Opts.help());
}
- const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : m_Parent.ParseBuildPartId(m_BuildPartId, Opts);
+ const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(m_BuildPartId, Opts);
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
- ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(ConsoleLog(),
+ *Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ m_Config.Verbose,
+ Workers,
+ *Storage.BuildStorage,
+ ZenTempFolderPath(GetZenFolderPath()) / "validate",
+ BuildId,
+ BuildPartId,
+ m_BuildPartName);
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Validate build part failed");
}
}
-BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test", "Run an upload/download test cycle"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsTestSubCmd::BuildsTestSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "test", "Run an upload/download test cycle")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddWorkerOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddWorkerOptions(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
- Parent.AddMultipartOptions(Opts);
- Parent.AddPartialBlockRequestOptions(Opts);
- Parent.AddWildcardOptions(Opts);
- Parent.AddAppendNewContentOptions(Opts);
- Parent.AddChunkingCacheOptions(Opts);
+ Config.AddMultipartOptions(Opts);
+ Config.AddPartialBlockRequestOptions(Opts);
+ Config.AddWildcardOptions(Opts);
+ Config.AddAppendNewContentOptions(Opts);
+ Config.AddChunkingCacheOptions(Opts);
Opts.add_option("",
"",
"enable-scavenge",
@@ -4101,38 +2185,35 @@ BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test"
void
BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
- m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(m_Parent.m_SystemRootDir);
- CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
- auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); });
+ m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_TestSystemRootDir);
+ CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
+ auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); });
- m_Parent.ParsePath(m_Path, Opts);
+ ParsePath(m_Path, Opts);
- if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty())
+ const bool CreatedTempStorage = m_Config.OverrideHost.empty() && m_Config.StoragePath.empty();
+ if (CreatedTempStorage)
{
- m_Parent.m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.m_StoragePath);
- CreateDirectories(m_Parent.m_StoragePath);
- m_Parent.m_StoragePath = m_Parent.m_StoragePath.generic_string();
+ m_TestStoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), m_TestStoragePath);
+ CreateDirectories(m_TestStoragePath);
+ m_TestStoragePath = m_TestStoragePath.generic_string();
}
- auto StorageGuard = MakeGuard([this]() {
- if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty())
+ auto StorageGuard = MakeGuard([this, CreatedTempStorage]() {
+ if (CreatedTempStorage)
{
- DeleteDirectories(m_Parent.m_StoragePath);
+ DeleteDirectories(m_TestStoragePath);
}
});
- EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts);
+ EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts);
BuildStorageBase::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
@@ -4143,37 +2224,35 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2");
const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3");
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3);
- auto DownloadGuard = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() {
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3);
+ auto DownloadGuard = MakeGuard([this, &Workers, DownloadPath, DownloadPath2, DownloadPath3]() {
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3);
});
- m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName);
- MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath);
-
std::unique_ptr<AuthMgr> Auth;
std::string TestBuildId;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- TestBuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- m_BuildId = Oid::NewOid().ToString();
- m_BuildPartId = Oid::NewOid().ToString();
- m_CreateBuild = true;
-
- const Oid BuildId = Oid::FromHexString(m_BuildId);
- const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
+ const ResolvedStorage Resolved = ParseStorageOptions(TestBuildId, {}, Opts, m_TestSystemRootDir, m_TestStoragePath);
+ // Place scratch next to the folder under test so upload does not self-include, and so the
+ // scratch area shares the volume of the source data.
+ {
+ const std::u8string PathStr = m_Path.generic_u8string();
+ const IoHash PathHash = IoHash::HashBuffer(PathStr.data(), PathStr.length());
+ SetZenFolderPath(m_Path.parent_path() / fmt::format("zen_{}", PathHash));
+ }
+ StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, StorageCacheStats, Auth, Resolved);
+ EnsureZenFolderExists();
+ auto ZenGuard = MakeGuard([this]() { CleanZenFolder(); });
+
+ const Oid BuildId = Oid::NewOid();
+ const Oid BuildPartId = Oid::NewOid();
+ m_BuildId = BuildId.ToString();
+ m_BuildPartId = BuildPartId.ToString();
+ m_CreateBuild = true;
auto MakeMetaData = [](const Oid& BuildId) -> CbObject {
CbObjectWriter BuildMetaDataWriter;
@@ -4196,18 +2275,21 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView());
}
- const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path);
+ const std::filesystem::path UploadTempDir = ZenTempFolderPath(GetZenFolderPath());
std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty()
+ std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty()
? CreateNullChunkingCache()
- : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+ : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u);
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
- UploadFolder(*Output,
+ UploadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
BuildId,
BuildPartId,
m_BuildPartName,
@@ -4219,12 +2301,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
UploadFolderOptions{.TempDir = UploadTempDir,
.FindBlockMaxCount = m_FindBlockMaxCount,
.BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.CreateBuild = true,
.IgnoreExistingBlocks = false,
- .UploadToZenCache = m_UploadToZenCache});
+ .UploadToZenCache = m_UploadToZenCache,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose});
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Upload build)");
}
@@ -4232,9 +2316,12 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path);
- UploadFolder(*Output,
+ UploadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
Oid::NewOid(),
Oid::NewOid(),
m_BuildPartName,
@@ -4246,52 +2333,70 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
UploadFolderOptions{.TempDir = UploadTempDir,
.FindBlockMaxCount = m_FindBlockMaxCount,
.BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.CreateBuild = true,
.IgnoreExistingBlocks = false,
- .UploadToZenCache = m_UploadToZenCache});
+ .UploadToZenCache = m_UploadToZenCache,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose});
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Upload again, chunking is cached)");
}
}
- ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(ConsoleLog(),
+ *Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ m_Config.Verbose,
+ Workers,
+ *Storage.BuildStorage,
+ ZenTempFolderPath(GetZenFolderPath()) / "validate",
+ BuildId,
+ BuildPartId,
+ m_BuildPartName);
- if (!m_Parent.m_IncludeWildcard.empty() || !m_Parent.m_ExcludeWildcard.empty())
+ if (!m_Config.IncludeWildcard.empty() || !m_Config.ExcludeWildcard.empty())
{
- auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); });
+ auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); });
ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
std::vector<std::string> IncludeWildcards;
std::vector<std::string> ExcludeWildcards;
- m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards);
+ ParseFileFilters(IncludeWildcards, ExcludeWildcards);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = true,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = false,
.AllowFileClone = m_AllowFileClone,
.IncludeWildcards = IncludeWildcards,
.ExcludeWildcards = ExcludeWildcards,
- .AppendNewContent = false});
- if (AbortFlag)
+ .AppendNewContent = false,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download build)");
}
@@ -4301,66 +2406,79 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
BuildPartId,
m_BuildPartName,
DownloadPath);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = true,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
.AllowFileClone = m_AllowFileClone,
.IncludeWildcards = ExcludeWildcards,
.ExcludeWildcards = IncludeWildcards,
- .AppendNewContent = true});
- if (AbortFlag)
+ .AppendNewContent = true,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download build)");
}
ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
.AllowFileClone = m_AllowFileClone,
.IncludeWildcards = {},
.ExcludeWildcards = {},
- .AppendNewContent = false});
- if (AbortFlag)
+ .AppendNewContent = false,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download build)");
}
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
@@ -4368,25 +2486,30 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = true,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = false,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download build)");
}
ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
@@ -4394,17 +2517,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Re-download identical target)");
}
@@ -4436,7 +2561,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
return true;
};
- ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+ ParallelWork Work(AbortFlag(), PauseFlag(), WorkerThreadPool::EMode::EnableBacklog);
uint32_t Randomizer = 0;
auto FileSizeIt = DownloadContent.FileSizes.begin();
@@ -4454,8 +2579,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
Work.ScheduleWork(
Workers.GetIOWorkerPool(),
- [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
- if (!AbortFlag)
+ [this, SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
+ if (!AbortFlag())
{
bool WasReadOnly = SetFileReadOnly(FilePath, false);
{
@@ -4483,7 +2608,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
case 1:
{
(void)SetFileReadOnly(FilePath, false);
- (void)RemoveFile(FilePath);
+ std::error_code Ec;
+ const bool Removed = RemoveFile(FilePath, Ec);
+ if (!Removed || Ec)
+ {
+ throw zen::runtime_error("ScrambleDir: failed to delete '{}'. Reason: '{}'",
+ FilePath,
+ Ec ? Ec.message() : "file no longer present");
+ }
}
break;
default:
@@ -4496,15 +2628,18 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
ZEN_UNUSED(IsAborted, IsPaused);
ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork);
});
- ZEN_ASSERT(!AbortFlag.load());
+ ZEN_ASSERT(!AbortFlag().load());
ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
};
ScrambleDir(DownloadPath);
ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
@@ -4512,17 +2647,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Re-download scrambled target)");
}
@@ -4539,9 +2676,12 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView());
}
- UploadFolder(*Output,
+ UploadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
BuildId2,
BuildPartId2,
m_BuildPartName,
@@ -4553,22 +2693,38 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
UploadFolderOptions{.TempDir = UploadTempDir,
.FindBlockMaxCount = m_FindBlockMaxCount,
.BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.CreateBuild = true,
.IgnoreExistingBlocks = false,
- .UploadToZenCache = m_UploadToZenCache});
+ .UploadToZenCache = m_UploadToZenCache,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose});
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Upload scrambled)");
}
- ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(ConsoleLog(),
+ *Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ m_Config.Verbose,
+ Workers,
+ *Storage.BuildStorage,
+ ZenTempFolderPath(GetZenFolderPath()) / "validate",
+ BuildId,
+ BuildPartId,
+ m_BuildPartName);
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
@@ -4576,133 +2732,156 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download original)");
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId2,
{BuildPartId2},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download scrambled)");
}
ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId2,
{BuildPartId2},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Re-download scrambled)");
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath2,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath2 / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download original)");
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3);
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath3,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath3 / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download original)");
}
}
-BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("multi-test-download", "Download multiple builds sequentially as a test")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "multi-test-download", "Download multiple builds sequentially as a test")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddWorkerOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddWorkerOptions(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
Opts.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>");
Opts.add_option("",
@@ -4724,42 +2903,37 @@ BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Pare
void
BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
- m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(m_Parent.m_SystemRootDir);
- CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
- auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); });
+ m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_TestSystemRootDir);
+ CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
+ auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); });
- m_Parent.ParsePath(m_Path, Opts);
-
- m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName);
-
- EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts);
+ ParsePath(m_Path, Opts);
+ std::string DummyBuildId;
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {}, Opts, m_TestSystemRootDir);
+ // Place scratch next to the download target so the scratch area shares its volume.
+ {
+ const std::u8string PathStr = m_Path.generic_u8string();
+ const IoHash PathHash = IoHash::HashBuffer(PathStr.data(), PathStr.length());
+ SetZenFolderPath(m_Path.parent_path() / fmt::format("zen_{}", PathHash));
+ }
+ StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ EnsureZenFolderExists();
+ auto ZenGuard = MakeGuard([this]() { CleanZenFolder(); });
- std::unique_ptr<AuthMgr> Auth;
- std::string DummyBuildId;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- DummyBuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
+ EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts);
+
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
Stopwatch Timer;
for (const std::string& BuildIdString : m_BuildIds)
@@ -4769,35 +2943,40 @@ BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), Opts.help());
}
- DownloadFolder(*Output,
+ DownloadFolder(ConsoleLog(),
+ *Progress,
Workers,
Storage,
- StorageCacheStats,
+ AbortFlag(),
+ PauseFlag(),
+ CacheStats,
BuildId,
/*BuildPartIds,*/ {},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
m_Path,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
- .ZenFolderPath = m_Parent.GetZenFolderPath(),
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
+ .ZenFolderPath = GetZenFolderPath(),
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = BuildIdString == m_BuildIds.front(),
.PostDownloadVerify = true,
- .PrimeCacheOnly = false,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = false,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Multitest aborted");
}
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("\n");
}
}
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
}
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 7ef71e176..338318757 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -2,65 +2,227 @@
#pragma once
-#include "../authutils.h"
-#include "../zen.h"
-
-#include <zenhttp/auth/authmgr.h>
-#include <zenhttp/httpclientauth.h>
+#include <zencore/scopeguard.h>
#include <zenremotestore/builds/buildstoragecache.h>
#include <zenremotestore/builds/buildstorageutil.h>
#include <zenremotestore/partialblockrequestmode.h>
-#include <filesystem>
+#include <zenremotestore/transferthreadworkers.h>
+
+#include "authutils.h"
+#include "consoleprogress.h"
+
+#include <optional>
namespace zen {
-class BuildsCommand;
+class ProgressBase;
+class AuthMgr;
+
+struct CreateBuildStorageOptions
+{
+ bool RequireNamespace = true;
+ bool RequireBucket = true;
+ bool BoostCacheBackgroundWorkers = false;
+};
+
+//////////////////////////////////////////////////////////////////////////
-class BuildsListNamespacesSubCmd : public ZenSubCmdBase
+struct BuildsConfiguration
+{
+ std::filesystem::path SystemRootDir;
+ bool UseSparseFiles = true;
+
+ bool PlainProgress = false;
+ bool LogProgress = false;
+ bool Verbose = false;
+ bool Quiet = false;
+ ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty;
+ bool BoostWorkerCount = false;
+ bool BoostWorkerMemory = false;
+ bool BoostWorkers = false;
+
+ std::string OverrideHost;
+ std::string Host;
+ std::string Url;
+ bool AssumeHttp2 = false;
+ bool VerboseHttp = false;
+ bool AllowRedirect = false;
+ std::string Namespace;
+ std::string Bucket;
+
+ std::filesystem::path StoragePath;
+ bool WriteMetadataAsJson = false;
+
+ std::string ZenCacheHost;
+
+ AuthCommandLineOptions AuthOptions;
+
+ std::string IncludeWildcard;
+ std::string ExcludeWildcard;
+ std::string ExcludeFolders;
+ std::string ExcludeExtensions;
+
+ std::filesystem::path ChunkingCachePath;
+
+ bool AllowMultiparts = true;
+ std::string AllowPartialBlockRequests = "true";
+
+ bool AppendNewContent = false;
+
+ std::filesystem::path ZenFolderPath;
+
+ void AddSystemOptions(cxxopts::Options& Ops);
+ void AddCloudOptions(cxxopts::Options& Ops);
+ void AddFileOptions(cxxopts::Options& Ops);
+ void AddCacheOptions(cxxopts::Options& Ops);
+ void AddOutputOptions(cxxopts::Options& Ops);
+ void AddWorkerOptions(cxxopts::Options& Ops);
+ void AddZenFolderOptions(cxxopts::Options& Ops);
+ void AddChunkingCacheOptions(cxxopts::Options& Ops);
+ void AddWildcardOptions(cxxopts::Options& Ops);
+ void AddExcludeFolderOption(cxxopts::Options& Ops);
+ void AddExcludeExtensionsOption(cxxopts::Options& Ops);
+ void AddMultipartOptions(cxxopts::Options& Ops);
+ void AddPartialBlockRequestOptions(cxxopts::Options& Ops);
+ void AddAppendNewContentOptions(cxxopts::Options& Ops);
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsSubCmdBase : public ZenSubCmdBase
{
public:
- explicit BuildsListNamespacesSubCmd(BuildsCommand& Parent);
+ BuildsSubCmdBase(BuildsConfiguration& Config, std::string_view Name, std::string_view Description)
+ : ZenSubCmdBase(Name, Description)
+ , m_Config(Config)
+ {
+ }
+
+protected:
+ const BuildsConfiguration& m_Config;
+ std::filesystem::path m_ResolvedZenFolderPath;
+ // Set by EnsureZenFolderExists: true if this command newly created the zen folder
+ // (so CleanZenFolder may remove the whole folder), false if it already existed
+ // (in which case only the temp subfolder is wiped, to preserve any pre-existing state).
+ bool m_CreatedZenFolder = false;
+
+ struct ResolvedStorage
+ {
+ std::filesystem::path SystemRootDir;
+ std::string Host;
+ std::string Namespace;
+ std::string Bucket;
+ std::filesystem::path StoragePath;
+ };
+
+ void LogBanner();
+ void LogWorkersInfo(const TransferThreadWorkers& Workers);
+
+ // SystemRootDirOverride / StoragePathOverride: empty = fall back to m_Config.
+ ResolvedStorage ParseStorageOptions(std::string& BuildId,
+ const CreateBuildStorageOptions& Options,
+ cxxopts::Options& SubOpts,
+ const std::filesystem::path& SystemRootDirOverride = {},
+ const std::filesystem::path& StoragePathOverride = {});
+
+ // Builds the storage instance using the supplied Resolved values. ZenFolder is the final zen folder
+ // path (caller resolves; pass GetZenFolderPath() after ResolveZenFolderPath). If ZenFolder is empty
+ // the storage runs without a temp directory and keeps all downloads in memory.
+ // Caller owns Stats/Auth lifetime. Stats must outlive the returned StorageInstance.
+ StorageInstance CreateBuildStorage(const std::filesystem::path& ZenFolder,
+ const CreateBuildStorageOptions& Options,
+ cxxopts::Options& SubOpts,
+ BuildStorageBase::Statistics& OutStorageStats,
+ BuildStorageCache::Statistics& OutCacheStats,
+ std::unique_ptr<AuthMgr>& OutAuth,
+ const ResolvedStorage& Resolved);
+ Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts);
+ Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts);
+ std::vector<Oid> ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts);
+ std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts);
+ CbObject ParseBuildMetadata(bool CreateBuild,
+ std::filesystem::path& BuildMetadataPath,
+ const std::string& BuildMetadata,
+ cxxopts::Options& SubOpts);
+ void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts);
+ IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts);
+ EPartialBlockRequestMode ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts);
+ void ParseZenProcessId(int& ZenProcessId);
+ void ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards);
+ void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions);
+
+ // Resolves the zen folder using this chain: --zen-folder-path; else LocalPath/ZenFolderName
+ // if LocalPath is non-empty; else cwd/ZenFolderName. Read-only commands that do not need a
+ // zen folder should skip this and pass an empty path to CreateBuildStorage.
+ void ResolveZenFolderPath(const std::filesystem::path& LocalPath);
+ // Assigns the zen folder to --zen-folder-path if given, else to Fallback directly (no
+ // ZenFolderName appended). For commands that want a specific scratch location rather than
+ // the standard LocalPath/.zen chain.
+ void SetZenFolderPath(const std::filesystem::path& Fallback);
+ const std::filesystem::path& GetZenFolderPath() const { return m_ResolvedZenFolderPath; }
+ // Creates the resolved zen folder and records whether it had to be created.
+ // Use together with CleanZenFolder via MakeGuard.
+ void EnsureZenFolderExists();
+
+ std::atomic<bool>& AbortFlag() const;
+ std::atomic<bool>& PauseFlag() const;
+ std::unique_ptr<ProgressBase> CreateProgress() const;
+
+ // Wipes temp state produced by the command. If EnsureZenFolderExists created the zen folder
+ // it is removed outright; otherwise only the temp subfolder is wiped so pre-existing state
+ // (e.g. current_state.cbo from a prior download in the same folder) is preserved.
+ void CleanZenFolder();
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsListNamespacesSubCmd : public BuildsSubCmdBase
+{
+public:
+ explicit BuildsListNamespacesSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
bool m_Recursive = false;
std::filesystem::path m_ResultPath;
};
-class BuildsListSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsListSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsListSubCmd(BuildsCommand& Parent);
+ explicit BuildsListSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
std::filesystem::path m_QueryPath;
std::filesystem::path m_ResultPath;
};
-class BuildsListBlocksSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsListBlocksSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsListBlocksSubCmd(BuildsCommand& Parent);
+ explicit BuildsListBlocksSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
std::string m_BuildId;
std::filesystem::path m_ResultPath;
uint32_t m_MaxCount = 16;
};
-class BuildsUploadSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsUploadSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsUploadSubCmd(BuildsCommand& Parent);
+ explicit BuildsUploadSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
std::filesystem::path m_Path;
std::string m_BuildId;
std::string m_BuildPartId;
@@ -76,14 +238,15 @@ private:
bool m_UploadToZenCache = true;
};
-class BuildsDownloadSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsDownloadSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsDownloadSubCmd(BuildsCommand& Parent);
+ explicit BuildsDownloadSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
std::filesystem::path m_Path;
std::string m_BuildId;
std::vector<std::string> m_BuildPartIds;
@@ -94,117 +257,129 @@ private:
bool m_EnableScavenging = true;
std::filesystem::path m_DownloadSpecPath;
bool m_UploadToZenCache = true;
- bool m_PrimeCacheOnly = false;
bool m_AllowFileClone = true;
};
-class BuildsLsSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsLsSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsLsSubCmd(BuildsCommand& Parent);
+ explicit BuildsLsSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
std::string m_BuildId;
std::vector<std::string> m_BuildPartIds;
std::vector<std::string> m_BuildPartNames;
std::filesystem::path m_ResultPath;
};
-class BuildsDiffSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsDiffSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsDiffSubCmd(BuildsCommand& Parent);
+ explicit BuildsDiffSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
std::filesystem::path m_Path;
std::filesystem::path m_DiffPath;
bool m_OnlyChunked = false;
};
-class BuildsFetchBlobSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsFetchBlobSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsFetchBlobSubCmd(BuildsCommand& Parent);
+ explicit BuildsFetchBlobSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
- std::string m_BuildId;
- std::string m_BlobHash;
+ std::string m_BuildId;
+ std::string m_BlobHash;
};
-class BuildsPrimeCacheSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsPrimeCacheSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsPrimeCacheSubCmd(BuildsCommand& Parent);
+ explicit BuildsPrimeCacheSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
std::string m_BuildId;
std::vector<std::string> m_BuildPartIds;
std::vector<std::string> m_BuildPartNames;
bool m_Force = false;
};
-class BuildsPauseSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsPauseSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsPauseSubCmd(BuildsCommand& Parent);
+ explicit BuildsPauseSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
- int m_ZenProcessId = -1;
+ int m_ZenProcessId = -1;
};
-class BuildsResumeSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsResumeSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsResumeSubCmd(BuildsCommand& Parent);
+ explicit BuildsResumeSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
- int m_ZenProcessId = -1;
+ int m_ZenProcessId = -1;
};
-class BuildsAbortSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsAbortSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsAbortSubCmd(BuildsCommand& Parent);
+ explicit BuildsAbortSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
- int m_ZenProcessId = -1;
+ int m_ZenProcessId = -1;
};
-class BuildsValidatePartSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsValidatePartSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsValidatePartSubCmd(BuildsCommand& Parent);
+ explicit BuildsValidatePartSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
- std::string m_BuildId;
- std::string m_BuildPartId;
- std::string m_BuildPartName;
+ std::string m_BuildId;
+ std::string m_BuildPartId;
+ std::string m_BuildPartName;
};
-class BuildsTestSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsTestSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsTestSubCmd(BuildsCommand& Parent);
+ explicit BuildsTestSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
+ // Fixture-only overrides of SystemRootDir/StoragePath; passed to ParseStorageOptions explicitly.
+ std::filesystem::path m_TestSystemRootDir;
+ std::filesystem::path m_TestStoragePath;
+
std::filesystem::path m_Path;
std::string m_BuildPartName;
std::string m_BuildId;
@@ -217,21 +392,27 @@ private:
bool m_AllowFileClone = true;
};
-class BuildsMultiTestDownloadSubCmd : public ZenSubCmdBase
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsMultiTestDownloadSubCmd : public BuildsSubCmdBase
{
public:
- explicit BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent);
+ explicit BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config);
void Run(const ZenCliOptions& GlobalOptions) override;
private:
- BuildsCommand& m_Parent;
+ // Fixture-only override of SystemRootDir; passed to CreateStorage explicitly.
+ std::filesystem::path m_TestSystemRootDir;
+
std::filesystem::path m_Path;
std::vector<std::string> m_BuildIds;
bool m_EnableScavenging = true;
bool m_AllowFileClone = true;
};
-class BuildsCommand : public CacheStoreCmdWithSubCommands
+//////////////////////////////////////////////////////////////////////////
+
+class BuildsCommand : public ZenCmdWithSubCommands
{
public:
static constexpr char Name[] = "builds";
@@ -243,99 +424,14 @@ public:
cxxopts::Options& Options() override { return m_Options; }
- // Option-adding helpers (called by subcommand constructors)
- void AddSystemOptions(cxxopts::Options& Ops);
- void AddCloudOptions(cxxopts::Options& Ops);
- void AddFileOptions(cxxopts::Options& Ops);
- void AddCacheOptions(cxxopts::Options& Ops);
- void AddOutputOptions(cxxopts::Options& Ops);
- void AddWorkerOptions(cxxopts::Options& Ops);
- void AddZenFolderOptions(cxxopts::Options& Ops);
- void AddChunkingCacheOptions(cxxopts::Options& Ops);
- void AddWildcardOptions(cxxopts::Options& Ops);
- void AddExcludeFolderOption(cxxopts::Options& Ops);
- void AddExcludeExtensionsOption(cxxopts::Options& Ops);
- void AddMultipartOptions(cxxopts::Options& Ops);
- void AddPartialBlockRequestOptions(cxxopts::Options& Ops);
- void AddAppendNewContentOptions(cxxopts::Options& Ops);
-
- // Shared parsing/factory methods used by subcommand Run() implementations
- void ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts);
- StorageInstance CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
- BuildStorageCache::Statistics& StorageCacheStats,
- const std::filesystem::path& TempPath,
- std::string& BuildId,
- bool RequireNamespace,
- bool RequireBucket,
- bool BoostCacheBackgroundWorkerPool,
- std::unique_ptr<AuthMgr>& Auth,
- cxxopts::Options& SubOpts);
- Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts);
- Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts);
- std::vector<Oid> ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts);
- std::vector<std::string> ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts);
- CbObject ParseBuildMetadata(bool CreateBuild,
- std::filesystem::path& BuildMetadataPath,
- const std::string& BuildMetadata,
- cxxopts::Options& SubOpts);
- void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts);
- IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts);
- EPartialBlockRequestMode ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts);
- void ParseZenProcessId(int& ZenProcessId);
- void ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards);
- void ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions);
-
- void ResolveZenFolderPath(const std::filesystem::path& DefaultPath);
- const std::filesystem::path& GetZenFolderPath() const { return m_ZenFolderPath; }
-
- cxxopts::Options m_Options{Name, Description};
- std::string m_SubCommand;
-
- // Shared state populated by AddXxxOptions helpers (bound via cxxopts::value references)
- std::filesystem::path m_SystemRootDir;
- bool m_UseSparseFiles = true;
-
- bool m_PlainProgress = false;
- bool m_LogProgress = false;
- bool m_Verbose = false;
- bool m_Quiet = false;
- bool m_BoostWorkerCount = false;
- bool m_BoostWorkerMemory = false;
- bool m_BoostWorkers = false;
-
- // cloud builds
- std::string m_OverrideHost;
- std::string m_Host;
- std::string m_Url;
- bool m_AssumeHttp2 = false;
- bool m_VerboseHttp = false;
- bool m_AllowRedirect = false;
- std::string m_Namespace;
- std::string m_Bucket;
-
- std::filesystem::path m_StoragePath;
- bool m_WriteMetadataAsJson = false;
-
- std::string m_ZenCacheHost;
-
- AuthCommandLineOptions m_AuthOptions;
-
- std::string m_IncludeWildcard;
- std::string m_ExcludeWildcard;
- std::string m_ExcludeFolders;
- std::string m_ExcludeExtensions;
-
- std::filesystem::path m_ChunkingCachePath;
-
- bool m_AllowMultiparts = true;
- std::string m_AllowPartialBlockRequests = "true";
-
- bool m_AppendNewContent = false;
+ BuildsConfiguration& GetConfiguration() { return m_Configuration; }
+ const BuildsConfiguration& GetConfiguration() const { return m_Configuration; }
private:
- std::filesystem::path m_ZenFolderPath;
+ cxxopts::Options m_Options{Name, Description};
+ std::string m_SubCommand;
+ BuildsConfiguration m_Configuration;
-protected:
BuildsListNamespacesSubCmd m_ListNamespacesSubCmd;
BuildsListSubCmd m_ListSubCmd;
BuildsListBlocksSubCmd m_ListBlocksSubCmd;
@@ -352,6 +448,11 @@ protected:
BuildsTestSubCmd m_TestSubCmd;
BuildsMultiTestDownloadSubCmd m_MultiTestDownloadSubCmd;
+ std::optional<ScopedSignalHandler> m_SigIntGuard;
+#if ZEN_PLATFORM_WINDOWS
+ std::optional<ScopedSignalHandler> m_SigBreakGuard;
+#endif
+
bool OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) override;
};
diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp
index a8c15f119..f93a5318c 100644
--- a/src/zen/cmds/cache_cmd.cpp
+++ b/src/zen/cmds/cache_cmd.cpp
@@ -2,25 +2,40 @@
#include "cache_cmd.h"
+#include "zenserviceclient.h"
+
#include <zencore/compactbinarybuilder.h>
#include <zencore/compress.h>
#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
+#include <zencore/process.h>
#include <zencore/scopeguard.h>
+#include <zencore/session.h>
+#include <zencore/stream.h>
#include <zencore/thread.h>
+#include <zencore/timer.h>
#include <zencore/workthreadpool.h>
+#include <zenhttp/formatters.h>
#include <zenhttp/httpclient.h>
#include <zenhttp/httpcommon.h>
#include <zenhttp/packageformat.h>
#include <zenstore/cache/cachepolicy.h>
+#include <zenutil/rpcrecording.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
#include <memory>
#include <random>
namespace zen {
+using namespace std::literals;
+
namespace {
IoBuffer CreateRandomBlob(uint64_t Size)
{
@@ -56,37 +71,112 @@ namespace {
}
} // namespace
-DropCommand::DropCommand()
+////////////////////////////////////////////////////////////////////////////////
+// CacheCommand
+
+CacheCommand::CacheCommand()
{
m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
- m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>");
- m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>");
- m_Options.parse_positional({"namespace", "bucket"});
+
+ AddSubCommand(m_DetailsSubCmd);
+ AddSubCommand(m_DropSubCmd);
+ AddSubCommand(m_GenSubCmd);
+ AddSubCommand(m_GetSubCmd);
+ AddSubCommand(m_InfoSubCmd);
+ AddSubCommand(m_RecordSubCmd);
+ AddSubCommand(m_ReplaySubCmd);
+ AddSubCommand(m_StatsSubCmd);
}
-DropCommand::~DropCommand() = default;
+CacheCommand::~CacheCommand() = default;
-void
-DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+////////////////////////////////////////////////////////////////////////////////
+// CacheSubCmdBase
+
+CacheSubCmdBase::CacheSubCmdBase(std::string_view Name, std::string_view Description) : ZenSubCmdBase(Name, Description)
{
- ZEN_UNUSED(GlobalOptions);
+ m_SubOptions.add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+}
- if (!ParseOptions(argc, argv))
+void
+CacheSubCmdBase::ResolveHost()
+{
+ m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName);
+ if (m_HostName.empty())
{
- return;
+ throw OptionParseException("Unable to resolve server specification", m_SubOptions.help());
}
+}
- m_HostName = ResolveTargetHostSpec(m_HostName);
+////////////////////////////////////////////////////////////////////////////////
+// Legacy shim dispatcher
- if (m_HostName.empty())
+namespace cache_legacy_shim {
+ static void Dispatch(std::span<const std::string_view> Injected, const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
+ // cxxopts treats argv as writable char** in the style of C main(argv).
+ // Stage the injected tokens in writable std::string storage so we never
+ // hand out pointers to string literals.
+ std::vector<std::string> Storage;
+ Storage.reserve(Injected.size());
+ for (std::string_view Token : Injected)
+ {
+ Storage.emplace_back(Token);
+ }
+
+ std::vector<char*> NewArgv;
+ NewArgv.reserve(static_cast<size_t>(argc) + Storage.size());
+ NewArgv.push_back(argv[0]);
+ for (std::string& Token : Storage)
+ {
+ NewArgv.push_back(Token.data());
+ }
+ for (int i = 1; i < argc; ++i)
+ {
+ NewArgv.push_back(argv[i]);
+ }
+
+ CacheCommand Impl;
+ Impl.Run(GlobalOptions, static_cast<int>(NewArgv.size()), NewArgv.data());
}
+ void RunAs(const char* SubCommandName, const ZenCliOptions& GlobalOptions, int argc, char** argv)
+ {
+ const std::string_view Tokens[] = {std::string_view(SubCommandName)};
+ Dispatch(Tokens, GlobalOptions, argc, argv);
+ }
+} // namespace cache_legacy_shim
+
+// RpcStopRecordingCommand is unique among legacy shims in that it needs to
+// inject two tokens ("record" and "stop") rather than a single subcommand name.
+void
+RpcStopRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+ using namespace std::literals;
+ const std::string_view Tokens[] = {"record"sv, "stop"sv};
+ cache_legacy_shim::Dispatch(Tokens, GlobalOptions, argc, argv);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// CacheDropSubCmd
+
+CacheDropSubCmd::CacheDropSubCmd() : CacheSubCmdBase("drop", "Drop cache namespace or bucket")
+{
+ m_SubOptions.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>");
+ m_SubOptions.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>");
+ m_SubOptions.parse_positional({"namespace", "bucket"});
+}
+
+void
+CacheDropSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ ResolveHost();
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "drop"});
+ HttpClient& Http = Service.Http();
+
if (m_NamespaceName.empty())
{
- throw OptionParseException("'--namespace' is required", m_Options.help());
+ throw OptionParseException("'--namespace' is required", m_SubOptions.help());
}
std::string Url;
@@ -94,18 +184,16 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_BucketName.empty())
{
- DropDescription = fmt::format("cache namespace '{}' from '{}'", m_NamespaceName, m_HostName);
+ DropDescription = fmt::format("cache namespace '{}' from '{}'", m_NamespaceName, Service.HostSpec());
Url = fmt::format("/z$/{}", m_NamespaceName);
}
else
{
- DropDescription = fmt::format("cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName);
+ DropDescription = fmt::format("cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, Service.HostSpec());
Url = fmt::format("/z$/{}/{}", m_NamespaceName, m_BucketName);
}
ZEN_CONSOLE("Dropping {}", DropDescription);
-
- HttpClient Http = CreateHttpClient(m_HostName);
if (HttpClient::Response Response = Http.Delete(Url))
{
ZEN_CONSOLE("{}", Response.ToText());
@@ -116,54 +204,43 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
}
-CacheInfoCommand::CacheInfoCommand()
+////////////////////////////////////////////////////////////////////////////////
+// CacheInfoSubCmd
+
+CacheInfoSubCmd::CacheInfoSubCmd() : CacheSubCmdBase("info", "Info on cache, namespace or bucket")
{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
- m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>");
- m_Options.add_option("",
- "",
- "bucketsizes",
- "Comma delimited list of bucket names to get size info from, * to get info on all buckets",
- cxxopts::value(m_SizeInfoBucketNames),
- "<bucketnames>");
- m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>");
- m_Options.add_option("", "", "bucketsize", "Show detailed bucket size info", cxxopts::value(m_BucketSizeInfo), "<bucketsize>");
-
- m_Options.parse_positional({"namespace", "bucket"});
+ m_SubOptions.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), "<namespacename>");
+ m_SubOptions.add_option("",
+ "",
+ "bucketsizes",
+ "Comma delimited list of bucket names to get size info from, * to get info on all buckets",
+ cxxopts::value(m_SizeInfoBucketNames),
+ "<bucketnames>");
+ m_SubOptions.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), "<bucketname>");
+ m_SubOptions.add_option("", "", "bucketsize", "Show detailed bucket size info", cxxopts::value(m_BucketSizeInfo), "<bucketsize>");
+ m_SubOptions.add_option("", "y", "yaml", "Output as YAML instead of JSON", cxxopts::value(m_YAML), "<yaml>");
+ m_SubOptions.parse_positional({"namespace", "bucket"});
}
-CacheInfoCommand::~CacheInfoCommand() = default;
-
void
-CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+CacheInfoSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- ZEN_UNUSED(GlobalOptions);
-
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
+ ResolveHost();
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "info"});
+ HttpClient& Http = Service.Http();
std::string Url;
- if (m_HostName.empty())
+ if (m_NamespaceName.empty())
{
if (!m_SizeInfoBucketNames.empty())
{
- throw OptionParseException("'--bucketsizes' requires '--namespace'", m_Options.help());
+ throw OptionParseException("'--bucketsizes' requires '--namespace'", m_SubOptions.help());
}
if (m_BucketSizeInfo)
{
- throw OptionParseException("'--bucketsize' requires '--namespace' and '--bucket'", m_Options.help());
+ throw OptionParseException("'--bucketsize' requires '--namespace' and '--bucket'", m_SubOptions.help());
}
- ZEN_CONSOLE("Info on cache from '{}'", m_HostName);
+ ZEN_CONSOLE("Info on cache from '{}'", Service.HostSpec());
Url = "/z$";
}
else if (m_BucketName.empty())
@@ -171,18 +248,18 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_BucketSizeInfo)
{
throw OptionParseException(fmt::format("'--bucketsize' requires '--namespace' and '--bucket' ('{}')", m_BucketName),
- m_Options.help());
+ m_SubOptions.help());
}
- ZEN_CONSOLE("Info on cache namespace '{}' from '{}'", m_NamespaceName, m_HostName);
+ ZEN_CONSOLE("Info on cache namespace '{}' from '{}'", m_NamespaceName, Service.HostSpec());
Url = fmt::format("/z$/{}", m_NamespaceName);
}
else
{
if (!m_SizeInfoBucketNames.empty())
{
- throw OptionParseException("'--bucketsizes' conflicts with '--bucket'", m_Options.help());
+ throw OptionParseException("'--bucketsizes' conflicts with '--bucket'", m_SubOptions.help());
}
- ZEN_CONSOLE("Info on cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName);
+ ZEN_CONSOLE("Info on cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, Service.HostSpec());
Url = fmt::format("/z$/{}/{}", m_NamespaceName, m_BucketName);
}
@@ -196,8 +273,9 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
Parameters.Entries.insert({"bucketsize", "true"});
}
- HttpClient Http = CreateHttpClient(m_HostName);
- if (HttpClient::Response Response = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON), Parameters))
+ const ZenContentType AcceptType = m_YAML ? ZenContentType::kYAML : ZenContentType::kJSON;
+
+ if (HttpClient::Response Response = Http.Get(Url, HttpClient::Accept(AcceptType), Parameters))
{
ZEN_CONSOLE("{}", Response.ToText());
}
@@ -207,76 +285,62 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
}
-CacheStatsCommand::CacheStatsCommand()
+////////////////////////////////////////////////////////////////////////////////
+// CacheStatsSubCmd
+
+CacheStatsSubCmd::CacheStatsSubCmd() : CacheSubCmdBase("stats", "Stats on cache")
{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+ m_SubOptions.add_option("", "y", "yaml", "Output as YAML instead of JSON", cxxopts::value(m_YAML), "<yaml>");
}
-CacheStatsCommand::~CacheStatsCommand() = default;
-
void
-CacheStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+CacheStatsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- ZEN_UNUSED(GlobalOptions);
+ ResolveHost();
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "stats"});
+ HttpClient& Http = Service.Http();
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
+ const ZenContentType AcceptType = m_YAML ? ZenContentType::kYAML : ZenContentType::kJSON;
- HttpClient Http = CreateHttpClient(m_HostName);
- if (HttpClient::Response Response = Http.Get("/stats/z$", HttpClient::Accept(ZenContentType::kJSON)))
+ if (HttpClient::Response Response = Http.Get("/stats/z$", HttpClient::Accept(AcceptType)))
{
ZEN_CONSOLE("{}", Response.ToText());
}
else
{
- Response.ThrowError("Info failed");
+ Response.ThrowError("Stats failed");
}
}
-CacheDetailsCommand::CacheDetailsCommand()
+////////////////////////////////////////////////////////////////////////////////
+// CacheDetailsSubCmd
+
+CacheDetailsSubCmd::CacheDetailsSubCmd() : CacheSubCmdBase("details", "Details on cache")
{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
- m_Options.add_option("", "c", "csv", "Info on csv format", cxxopts::value(m_CSV), "<csv>");
- m_Options.add_option("", "d", "details", "Get detailed information about records", cxxopts::value(m_Details), "<details>");
- m_Options.add_option("",
- "a",
- "attachmentdetails",
- "Get detailed information about attachments",
- cxxopts::value(m_AttachmentDetails),
- "<attachmentdetails>");
- m_Options.add_option("", "n", "namespace", "Namespace name to get info for", cxxopts::value(m_Namespace), "<namespace>");
- m_Options.add_option("", "b", "bucket", "Filter on bucket name", cxxopts::value(m_Bucket), "<bucket>");
- m_Options.add_option("", "v", "valuekey", "Filter on value key hash string", cxxopts::value(m_ValueKey), "<valuekey>");
+ m_SubOptions.add_option("", "c", "csv", "Output as CSV instead of JSON", cxxopts::value(m_CSV), "<csv>");
+ m_SubOptions.add_option("", "y", "yaml", "Output as YAML instead of JSON", cxxopts::value(m_YAML), "<yaml>");
+ m_SubOptions.add_option("", "d", "details", "Get detailed information about records", cxxopts::value(m_Details), "<details>");
+ m_SubOptions.add_option("",
+ "a",
+ "attachmentdetails",
+ "Get detailed information about attachments",
+ cxxopts::value(m_AttachmentDetails),
+ "<attachmentdetails>");
+ m_SubOptions.add_option("", "n", "namespace", "Namespace name to get info for", cxxopts::value(m_Namespace), "<namespace>");
+ m_SubOptions.add_option("", "b", "bucket", "Filter on bucket name", cxxopts::value(m_Bucket), "<bucket>");
+ m_SubOptions.add_option("", "v", "valuekey", "Filter on value key hash string", cxxopts::value(m_ValueKey), "<valuekey>");
}
-CacheDetailsCommand::~CacheDetailsCommand() = default;
-
void
-CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+CacheDetailsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- ZEN_UNUSED(GlobalOptions);
+ ResolveHost();
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "details"});
+ HttpClient& Http = Service.Http();
- if (!ParseOptions(argc, argv))
+ if (m_CSV && m_YAML)
{
- return;
- }
-
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
+ throw OptionParseException("'--csv' conflicts with '--yaml'", m_SubOptions.help());
}
HttpClient::KeyValueMap Parameters;
@@ -296,7 +360,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
}
else
{
- Headers = HttpClient::Accept(ZenContentType::kJSON);
+ Headers = HttpClient::Accept(m_YAML ? ZenContentType::kYAML : ZenContentType::kJSON);
}
std::string Url;
@@ -304,11 +368,11 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
{
if (m_Namespace.empty())
{
- throw OptionParseException("'--namespace' is required", m_Options.help());
+ throw OptionParseException("'--namespace' is required", m_SubOptions.help());
}
if (m_Bucket.empty())
{
- throw OptionParseException("'--bucket' is required", m_Options.help());
+ throw OptionParseException("'--bucket' is required", m_SubOptions.help());
}
Url = fmt::format("/z$/details$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey);
}
@@ -316,7 +380,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
{
if (m_Namespace.empty())
{
- throw OptionParseException("'--namespace' is required", m_Options.help());
+ throw OptionParseException("'--namespace' is required", m_SubOptions.help());
}
Url = fmt::format("/z$/details$/{}/{}", m_Namespace, m_Bucket);
}
@@ -329,61 +393,49 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
Url = "/z$/details$";
}
- HttpClient Http = CreateHttpClient(m_HostName);
if (HttpClient::Response Response = Http.Get(Url, Headers, Parameters))
{
ZEN_CONSOLE("{}", Response.ToText());
}
else
{
- Response.ThrowError("Info failed");
+ Response.ThrowError("Details failed");
}
}
-CacheGenerateCommand::CacheGenerateCommand()
+////////////////////////////////////////////////////////////////////////////////
+// CacheGenSubCmd
+
+CacheGenSubCmd::CacheGenSubCmd() : CacheSubCmdBase("gen", "Generates cache values into a bucket")
{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
- m_Options
+ m_SubOptions
.add_option("", "n", "namespace", "Namespace to generate cache values/records for", cxxopts::value(m_Namespace), "<namespace>");
- m_Options.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), "<bucket>");
- m_Options.add_option("", "", "count", "Number of cache values/records to generate", cxxopts::value(m_Count), "<count>");
- m_Options.add_option("", "", "min-size", "Minimum size of cache value/attachments", cxxopts::value(m_MinSize), "<min>");
- m_Options.add_option("", "", "max-size", "Maximum size of cache value/attachments", cxxopts::value(m_MaxSize), "<max>");
- m_Options.add_option("",
- "",
- "min-attachments",
- "Minimum number of attachments when creating record based values",
- cxxopts::value(m_MinAttachmentCount),
- "<minattachments>");
- m_Options.add_option("",
- "",
- "max-attachments",
- "Minimum number of attachments when creating record based values, 0 to only create cache values",
- cxxopts::value(m_MaxAttachmentCount),
- "<maxattachments>");
- m_Options.parse_positional({"namespace", "bucket", "count"});
- m_Options.positional_help("namespace bucket count");
+ m_SubOptions.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), "<bucket>");
+ m_SubOptions.add_option("", "", "count", "Number of cache values/records to generate", cxxopts::value(m_Count), "<count>");
+ m_SubOptions.add_option("", "", "min-size", "Minimum size of cache value/attachments", cxxopts::value(m_MinSize), "<min>");
+ m_SubOptions.add_option("", "", "max-size", "Maximum size of cache value/attachments", cxxopts::value(m_MaxSize), "<max>");
+ m_SubOptions.add_option("",
+ "",
+ "min-attachments",
+ "Minimum number of attachments when creating record based values",
+ cxxopts::value(m_MinAttachmentCount),
+ "<minattachments>");
+ m_SubOptions.add_option("",
+ "",
+ "max-attachments",
+ "Minimum number of attachments when creating record based values, 0 to only create cache values",
+ cxxopts::value(m_MaxAttachmentCount),
+ "<maxattachments>");
+ m_SubOptions.parse_positional({"namespace", "bucket", "count"});
+ m_SubOptions.positional_help("namespace bucket count");
}
-CacheGenerateCommand::~CacheGenerateCommand() = default;
-
void
-CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+CacheGenSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- ZEN_UNUSED(GlobalOptions);
-
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
+ ResolveHost();
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "gen"});
+ HttpClient& Http = Service.Http();
if (m_MaxSize == 0 && m_MinSize == 0)
{
@@ -400,6 +452,16 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
}
}
+ // The size-range expansion below requires MinSize >= 1 (it uses
+ // `MinSize - 1` as a uniform distribution upper bound, which would
+ // underflow on an unsigned zero) and MaxSize >= MinSize.
+ if (m_MinSize == 0 || m_MaxSize < m_MinSize)
+ {
+ throw OptionParseException(
+ fmt::format("'--min-size' ({}) must be >= 1 and '--max-size' ({}) must be >= '--min-size'", m_MinSize, m_MaxSize),
+ m_SubOptions.help());
+ }
+
std::vector<std::uniform_int_distribution<uint64_t>> Variations;
std::vector<size_t> SizeRanges;
SizeRanges.push_back(m_MinSize);
@@ -431,8 +493,6 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
std::uniform_int_distribution<uint64_t> KeyDistribution;
- HttpClient Http = CreateHttpClient(m_HostName);
-
auto GeneratePutCacheValueRequest(
[this, &KeyDistribution, &Generator](std::span<std::uint64_t> BatchSizes, uint64_t RequestIndex) -> CbPackage {
CbPackage Package;
@@ -583,68 +643,56 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
}
}
-CacheGetCommand::CacheGetCommand()
+////////////////////////////////////////////////////////////////////////////////
+// CacheGetSubCmd
+
+CacheGetSubCmd::CacheGetSubCmd() : CacheSubCmdBase("get", "Get cache values/records or attachments from a bucket")
{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
- m_Options
- .add_option("", "n", "namespace", "Namespace to generate cache values/records for", cxxopts::value(m_Namespace), "<namespace>");
- m_Options.add_option("", "b", "bucket", "Bucket name to generate cache values/records for", cxxopts::value(m_Bucket), "<bucket>");
- m_Options.add_option("", "v", "valuekey", "Cache entry iohash id", cxxopts::value(m_ValueKey), "<valuekey>");
- m_Options.add_option("",
- "a",
- "attachmenthash",
- "For a cache entry record, get a particular attachment based on the 'RawHash'",
- cxxopts::value(m_AttachmentHash),
- "<attachmenthash>");
- m_Options.add_option("", "o", "output-path", "File path for output data", cxxopts::value(m_OutputPath), "<path>");
- m_Options.add_option("", "t", "text", "Ouput content of cache entry record as text", cxxopts::value(m_AsText), "<text>");
- m_Options
+ m_SubOptions.add_option("", "n", "namespace", "Namespace of the cache entry", cxxopts::value(m_Namespace), "<namespace>");
+ m_SubOptions.add_option("", "b", "bucket", "Bucket of the cache entry", cxxopts::value(m_Bucket), "<bucket>");
+ m_SubOptions.add_option("", "v", "valuekey", "Cache entry iohash id", cxxopts::value(m_ValueKey), "<valuekey>");
+ m_SubOptions.add_option("",
+ "a",
+ "attachmenthash",
+ "For a cache entry record, get a particular attachment based on the 'RawHash'",
+ cxxopts::value(m_AttachmentHash),
+ "<attachmenthash>");
+ m_SubOptions.add_option("", "o", "output-path", "File path for output data", cxxopts::value(m_OutputPath), "<path>");
+ m_SubOptions.add_option("", "t", "text", "Output content of cache entry record as text", cxxopts::value(m_AsText), "<text>");
+ m_SubOptions
.add_option("", "d", "decompress", "Decompress data when applicable. Default = true", cxxopts::value(m_Decompress), "<decompress>");
- m_Options.parse_positional({"namespace", "bucket", "valuekey", "attachmenthash"});
- m_Options.positional_help("namespace bucket valuekey attachmenthash");
+ m_SubOptions.parse_positional({"namespace", "bucket", "valuekey", "attachmenthash"});
+ m_SubOptions.positional_help("namespace bucket valuekey attachmenthash");
}
-CacheGetCommand::~CacheGetCommand() = default;
-
void
-CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+CacheGetSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- ZEN_UNUSED(GlobalOptions);
-
using namespace std::literals;
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
+ ResolveHost();
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "get"});
+ HttpClient& Http = Service.Http();
if (m_Namespace.empty())
{
- throw OptionParseException("'--namespace' is required", m_Options.help());
+ throw OptionParseException("'--namespace' is required", m_SubOptions.help());
}
if (m_Bucket.empty())
{
- throw OptionParseException("'--bucket' is required", m_Options.help());
+ throw OptionParseException("'--bucket' is required", m_SubOptions.help());
}
if (m_ValueKey.empty())
{
- throw OptionParseException("'--valuekey' is required", m_Options.help());
+ throw OptionParseException("'--valuekey' is required", m_SubOptions.help());
}
IoHash ValueId;
if (!IoHash::TryParse(m_ValueKey, ValueId))
{
- throw OptionParseException(fmt::format("'--value-key' ('{}') is malformed", m_ValueKey), m_Options.help());
+ throw OptionParseException(fmt::format("'--valuekey' ('{}') is malformed", m_ValueKey), m_SubOptions.help());
}
IoHash AttachmentHash;
@@ -652,11 +700,14 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (!IoHash::TryParse(m_AttachmentHash, AttachmentHash))
{
- throw OptionParseException(fmt::format("'--attachmenthash' ('{}') is malformed", m_AttachmentHash), m_Options.help());
+ throw OptionParseException(fmt::format("'--attachmenthash' ('{}') is malformed", m_AttachmentHash), m_SubOptions.help());
}
}
- HttpClient Http = CreateHttpClient(m_HostName);
+ if (m_OutputPath.empty() && !m_AsText)
+ {
+ throw OptionParseException("'--output-path' is required (or pass '--as-text' to print to stdout)", m_SubOptions.help());
+ }
if (!m_OutputPath.empty())
{
@@ -669,18 +720,19 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
CreateDirectories(m_OutputPath.parent_path());
}
}
- if (m_OutputPath.empty())
- {
- m_OutputPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash);
- }
- std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey);
+ std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, ValueId);
if (AttachmentHash != IoHash::Zero)
{
Url = fmt::format("{}/{}", Url, AttachmentHash);
}
if (HttpClient::Response Result = Http.Download(Url, std::filesystem::temp_directory_path()); Result)
{
+ // `Http.Download` parks the payload in the system temp dir and returns
+ // a buffer that already has delete-on-close set, so every exit path
+ // (exception, fallback WriteFile, `--as-text` console print) reaps it.
+ // A successful MoveToFile below clears the flag so the payload's
+ // handle-close doesn't delete the caller's output afterwards.
auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer {
IoHash RawHash;
uint64_t RawSize;
@@ -707,7 +759,17 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- if (!MoveToFile(m_OutputPath, ChunkData))
+ if (std::error_code MoveEc = MoveToFile(m_OutputPath, ChunkData); MoveEc)
+ {
+ // The file was renamed into place; clearing DeleteOnClose prevents
+ // the move'd-out file at m_OutputPath from being deleted when the
+ // payload's handle closes. When m_Decompress is false ChunkData
+ // shares a core with ResponsePayload so either clear suffices;
+ // when decompressed ChunkData is in-memory and MoveToFile would
+ // have failed, so we don't reach this branch.
+ Result.ResponsePayload.SetDeleteOnClose(false);
+ }
+ else
{
WriteFile(m_OutputPath, ChunkData);
}
@@ -720,4 +782,428 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
}
+////////////////////////////////////////////////////////////////////////////////
+// CacheRecordSubCmd
+
+CacheRecordSubCmd::CacheRecordSubCmd()
+: CacheSubCmdBase("record", "Start recording cache rpc requests ('cache record <path>'), or stop ('cache record stop')")
+{
+ m_SubOptions.add_option("", "p", "path", "Recording file path, or 'stop' to stop recording", cxxopts::value(m_Path), "<path>");
+ m_SubOptions.parse_positional("path");
+ m_SubOptions.positional_help("<path>|stop");
+}
+
+void
+CacheRecordSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ ResolveHost();
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "record"});
+ HttpClient& Http = Service.Http();
+
+ if (m_Path == "stop")
+ {
+ if (HttpClient::Response Response = Http.Post("/z$/exec$/stop-recording"sv))
+ {
+ ZEN_CONSOLE("{}", Response.ToText());
+ }
+ else
+ {
+ Response.ThrowError("Failed to stop recording");
+ }
+ return;
+ }
+
+ if (m_Path.empty())
+ {
+ throw OptionParseException("recording path is required (use '<path>' to start, 'stop' to stop)", m_SubOptions.help());
+ }
+
+ if (HttpClient::Response Response =
+ Http.Post("/z$/exec$/start-recording"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap({{"path", m_Path}})))
+ {
+ ZEN_CONSOLE("{}", Response.ToText());
+ }
+ else
+ {
+ Response.ThrowError("Failed to start recording");
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// CacheReplaySubCmd
+
+CacheReplaySubCmd::CacheReplaySubCmd() : CacheSubCmdBase("replay", "Replays a previously recorded session of rpc requests")
+{
+ m_SubOptions.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), "<path>");
+ m_SubOptions.add_option("", "", "dry", "Do a dry run", cxxopts::value(m_DryRun), "<enable>");
+ m_SubOptions.add_option("",
+ "w",
+ "numthreads",
+ "Number of worker threads per process",
+ cxxopts::value(m_ThreadCount)->default_value(fmt::format("{}", GetHardwareConcurrency())),
+ "<count>");
+ m_SubOptions.add_option("", "", "onhost", "Replay on host, bypassing http/network layer", cxxopts::value(m_OnHost), "<onhost>");
+ m_SubOptions.add_option("",
+ "",
+ "showmethodstats",
+ "Show statistics of which RPC methods are used",
+ cxxopts::value(m_ShowMethodStats),
+ "<showmethodstats>");
+ m_SubOptions.add_option("",
+ "",
+ "offset",
+ "Offset into request recording to start replay",
+ cxxopts::value(m_Offset)->default_value("0"),
+ "<offset>");
+ m_SubOptions.add_option("",
+ "",
+ "stride",
+ "Stride for request recording when replaying requests",
+ cxxopts::value(m_Stride)->default_value("1"),
+ "<stride>");
+ m_SubOptions.add_option("", "", "numproc", "Number of worker processes", cxxopts::value(m_ProcessCount)->default_value("1"), "<count>");
+ m_SubOptions.add_option("",
+ "",
+ "forceallowlocalrefs",
+ "Force enable local refs in requests",
+ cxxopts::value(m_ForceAllowLocalRefs),
+ "<enable>");
+ m_SubOptions
+ .add_option("", "", "disablelocalrefs", "Force disable local refs in requests", cxxopts::value(m_DisableLocalRefs), "<enable>");
+ m_SubOptions.add_option("",
+ "",
+ "forceallowlocalhandlerefs",
+ "Force enable local refs as handles in requests",
+ cxxopts::value(m_ForceAllowLocalHandleRef),
+ "<enable>");
+ m_SubOptions.add_option("",
+ "",
+ "disablelocalhandlerefs",
+ "Force disable local refs as handles in requests",
+ cxxopts::value(m_DisableLocalHandleRefs),
+ "<enable>");
+ m_SubOptions.add_option("",
+ "",
+ "forceallowpartiallocalrefs",
+ "Force enable local refs for all sizes",
+ cxxopts::value(m_ForceAllowPartialLocalRefs),
+ "<enable>");
+ m_SubOptions.add_option("",
+ "",
+ "disablepartiallocalrefs",
+ "Force disable local refs for all sizes",
+ cxxopts::value(m_DisablePartialLocalRefs),
+ "<enable>");
+ m_SubOptions.parse_positional("path");
+}
+
+void
+CacheReplaySubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
+{
+ if (m_RecordingPath.empty())
+ {
+ throw OptionParseException("'--path' is required", m_SubOptions.help());
+ }
+
+ if (!IsDir(m_RecordingPath))
+ {
+ throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath));
+ }
+
+ if (m_Stride == 0)
+ {
+ throw OptionParseException("'--stride' must be >= 1", m_SubOptions.help());
+ }
+
+ m_ThreadCount = Max(m_ThreadCount, 1);
+
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "replay"});
+ m_HostName = Service.HostSpec();
+
+ ZEN_CONSOLE("Replay '{}' (start offset {}, stride {}) to '{}', {} threads",
+ m_RecordingPath,
+ m_Offset,
+ m_Stride,
+ m_HostName,
+ m_ThreadCount);
+
+ Stopwatch TotalTimer;
+
+ if (m_OnHost)
+ {
+ HttpClient& Http = Service.Http();
+ if (HttpClient::Response Response =
+ Http.Post("/z$/exec$/replay-recording"sv,
+ HttpClient::KeyValueMap{},
+ HttpClient::KeyValueMap({{"path", m_RecordingPath}, {"thread-count", fmt::format("{}", m_ThreadCount)}})))
+ {
+ ZEN_CONSOLE("{}", Response.ToText());
+
+ return;
+ }
+ else
+ {
+ Response.ThrowError("Failed to start replay");
+ }
+ }
+
+ std::unique_ptr<cache::IRpcRequestReplayer> Replayer = cache::MakeDiskRequestReplayer(m_RecordingPath, true);
+ uint64_t EntryCount = Replayer->GetRequestCount();
+
+ if (m_Offset >= EntryCount)
+ {
+ ZEN_CONSOLE("Offset {} is at or past the end of the recording ({} entries); nothing to replay", m_Offset, EntryCount);
+ return;
+ }
+
+ std::atomic_uint64_t EntryOffset = m_Offset;
+ std::atomic_uint64_t BytesSent = 0;
+ std::atomic_uint64_t BytesReceived = 0;
+
+ Stopwatch Timer;
+
+ // The subcommand API does not receive argv, so look the zen executable path
+ // up from the current process to spawn child workers.
+ const std::filesystem::path SelfExePath = GetRunningExecutablePath();
+
+ if (m_ProcessCount > 1)
+ {
+ std::vector<std::unique_ptr<ProcessHandle>> WorkerProcesses;
+ WorkerProcesses.resize(m_ProcessCount);
+
+ ProcessMonitor Monitor;
+ for (int ProcessIndex = 0; ProcessIndex < m_ProcessCount; ++ProcessIndex)
+ {
+ std::string CommandLine =
+ fmt::format("{} cache replay --hosturl {} --path \"{}\" --offset {} --stride {} --numthreads {} --numproc {}"sv,
+ SelfExePath.string(),
+ m_HostName,
+ m_RecordingPath,
+ m_Stride == 1 ? 0 : m_Offset + ProcessIndex,
+ m_Stride,
+ m_ThreadCount,
+ 1);
+ CreateProcResult Result(CreateProc(SelfExePath, CommandLine));
+ WorkerProcesses[ProcessIndex] = std::make_unique<ProcessHandle>();
+ WorkerProcesses[ProcessIndex]->Initialize(Result);
+ Monitor.AddPid(WorkerProcesses[ProcessIndex]->Pid());
+ }
+ while (Monitor.IsRunning())
+ {
+ ZEN_CONSOLE("Waiting for worker processes...");
+ Sleep(1000);
+ }
+ return;
+ }
+ else
+ {
+ std::map<std::string, size_t> MethodTypes;
+ RwLock MethodTypesLock;
+
+ WorkerThreadPool WorkerPool(m_ThreadCount);
+
+ Latch WorkLatch(m_ThreadCount);
+ for (int WorkerIndex = 0; WorkerIndex < m_ThreadCount; ++WorkerIndex)
+ {
+ WorkerPool.ScheduleWork(
+ [this, &WorkLatch, EntryCount, &EntryOffset, &Replayer, &BytesSent, &BytesReceived, &MethodTypes, &MethodTypesLock]() {
+ auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
+
+ std::map<std::string, size_t> LocalMethodTypes;
+
+ auto ReduceTypes = MakeGuard([&] {
+ RwLock::ExclusiveLockScope __(MethodTypesLock);
+
+ for (auto& Entry : LocalMethodTypes)
+ {
+ MethodTypes[Entry.first] += Entry.second;
+ }
+ });
+
+ HttpClient Http = CacheCommand::CreateHttpClient(m_HostName);
+
+ uint64_t EntryIndex = EntryOffset.fetch_add(m_Stride);
+ while (EntryIndex < EntryCount)
+ {
+ IoBuffer Payload;
+ const zen::cache::RecordedRequestInfo RequestInfo = Replayer->GetRequest(EntryIndex, /* out */ Payload);
+
+ if (RequestInfo != zen::cache::RecordedRequestInfo::NullRequest)
+ {
+ CbPackage RequestPackage;
+ CbObject Request;
+
+ switch (RequestInfo.ContentType)
+ {
+ case ZenContentType::kCbPackage:
+ {
+ if (ParsePackageMessageWithLegacyFallback(Payload, RequestPackage))
+ {
+ Request = RequestPackage.GetObject();
+ }
+ }
+ break;
+ case ZenContentType::kCbObject:
+ {
+ Request = LoadCompactBinaryObject(Payload);
+ }
+ break;
+ }
+
+ RpcAcceptOptions OriginalAcceptOptions = static_cast<RpcAcceptOptions>(Request["AcceptFlags"sv].AsUInt16(0u));
+ int OriginalProcessPid = Request["Pid"sv].AsInt32(0);
+
+ int AdjustedPid = 0;
+ RpcAcceptOptions AdjustedAcceptOptions = RpcAcceptOptions::kNone;
+
+ if (!m_DisableLocalRefs)
+ {
+ if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowLocalReferences) ||
+ m_ForceAllowLocalRefs)
+ {
+ AdjustedAcceptOptions |= RpcAcceptOptions::kAllowLocalReferences;
+ if (!m_DisablePartialLocalRefs)
+ {
+ if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowPartialLocalReferences) ||
+ m_ForceAllowPartialLocalRefs)
+ {
+ AdjustedAcceptOptions |= RpcAcceptOptions::kAllowPartialLocalReferences;
+ }
+ }
+ if (!m_DisableLocalHandleRefs)
+ {
+ if (OriginalProcessPid != 0 || m_ForceAllowLocalHandleRef)
+ {
+ AdjustedPid = GetCurrentProcessId();
+ }
+ }
+ }
+ }
+
+ if (m_ShowMethodStats)
+ {
+ std::string MethodName = std::string(Request["Method"sv].AsString());
+ if (auto It = LocalMethodTypes.find(MethodName); It != LocalMethodTypes.end())
+ {
+ It->second++;
+ }
+ else
+ {
+ LocalMethodTypes[MethodName] = 1;
+ }
+ }
+
+ if (OriginalAcceptOptions != AdjustedAcceptOptions || OriginalProcessPid != AdjustedPid)
+ {
+ CbObjectWriter RequestCopyWriter;
+ for (const CbFieldView& Field : Request)
+ {
+ if (!Field.HasName())
+ {
+ RequestCopyWriter.AddField(Field);
+ continue;
+ }
+ std::string_view FieldName = Field.GetName();
+ if (FieldName == "Pid"sv)
+ {
+ continue;
+ }
+ if (FieldName == "AcceptFlags"sv)
+ {
+ continue;
+ }
+ RequestCopyWriter.AddField(FieldName, Field);
+ }
+ if (AdjustedPid != 0)
+ {
+ RequestCopyWriter.AddInteger("Pid"sv, AdjustedPid);
+ }
+ if (AdjustedAcceptOptions != RpcAcceptOptions::kNone)
+ {
+ RequestCopyWriter.AddInteger("AcceptFlags"sv, static_cast<uint16_t>(AdjustedAcceptOptions));
+ }
+
+ if (RequestInfo.ContentType == ZenContentType::kCbPackage)
+ {
+ RequestPackage.SetObject(RequestCopyWriter.Save());
+ std::vector<IoBuffer> Buffers = FormatPackageMessage(RequestPackage);
+ std::vector<SharedBuffer> SharedBuffers(Buffers.begin(), Buffers.end());
+ Payload = CompositeBuffer(std::move(SharedBuffers)).Flatten().AsIoBuffer();
+ }
+ else
+ {
+ RequestCopyWriter.Finalize();
+ Payload = IoBuffer(RequestCopyWriter.GetSaveSize());
+ RequestCopyWriter.Save(Payload.GetMutableView());
+ }
+ }
+
+ if (!m_DryRun)
+ {
+ Http.SetSessionId(RequestInfo.SessionId);
+ Payload.SetContentType(RequestInfo.ContentType);
+
+ HttpClient::Response Response =
+ Http.Post("/z$/$rpc", Payload, {HttpClient::Accept(RequestInfo.AcceptType)});
+
+ BytesSent.fetch_add(Payload.GetSize());
+ if (!Response)
+ {
+ ZEN_CONSOLE_ERROR("{}", Response);
+ break;
+ }
+ BytesReceived.fetch_add(Response.DownloadedBytes);
+ }
+ }
+
+ EntryIndex = EntryOffset.fetch_add(m_Stride);
+ }
+ },
+ WorkerThreadPool::EMode::EnableBacklog);
+ }
+
+ while (!WorkLatch.Wait(1000))
+ {
+ // EntryCount > m_Offset is guaranteed by the early-return above.
+ // EntryOffset atomically overshoots EntryCount (fetch_add past the
+ // end) when the workload finishes, so clamp before subtracting.
+ const uint64_t RequestsTotal = (EntryCount - m_Offset) / m_Stride;
+ const uint64_t CurrentOffset = EntryOffset.load();
+ const uint64_t RequestsRemaining = CurrentOffset < EntryCount ? (EntryCount - CurrentOffset) / m_Stride : 0;
+ const uint64_t PercentDone = RequestsTotal > 0 ? (RequestsTotal - RequestsRemaining) * 100 / RequestsTotal : 100;
+
+ ZEN_CONSOLE("[{:3}%] [{}] {} requests, {} remaining (sent {}, received {})",
+ PercentDone,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ RequestsTotal,
+ RequestsRemaining,
+ NiceBytes(BytesSent.load()),
+ NiceBytes(BytesReceived.load()));
+ }
+
+ if (m_ShowMethodStats)
+ {
+ for (const auto& It : MethodTypes)
+ {
+ ZEN_CONSOLE("{:18}: {:10}", It.first, It.second);
+ }
+ }
+ }
+
+ const uint64_t RequestsSent = (EntryOffset.load() - m_Offset) / m_Stride;
+ const uint64_t ElapsedMS = Timer.GetElapsedTimeMs();
+ const uint64_t Sent = BytesSent.load();
+ const uint64_t Received = BytesReceived.load();
+
+ ZEN_CONSOLE("Processed requests: {} ({}), payloads sent {} ({}), payloads received {} ({}) in {}.\nTotal runtime: {}",
+ RequestsSent,
+ NiceRate(RequestsSent, ElapsedMS, "req"),
+ NiceBytes(Sent),
+ NiceByteRate(Sent, ElapsedMS),
+ NiceBytes(Received),
+ NiceByteRate(Received, ElapsedMS),
+ NiceTimeSpanMs(ElapsedMS),
+ NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
+}
+
} // namespace zen
diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h
index 4f5b90f4d..a2834f73d 100644
--- a/src/zen/cmds/cache_cmd.h
+++ b/src/zen/cmds/cache_cmd.h
@@ -4,131 +4,319 @@
#include "../zen.h"
+#include <filesystem>
+
namespace zen {
-class DropCommand : public CacheStoreCommand
+// Base for `cache` subcommands. Registers the shared --hosturl option and
+// exposes ResolveHost() which subcommands must call before issuing HTTP
+// requests (it normalises m_HostName and throws if no host could be resolved).
+class CacheSubCmdBase : public ZenSubCmdBase
{
public:
- static constexpr char Name[] = "drop";
- static constexpr char Description[] = "Drop cache namespace or bucket";
+ CacheSubCmdBase(std::string_view Name, std::string_view Description);
+
+protected:
+ void ResolveHost();
+
+ std::string m_HostName;
+};
+
+class CacheDetailsSubCmd : public CacheSubCmdBase
+{
+public:
+ CacheDetailsSubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ bool m_CSV = false;
+ bool m_YAML = false;
+ bool m_Details = false;
+ bool m_AttachmentDetails = false;
+ std::string m_Namespace;
+ std::string m_Bucket;
+ std::string m_ValueKey;
+};
+
+class CacheDropSubCmd : public CacheSubCmdBase
+{
+public:
+ CacheDropSubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ std::string m_NamespaceName;
+ std::string m_BucketName;
+};
+
+class CacheGenSubCmd : public CacheSubCmdBase
+{
+public:
+ CacheGenSubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ std::string m_Namespace;
+ std::string m_Bucket;
+ uint64_t m_Count = 1;
+ uint64_t m_MinSize = 0;
+ uint64_t m_MaxSize = 0;
+ uint32_t m_MinAttachmentCount = 0;
+ uint32_t m_MaxAttachmentCount = 0;
+};
+
+class CacheGetSubCmd : public CacheSubCmdBase
+{
+public:
+ CacheGetSubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ std::string m_Namespace;
+ std::string m_Bucket;
+ std::string m_ValueKey;
+ std::string m_AttachmentHash;
+ std::filesystem::path m_OutputPath;
+ bool m_AsText = false;
+ bool m_Decompress = true;
+};
+
+class CacheInfoSubCmd : public CacheSubCmdBase
+{
+public:
+ CacheInfoSubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ std::string m_NamespaceName;
+ std::string m_SizeInfoBucketNames;
+ bool m_BucketSizeInfo = false;
+ bool m_YAML = false;
+ std::string m_BucketName;
+};
- DropCommand();
- ~DropCommand();
+class CacheRecordSubCmd : public CacheSubCmdBase
+{
+public:
+ CacheRecordSubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ std::string m_Path;
+};
+
+class CacheReplaySubCmd : public CacheSubCmdBase
+{
+public:
+ CacheReplaySubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
+private:
+ std::string m_RecordingPath;
+ bool m_OnHost = false;
+ bool m_ShowMethodStats = false;
+ int m_ProcessCount = 1;
+ int m_ThreadCount = 0;
+ uint64_t m_Offset = 0;
+ uint64_t m_Stride = 1;
+ bool m_ForceAllowLocalRefs = false;
+ bool m_DisableLocalRefs = false;
+ bool m_ForceAllowLocalHandleRef = false;
+ bool m_DisableLocalHandleRefs = false;
+ bool m_ForceAllowPartialLocalRefs = false;
+ bool m_DisablePartialLocalRefs = false;
+ bool m_DryRun = false;
+};
+
+class CacheStatsSubCmd : public CacheSubCmdBase
+{
+public:
+ CacheStatsSubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ bool m_YAML = false;
+};
+
+class CacheCommand : public CacheStoreCmdWithSubCommands
+{
+public:
+ static constexpr char Name[] = "cache";
+ static constexpr char Description[] = "Manage cache - info, stats, details, get, gen, drop, record, replay";
+
+ CacheCommand();
+ ~CacheCommand();
+
+ cxxopts::Options& Options() override { return m_Options; }
+
+private:
+ cxxopts::Options m_Options{Name, Description};
+
+ CacheDetailsSubCmd m_DetailsSubCmd;
+ CacheDropSubCmd m_DropSubCmd;
+ CacheGenSubCmd m_GenSubCmd;
+ CacheGetSubCmd m_GetSubCmd;
+ CacheInfoSubCmd m_InfoSubCmd;
+ CacheRecordSubCmd m_RecordSubCmd;
+ CacheReplaySubCmd m_ReplaySubCmd;
+ CacheStatsSubCmd m_StatsSubCmd;
+};
+
+// ---------------------------------------------------------------------------
+// Deprecated legacy top-level commands. These forward to the corresponding
+// 'cache <sub>' subcommand so that existing scripts keep working. They are
+// hidden from the top-level `zen --help` listing; `zen cache --help` is the
+// canonical discovery surface now.
+
+namespace cache_legacy_shim {
+ void RunAs(const char* SubCommandName, const ZenCliOptions& GlobalOptions, int argc, char** argv);
+}
+
+class DeprecatedCacheStoreCommand : public CacheStoreCommand
+{
+public:
+ bool IsHidden() const override { return true; }
+};
+
+class DropCommand : public DeprecatedCacheStoreCommand
+{
+public:
+ static constexpr char Name[] = "drop";
+ static constexpr char Description[] = "(deprecated, use 'cache drop') Drop cache namespace or bucket";
+
+ void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override
+ {
+ cache_legacy_shim::RunAs("drop", GlobalOptions, argc, argv);
+ }
+ cxxopts::Options& Options() override { return m_Options; }
private:
cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- std::string m_NamespaceName;
- std::string m_BucketName;
};
-class CacheInfoCommand : public CacheStoreCommand
+class CacheInfoCommand : public DeprecatedCacheStoreCommand
{
public:
static constexpr char Name[] = "cache-info";
- static constexpr char Description[] = "Info on cache, namespace or bucket";
+ static constexpr char Description[] = "(deprecated, use 'cache info') Info on cache, namespace or bucket";
- CacheInfoCommand();
- ~CacheInfoCommand();
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
+ void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override
+ {
+ cache_legacy_shim::RunAs("info", GlobalOptions, argc, argv);
+ }
+ cxxopts::Options& Options() override { return m_Options; }
private:
cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- std::string m_NamespaceName;
- std::string m_SizeInfoBucketNames;
- bool m_BucketSizeInfo = false;
- std::string m_BucketName;
};
-class CacheStatsCommand : public CacheStoreCommand
+class CacheStatsCommand : public DeprecatedCacheStoreCommand
{
public:
static constexpr char Name[] = "cache-stats";
- static constexpr char Description[] = "Stats on cache";
+ static constexpr char Description[] = "(deprecated, use 'cache stats') Stats on cache";
- CacheStatsCommand();
- ~CacheStatsCommand();
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
+ void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override
+ {
+ cache_legacy_shim::RunAs("stats", GlobalOptions, argc, argv);
+ }
+ cxxopts::Options& Options() override { return m_Options; }
private:
cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
};
-class CacheDetailsCommand : public CacheStoreCommand
+class CacheDetailsCommand : public DeprecatedCacheStoreCommand
{
public:
static constexpr char Name[] = "cache-details";
- static constexpr char Description[] = "Details on cache";
+ static constexpr char Description[] = "(deprecated, use 'cache details') Details on cache";
- CacheDetailsCommand();
- ~CacheDetailsCommand();
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
+ void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override
+ {
+ cache_legacy_shim::RunAs("details", GlobalOptions, argc, argv);
+ }
+ cxxopts::Options& Options() override { return m_Options; }
private:
cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- bool m_CSV = false;
- bool m_Details = false;
- bool m_AttachmentDetails = false;
- std::string m_Namespace;
- std::string m_Bucket;
- std::string m_ValueKey;
};
-class CacheGenerateCommand : public CacheStoreCommand
+class CacheGenerateCommand : public DeprecatedCacheStoreCommand
{
public:
static constexpr char Name[] = "cache-gen";
- static constexpr char Description[] = "Generates cache values into a bucket";
+ static constexpr char Description[] = "(deprecated, use 'cache gen') Generates cache values into a bucket";
- CacheGenerateCommand();
- ~CacheGenerateCommand();
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
+ void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override
+ {
+ cache_legacy_shim::RunAs("gen", GlobalOptions, argc, argv);
+ }
+ cxxopts::Options& Options() override { return m_Options; }
private:
cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- std::string m_Namespace;
- std::string m_Bucket;
- uint64_t m_Count = 1;
-
- uint64_t m_MinSize = 0;
- uint64_t m_MaxSize = 0;
- uint32_t m_MinAttachmentCount = 0;
- uint32_t m_MaxAttachmentCount = 0;
};
-class CacheGetCommand : public CacheStoreCommand
+class CacheGetCommand : public DeprecatedCacheStoreCommand
{
public:
static constexpr char Name[] = "cache-get";
- static constexpr char Description[] = "Get cache values/records or attachments from a bucket";
+ static constexpr char Description[] = "(deprecated, use 'cache get') Get cache values/records or attachments from a bucket";
- CacheGetCommand();
- ~CacheGetCommand();
+ void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override
+ {
+ cache_legacy_shim::RunAs("get", GlobalOptions, argc, argv);
+ }
+ cxxopts::Options& Options() override { return m_Options; }
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
+private:
+ cxxopts::Options m_Options{Name, Description};
+};
+
+class RpcStartRecordingCommand : public DeprecatedCacheStoreCommand
+{
+public:
+ static constexpr char Name[] = "rpc-record-start";
+ static constexpr char Description[] = "(deprecated, use 'cache record <path>') Starts recording of cache rpc requests on a host";
+
+ void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override
+ {
+ cache_legacy_shim::RunAs("record", GlobalOptions, argc, argv);
+ }
+ cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- std::string m_Namespace;
- std::string m_Bucket;
- std::string m_ValueKey;
- std::string m_AttachmentHash;
- std::filesystem::path m_OutputPath;
- bool m_AsText = false;
- bool m_Decompress = true;
+ cxxopts::Options m_Options{Name, Description};
+};
+
+class RpcStopRecordingCommand : public DeprecatedCacheStoreCommand
+{
+public:
+ static constexpr char Name[] = "rpc-record-stop";
+ static constexpr char Description[] = "(deprecated, use 'cache record stop') Stops recording of cache rpc requests on a host";
+
+ void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+ cxxopts::Options& Options() override { return m_Options; }
+
+private:
+ cxxopts::Options m_Options{Name, Description};
+};
+
+class RpcReplayCommand : public DeprecatedCacheStoreCommand
+{
+public:
+ static constexpr char Name[] = "rpc-record-replay";
+ static constexpr char Description[] = "(deprecated, use 'cache replay') Replays a previously recorded session of rpc requests";
+
+ void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override
+ {
+ cache_legacy_shim::RunAs("replay", GlobalOptions, argc, argv);
+ }
+ cxxopts::Options& Options() override { return m_Options; }
+
+private:
+ cxxopts::Options m_Options{Name, Description};
};
} // namespace zen
diff --git a/src/zen/cmds/compute_cmd.cpp b/src/zen/cmds/compute_cmd.cpp
new file mode 100644
index 000000000..9a350c69c
--- /dev/null
+++ b/src/zen/cmds/compute_cmd.cpp
@@ -0,0 +1,88 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "compute_cmd.h"
+
+#if ZEN_WITH_COMPUTE_SERVICES
+
+# include "zenserviceclient.h"
+
+# include <zencore/compactbinary.h>
+# include <zencore/logging.h>
+# include <zenhttp/httpclient.h>
+
+using namespace std::literals;
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+// ComputeRecordStartSubCmd
+
+ComputeRecordStartSubCmd::ComputeRecordStartSubCmd() : ZenSubCmdBase("record-start", "Start recording compute actions")
+{
+ SubOptions().add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+}
+
+void
+ComputeRecordStartSubCmd::Run(const ZenCliOptions& GlobalOptions)
+{
+ ZEN_UNUSED(GlobalOptions);
+
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "record-start"});
+ HttpClient& Http = Service.Http();
+ if (HttpClient::Response Response = Http.Post("/compute/record/start"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{}))
+ {
+ CbObject Obj = Response.AsObject();
+ std::string_view Path = Obj["path"sv].AsString();
+ ZEN_CONSOLE("recording started: " ZEN_BRIGHT_GREEN("{}"), Path);
+ }
+ else
+ {
+ Response.ThrowError("Failed to start recording");
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+// ComputeRecordStopSubCmd
+
+ComputeRecordStopSubCmd::ComputeRecordStopSubCmd() : ZenSubCmdBase("record-stop", "Stop recording compute actions")
+{
+ SubOptions().add_option("", "u", "hosturl", ZenCmdBase::kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+}
+
+void
+ComputeRecordStopSubCmd::Run(const ZenCliOptions& GlobalOptions)
+{
+ ZEN_UNUSED(GlobalOptions);
+
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = "record-stop"});
+ HttpClient& Http = Service.Http();
+ if (HttpClient::Response Response = Http.Post("/compute/record/stop"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{}))
+ {
+ CbObject Obj = Response.AsObject();
+ std::string_view Path = Obj["path"sv].AsString();
+ ZEN_CONSOLE("recording stopped: " ZEN_BRIGHT_GREEN("{}"), Path);
+ }
+ else
+ {
+ Response.ThrowError("Failed to stop recording");
+ }
+}
+
+//////////////////////////////////////////////////////////////////////////
+// ComputeCommand
+
+ComputeCommand::ComputeCommand()
+{
+ m_Options.add_options()("h,help", "Print help");
+ m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), "");
+ m_Options.parse_positional({"subcommand"});
+
+ AddSubCommand(m_RecordStartSubCmd);
+ AddSubCommand(m_RecordStopSubCmd);
+}
+
+ComputeCommand::~ComputeCommand() = default;
+
+} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/src/zen/cmds/compute_cmd.h b/src/zen/cmds/compute_cmd.h
new file mode 100644
index 000000000..b26f639c4
--- /dev/null
+++ b/src/zen/cmds/compute_cmd.h
@@ -0,0 +1,53 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+#include <string>
+
+#if ZEN_WITH_COMPUTE_SERVICES
+
+namespace zen {
+
+class ComputeRecordStartSubCmd : public ZenSubCmdBase
+{
+public:
+ ComputeRecordStartSubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ std::string m_HostName;
+};
+
+class ComputeRecordStopSubCmd : public ZenSubCmdBase
+{
+public:
+ ComputeRecordStopSubCmd();
+ void Run(const ZenCliOptions& GlobalOptions) override;
+
+private:
+ std::string m_HostName;
+};
+
+class ComputeCommand : public ZenCmdWithSubCommands
+{
+public:
+ static constexpr char Name[] = "compute";
+ static constexpr char Description[] = "Compute service operations";
+
+ ComputeCommand();
+ ~ComputeCommand();
+
+ cxxopts::Options& Options() override { return m_Options; }
+
+private:
+ cxxopts::Options m_Options{Name, Description};
+ std::string m_SubCommand;
+ ComputeRecordStartSubCmd m_RecordStartSubCmd;
+ ComputeRecordStopSubCmd m_RecordStopSubCmd;
+};
+
+} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp
deleted file mode 100644
index 530661607..000000000
--- a/src/zen/cmds/copy_cmd.cpp
+++ /dev/null
@@ -1,207 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "copy_cmd.h"
-
-#include <zencore/filesystem.h>
-#include <zencore/fmtutils.h>
-#include <zencore/logging.h>
-#include <zencore/string.h>
-#include <zencore/timer.h>
-
-namespace zen {
-
-CopyCommand::CopyCommand()
-{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_options()("no-clone", "Do not perform block clone", cxxopts::value(m_NoClone)->default_value("false"));
- m_Options.add_options()("must-clone",
- "Always perform block clone (fails if clone is not possible)",
- cxxopts::value(m_MustClone)->default_value("false"));
- m_Options.add_option("", "s", "source", "Copy source", cxxopts::value(m_CopySource), "<file/directory>");
- m_Options.add_option("", "t", "target", "Copy target", cxxopts::value(m_CopyTarget), "<file/directory>");
- m_Options.parse_positional({"source", "target"});
-}
-
-CopyCommand::~CopyCommand() = default;
-
-void
-CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
-{
- ZEN_UNUSED(GlobalOptions);
-
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- // Validate arguments
-
- if (m_CopySource.empty())
- throw OptionParseException("'--source' is required", m_Options.help());
-
- if (m_CopyTarget.empty())
- throw OptionParseException("'--target' is required", m_Options.help());
-
- std::filesystem::path FromPath = m_CopySource;
- std::filesystem::path ToPath = m_CopyTarget;
-
- std::error_code Ec;
- std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec);
-
- if (!Ec)
- {
- std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec);
-
- if (!Ec)
- {
- if (FromCanonical == ToCanonical)
- {
- throw std::runtime_error("Target and source must be distinct files or directories");
- }
- }
- }
-
- const bool IsFileCopy = IsFile(m_CopySource);
- const bool IsDirCopy = IsDir(m_CopySource);
-
- if (!IsFileCopy && !IsDirCopy)
- {
- throw std::runtime_error("Invalid source specification (neither directory nor file)");
- }
-
- if (IsFileCopy && IsDirCopy)
- {
- throw std::runtime_error("Invalid source specification (both directory AND file!?)");
- }
-
- if (IsDirCopy)
- {
- if (IsFile(ToPath))
- {
- throw std::runtime_error("Attempted copy of directory into file");
- }
-
- if (!IsDir(ToPath))
- {
- CreateDirectories(ToPath);
- }
-
- std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec);
-
- if (!Ec)
- {
- if (ToCanonical.generic_string().starts_with(FromCanonical.generic_string()) ||
- FromCanonical.generic_string().starts_with(ToCanonical.generic_string()))
- {
- throw std::runtime_error("Invalid parent/child relationship for source/target directories");
- }
- }
-
- // Multi file copy
-
- ZEN_CONSOLE("copying {} -> {}", FromPath, ToPath);
-
- zen::Stopwatch Timer;
-
- struct CopyVisitor : public FileSystemTraversal::TreeVisitor
- {
- CopyVisitor(std::filesystem::path InBasePath, zen::CopyFileOptions InCopyOptions)
- : BasePath(InBasePath)
- , CopyOptions(InCopyOptions)
- {
- }
-
- virtual void VisitFile(const std::filesystem::path& Parent,
- const path_view& File,
- uint64_t FileSize,
- uint32_t,
- uint64_t) override
- {
- ZEN_UNUSED(FileSize);
- std::error_code Ec;
- const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec);
-
- if (Ec)
- {
- FailedFileCount++;
- }
- else
- {
- const std::filesystem::path FromPath = Parent / File;
- const std::filesystem::path ToPath = TargetPath / Relative / File;
-
- try
- {
- zen::CreateDirectories(TargetPath / Relative);
- if (zen::CopyFile(FromPath, ToPath, CopyOptions))
- {
- ++FileCount;
- ByteCount += FileSize;
- }
- else
- {
- throw std::logic_error("CopyFile failed in an unexpected way");
- }
- }
- catch (const std::exception& Ex)
- {
- ++FailedFileCount;
-
- ZEN_CONSOLE_ERROR("Failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what());
- }
- }
- }
-
- virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; }
-
- std::filesystem::path BasePath;
- std::filesystem::path TargetPath;
- zen::CopyFileOptions CopyOptions;
- int FileCount = 0;
- uint64_t ByteCount = 0;
- int FailedFileCount = 0;
- };
-
- zen::CopyFileOptions CopyOptions;
- CopyOptions.EnableClone = !m_NoClone;
- CopyOptions.MustClone = m_MustClone;
-
- CopyVisitor Visitor{FromPath, CopyOptions};
- Visitor.TargetPath = ToPath;
-
- FileSystemTraversal Traversal;
- Traversal.TraverseFileSystem(FromPath, Visitor);
-
- ZEN_CONSOLE("Copy of {} files ({}) completed in {} ({})",
- Visitor.FileCount,
- NiceBytes(Visitor.ByteCount),
- zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- zen::NiceRate(Visitor.ByteCount, (uint32_t)Timer.GetElapsedTimeMs()));
-
- if (Visitor.FailedFileCount)
- {
- throw std::runtime_error(fmt::format("{} file copy operations FAILED", Visitor.FailedFileCount));
- }
- }
- else
- {
- // Single file copy
-
- zen::Stopwatch Timer;
-
- zen::CopyFileOptions CopyOptions;
- CopyOptions.EnableClone = !m_NoClone;
-
- zen::CreateDirectories(ToPath.parent_path());
- if (zen::CopyFile(FromPath, ToPath, CopyOptions))
- {
- ZEN_CONSOLE("Copy completed in {}", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- }
- else
- {
- throw std::runtime_error(fmt::format("Failed to copy '{}' to '{}'", FromPath, ToPath));
- }
- }
-}
-
-} // namespace zen
diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h
deleted file mode 100644
index 757a8e691..000000000
--- a/src/zen/cmds/copy_cmd.h
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include "../zen.h"
-
-namespace zen {
-
-/** Copy files, possibly using block cloning
- */
-class CopyCommand : public ZenCmdBase
-{
-public:
- static constexpr char Name[] = "copy";
- static constexpr char Description[] = "Copy file(s)";
-
- CopyCommand();
- ~CopyCommand();
-
- virtual cxxopts::Options& Options() override { return m_Options; }
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
-
-private:
- cxxopts::Options m_Options{Name, Description};
- std::filesystem::path m_CopySource;
- std::filesystem::path m_CopyTarget;
- bool m_NoClone = false;
- bool m_MustClone = false;
-};
-
-} // namespace zen
diff --git a/src/zen/cmds/dedup_cmd.cpp b/src/zen/cmds/dedup_cmd.cpp
index 9ef50a97d..18ad56aec 100644
--- a/src/zen/cmds/dedup_cmd.cpp
+++ b/src/zen/cmds/dedup_cmd.cpp
@@ -240,7 +240,12 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
zen::BLAKE3Stream b3s;
- zen::ScanFile(Entry.path(), 64 * 1024, [&](const void* Data, size_t Size) { b3s.Append(Data, Size); });
+ if (std::error_code ScanEc =
+ zen::ScanFile(Entry.path(), 64 * 1024, [&](const void* Data, size_t Size) { b3s.Append(Data, Size); });
+ ScanEc)
+ {
+ throw std::system_error(ScanEc, fmt::format("Failed to scan file '{}'", Entry.path()));
+ }
Hash = b3s.GetHash();
}
@@ -279,7 +284,11 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
Options.EnableClone = true;
Options.MustClone = true;
- zen::CopyFile(Dupe->path(), Entry.path(), Options);
+ if (std::error_code Ec = zen::CopyFile(Dupe->path(), Entry.path(), Options); Ec)
+ {
+ ZEN_ERROR("Failed to clone '{}' to '{}': {}", Dupe->path(), Entry.path(), Ec.message());
+ continue;
+ }
DupeBytes += Entry.file_size();
}
diff --git a/src/zen/cmds/exec_cmd.cpp b/src/zen/cmds/exec_cmd.cpp
index 9719fce77..60968b521 100644
--- a/src/zen/cmds/exec_cmd.cpp
+++ b/src/zen/cmds/exec_cmd.cpp
@@ -2,6 +2,8 @@
#include "exec_cmd.h"
+#include "zenserviceclient.h"
+
#include <zencompute/computeservice.h>
#include <zencompute/recordingreader.h>
#include <zencore/compactbinary.h>
@@ -23,6 +25,8 @@
#include <zenhttp/httpclient.h>
#include <zenhttp/packageformat.h>
+#include "consoleprogress.h"
+
#include <EASTL/hash_map.h>
#include <EASTL/hash_set.h>
#include <EASTL/map.h>
@@ -114,7 +118,7 @@ namespace {
} // namespace
//////////////////////////////////////////////////////////////////////////
-// ExecSessionConfig — read-only configuration for a session run
+// ExecSessionConfig - read-only configuration for a session run
struct ExecSessionConfig
{
@@ -124,17 +128,18 @@ struct ExecSessionConfig
std::vector<ExecFunctionDefinition>& FunctionList; // mutable for EmitFunctionListOnce
std::string_view OrchestratorUrl;
const std::filesystem::path& OutputPath;
- int Offset = 0;
- int Stride = 1;
- int Limit = 0;
- bool Verbose = false;
- bool Quiet = false;
- bool DumpActions = false;
- bool Binary = false;
+ int Offset = 0;
+ int Stride = 1;
+ int Limit = 0;
+ bool Verbose = false;
+ bool Quiet = false;
+ bool DumpActions = false;
+ bool Binary = false;
+ ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty;
};
//////////////////////////////////////////////////////////////////////////
-// ExecSessionRunner — owns per-run state, drives the session lifecycle
+// ExecSessionRunner - owns per-run state, drives the session lifecycle
class ExecSessionRunner
{
@@ -345,8 +350,6 @@ ExecSessionRunner::DrainCompletedJobs()
}
m_PendingJobs.Remove(CompleteLsn);
-
- ZEN_CONSOLE("completed: LSN {} ({} still pending)", CompleteLsn, m_PendingJobs.GetSize());
}
}
}
@@ -897,17 +900,22 @@ ExecSessionRunner::Run()
// Then submit work items
- int FailedWorkCounter = 0;
- size_t RemainingWorkItems = m_Config.RecordingReader.GetActionCount();
- int SubmittedWorkItems = 0;
+ std::atomic<int> FailedWorkCounter{0};
+ std::atomic<size_t> RemainingWorkItems{m_Config.RecordingReader.GetActionCount()};
+ std::atomic<int> SubmittedWorkItems{0};
+ size_t TotalWorkItems = RemainingWorkItems.load();
- ZEN_CONSOLE("submitting {} work items", RemainingWorkItems);
+ std::unique_ptr<ProgressBase> ProgressOwner(CreateConsoleProgress(m_Config.ProgressMode));
+ std::unique_ptr<ProgressBase::ProgressBar> SubmitProgress = ProgressOwner->CreateProgressBar("Submit");
+ SubmitProgress->UpdateState(
+ {.Task = "Submitting work items", .TotalCount = TotalWorkItems, .RemainingCount = RemainingWorkItems.load()},
+ false);
int OffsetCounter = m_Config.Offset;
int StrideCounter = m_Config.Stride;
auto ShouldSchedule = [&]() -> bool {
- if (m_Config.Limit && SubmittedWorkItems >= m_Config.Limit)
+ if (m_Config.Limit && SubmittedWorkItems.load() >= m_Config.Limit)
{
// Limit reached, ignore
@@ -1005,17 +1013,14 @@ ExecSessionRunner::Run()
{
const int32_t LsnField = EnqueueResult.Lsn;
- --RemainingWorkItems;
- ++SubmittedWorkItems;
+ size_t Remaining = --RemainingWorkItems;
+ int Submitted = ++SubmittedWorkItems;
- if (!m_Config.Quiet)
- {
- ZEN_CONSOLE("submitted work item #{} - LSN {} - {}. {} remaining",
- SubmittedWorkItems,
- LsnField,
- NiceTimeSpanMs(SubmitTimer.GetElapsedTimeMs()),
- RemainingWorkItems);
- }
+ SubmitProgress->UpdateState({.Task = "Submitting work items",
+ .Details = fmt::format("#{} LSN {}", Submitted, LsnField),
+ .TotalCount = TotalWorkItems,
+ .RemainingCount = Remaining},
+ false);
if (!m_Config.OutputPath.empty())
{
@@ -1055,22 +1060,37 @@ ExecSessionRunner::Run()
},
TargetParallelism);
+ SubmitProgress->Finish();
+
// Wait until all pending work is complete
+ size_t TotalPendingJobs = m_PendingJobs.GetSize();
+
+ std::unique_ptr<ProgressBase::ProgressBar> CompletionProgress = ProgressOwner->CreateProgressBar("Execute");
+
while (!m_PendingJobs.IsEmpty())
{
- // TODO: improve this logic
- zen::Sleep(500);
+ size_t PendingCount = m_PendingJobs.GetSize();
+ CompletionProgress->UpdateState(
+ {.Task = "Executing work items",
+ .Details = fmt::format("{} completed, {} remaining", TotalPendingJobs - PendingCount, PendingCount),
+ .TotalCount = TotalPendingJobs,
+ .RemainingCount = PendingCount},
+ false);
+
+ zen::Sleep(ProgressOwner->GetProgressUpdateDelayMS());
DrainCompletedJobs();
SendOrchestratorHeartbeat();
}
+ CompletionProgress->Finish();
+
// Write summary files
WriteSummaryFiles();
- if (FailedWorkCounter)
+ if (FailedWorkCounter.load())
{
return 1;
}
@@ -1089,7 +1109,7 @@ ExecHttpSubCmd::ExecHttpSubCmd(ExecCommand& Parent) : ZenSubCmdBase("http", "For
void
ExecHttpSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = ExecCommand::Name});
ZEN_ASSERT(m_Parent.m_ChunkResolver);
ChunkResolver& Resolver = *m_Parent.m_ChunkResolver;
@@ -1097,7 +1117,7 @@ ExecHttpSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp");
zen::compute::ComputeServiceSession ComputeSession(Resolver);
- ComputeSession.AddRemoteRunner(Resolver, TempPath, m_HostName);
+ ComputeSession.AddRemoteRunner(Resolver, TempPath, Service.HostSpec());
Stopwatch ExecTimer;
int ReturnValue = m_Parent.RunSession(ComputeSession);
@@ -1423,6 +1443,12 @@ ExecCommand::OnParentOptionsParsed(const ZenCliOptions& GlobalOptions)
int
ExecCommand::RunSession(zen::compute::ComputeServiceSession& ComputeSession, std::string_view OrchestratorUrl)
{
+ ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty;
+ if (m_QuietLogging)
+ {
+ ProgressMode = ConsoleProgressMode::Quiet;
+ }
+
ExecSessionConfig Config{
.Resolver = *m_ChunkResolver,
.RecordingReader = *m_RecordingReader,
@@ -1437,6 +1463,7 @@ ExecCommand::RunSession(zen::compute::ComputeServiceSession& ComputeSession, std
.Quiet = m_QuietLogging,
.DumpActions = m_DumpActions,
.Binary = m_Binary,
+ .ProgressMode = ProgressMode,
};
ExecSessionRunner Runner(ComputeSession, Config);
diff --git a/src/zen/cmds/history_cmd.cpp b/src/zen/cmds/history_cmd.cpp
new file mode 100644
index 000000000..27faae1eb
--- /dev/null
+++ b/src/zen/cmds/history_cmd.cpp
@@ -0,0 +1,228 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "history_cmd.h"
+
+#include <zenbase/zenbase.h>
+#include <zencore/except_fmt.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/process.h>
+#include <zenutil/consoletui.h>
+#include <zenutil/invocationhistory.h>
+
+#include <algorithm>
+
+namespace zen {
+
+HistoryCommand::HistoryCommand()
+{
+ m_Options.add_options()("h,help", "Print help");
+ m_Options.add_options()("filter",
+ "Filter by executable ('zen' or 'zenserver' / 'server')",
+ cxxopts::value<std::string>(m_Filter)->default_value(""));
+ m_Options.add_options()("print", "Print the selected command line instead of running it", cxxopts::value<bool>(m_Print));
+ m_Options.add_options()("l,list", "List all invocations to stdout instead of showing the picker", cxxopts::value<bool>(m_List));
+}
+
+HistoryCommand::~HistoryCommand()
+{
+}
+
+namespace {
+
+ bool KeepRecord(const HistoryRecord& Rec, std::string_view Filter)
+ {
+ if (Filter.empty())
+ {
+ return true;
+ }
+ if (Filter == "zen")
+ {
+ return Rec.Exe == "zen";
+ }
+ if (Filter == "zenserver" || Filter == "server")
+ {
+ return Rec.Exe == "zenserver";
+ }
+ return true;
+ }
+
+ std::string BuildLabel(const HistoryRecord& Rec, int32_t TerminalCols)
+ {
+ constexpr int32_t kIndicator = 3; // " > " or " " prefix from TuiPickOne
+ constexpr int32_t kEllipsis = 3; // "..."
+
+ std::string Exe = Rec.Exe.empty() ? std::string("?") : Rec.Exe;
+ std::string Lbl = fmt::format("{} {:<9} pid {:<7}", Rec.Ts, Exe, Rec.Pid);
+
+ if (!Rec.CmdLine.empty())
+ {
+ int32_t Available = TerminalCols - kIndicator - 2 - static_cast<int32_t>(Lbl.size());
+ if (Available > kEllipsis)
+ {
+ Lbl += " ";
+ if (static_cast<int32_t>(Rec.CmdLine.size()) <= Available)
+ {
+ Lbl += Rec.CmdLine;
+ }
+ else
+ {
+ Lbl.append(Rec.CmdLine, 0, static_cast<size_t>(Available - kEllipsis));
+ Lbl += "...";
+ }
+ }
+ }
+ return Lbl;
+ }
+
+ int SpawnZen(const HistoryRecord& Rec)
+ {
+ const std::filesystem::path TargetPath = GetRunningExecutablePath();
+ ZEN_CONSOLE("Running: {}", Rec.CmdLine);
+
+ CreateProcOptions Opts{};
+ CreateProcResult Result = CreateProc(TargetPath, Rec.CmdLine, Opts);
+ if (!Result)
+ {
+ throw zen::runtime_error("failed to launch '{}'", TargetPath);
+ }
+
+ ProcessHandle Proc;
+ Proc.Initialize(Result);
+ return Proc.WaitExitCode();
+ }
+
+ void SpawnZenServerDetached(const HistoryRecord& Rec)
+ {
+ const std::filesystem::path TargetPath = GetRunningExecutablePath().parent_path() / ("zenserver" ZEN_EXE_SUFFIX_LITERAL);
+
+ ZEN_CONSOLE("Launching detached: {}", Rec.CmdLine);
+
+ CreateProcOptions Opts{};
+ Opts.Flags = CreateProcOptions::Flag_NoConsole | CreateProcOptions::Flag_NewProcessGroup;
+ CreateProcResult Result = CreateProc(TargetPath, Rec.CmdLine, Opts);
+ if (!Result)
+ {
+ throw zen::runtime_error("failed to launch '{}'", TargetPath);
+ }
+
+#if ZEN_PLATFORM_WINDOWS
+ // Take ownership of the handle just to release it; we do not wait.
+ ProcessHandle Proc;
+ Proc.Initialize(Result);
+ ZEN_CONSOLE("Launched {} (pid {})", TargetPath, Proc.Pid());
+#else
+ ZEN_CONSOLE("Launched {} (pid {})", TargetPath, static_cast<int>(Result));
+#endif
+ }
+
+ void PrintPlainTable(const std::vector<HistoryRecord>& Records)
+ {
+ if (Records.empty())
+ {
+ ZEN_CONSOLE("No invocation history available.");
+ return;
+ }
+ for (const HistoryRecord& Rec : Records)
+ {
+ std::string Line = fmt::format("{} {:<9} pid {:<7}", Rec.Ts, Rec.Exe, Rec.Pid);
+ if (!Rec.CmdLine.empty())
+ {
+ Line += " ";
+ Line += Rec.CmdLine;
+ }
+ ZEN_CONSOLE("{}", Line);
+ }
+ }
+
+} // namespace
+
+void
+HistoryCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+ ZEN_UNUSED(GlobalOptions);
+
+ if (!ParseOptions(argc, argv))
+ {
+ return;
+ }
+
+ if (m_List && m_Print)
+ {
+ throw OptionParseException("'--list' conflicts with '--print'", m_Options.help());
+ }
+
+ std::vector<HistoryRecord> All = ReadInvocationHistory();
+
+ std::vector<HistoryRecord> Filtered;
+ Filtered.reserve(All.size());
+ for (HistoryRecord& Rec : All)
+ {
+ if (KeepRecord(Rec, m_Filter))
+ {
+ Filtered.push_back(std::move(Rec));
+ }
+ }
+
+ std::reverse(Filtered.begin(), Filtered.end());
+
+ if (Filtered.empty())
+ {
+ ZEN_CONSOLE("No invocation history available{}.", m_Filter.empty() ? "" : " for this filter");
+ return;
+ }
+
+ if (m_List || !IsTuiAvailable())
+ {
+ PrintPlainTable(Filtered);
+ return;
+ }
+
+ const int32_t Cols = static_cast<int32_t>(TuiConsoleColumns());
+ std::vector<std::string> Labels;
+ Labels.reserve(Filtered.size());
+ for (const HistoryRecord& Rec : Filtered)
+ {
+ Labels.push_back(BuildLabel(Rec, Cols));
+ }
+
+ int Selected = TuiPickOne("Recent invocations. Select one to re-run:", Labels);
+ if (Selected < 0)
+ {
+ return;
+ }
+
+ const HistoryRecord& Pick = Filtered[Selected];
+
+ if (m_Print)
+ {
+ ZEN_CONSOLE("{}", Pick.CmdLine);
+ return;
+ }
+
+ if (Pick.CmdLine.empty())
+ {
+ throw zen::runtime_error("selected record has no command line");
+ }
+
+ if (Pick.Exe == "zenserver")
+ {
+ SpawnZenServerDetached(Pick);
+ return;
+ }
+
+ if (Pick.Exe == "zen")
+ {
+ int ExitCode = SpawnZen(Pick);
+ if (ExitCode != 0)
+ {
+ throw ErrorWithReturnCode(fmt::format("zen exited with code {}", ExitCode), ExitCode);
+ }
+ return;
+ }
+
+ throw zen::runtime_error("unknown executable '{}' in history record", Pick.Exe);
+}
+
+} // namespace zen
diff --git a/src/zen/cmds/trace_cmd.h b/src/zen/cmds/history_cmd.h
index 6eb0ba22b..d7dd2f078 100644
--- a/src/zen/cmds/trace_cmd.h
+++ b/src/zen/cmds/history_cmd.h
@@ -6,24 +6,23 @@
namespace zen {
-class TraceCommand : public ZenCmdBase
+class HistoryCommand : public ZenCmdBase
{
public:
- static constexpr char Name[] = "trace";
- static constexpr char Description[] = "Control zen realtime tracing";
+ HistoryCommand();
+ ~HistoryCommand();
- TraceCommand();
- ~TraceCommand();
+ static constexpr char Name[] = "history";
+ static constexpr char Description[] = "List recent zen/zenserver invocations and optionally re-run one";
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- bool m_Stop = false;
- std::string m_TraceHost;
- std::string m_TraceFile;
+ std::string m_Filter;
+ bool m_Print = false;
+ bool m_List = false;
};
} // namespace zen
diff --git a/src/zen/cmds/hub_cmd.cpp b/src/zen/cmds/hub_cmd.cpp
index 5bdd3a922..75408a5e9 100644
--- a/src/zen/cmds/hub_cmd.cpp
+++ b/src/zen/cmds/hub_cmd.cpp
@@ -2,6 +2,8 @@
#include "hub_cmd.h"
+#include "zenserviceclient.h"
+
#include <zencore/compactbinary.h>
#include <zencore/compactbinaryutil.h>
#include <zencore/filesystem.h>
@@ -210,17 +212,13 @@ HubProvisionSubCmd::Run(const ZenCliOptions& GlobalOptions)
{
ZEN_UNUSED(GlobalOptions);
- m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName);
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve hub host specification", SubOptions().help());
- }
if (m_ModuleId.empty())
{
throw OptionParseException("moduleid is required", SubOptions().help());
}
- HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Resp =
Http.Post(fmt::format("/hub/modules/{}/provision", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{}))
{
@@ -251,17 +249,13 @@ HubDeprovisionSubCmd::Run(const ZenCliOptions& GlobalOptions)
{
ZEN_UNUSED(GlobalOptions);
- m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName);
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve hub host specification", SubOptions().help());
- }
if (m_ModuleId.empty())
{
throw OptionParseException("moduleid is required", SubOptions().help());
}
- HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Resp =
Http.Post(fmt::format("/hub/modules/{}/deprovision", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{}))
{
@@ -290,17 +284,13 @@ HubHibernateSubCmd::Run(const ZenCliOptions& GlobalOptions)
{
ZEN_UNUSED(GlobalOptions);
- m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName);
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve hub host specification", SubOptions().help());
- }
if (m_ModuleId.empty())
{
throw OptionParseException("moduleid is required", SubOptions().help());
}
- HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Resp =
Http.Post(fmt::format("/hub/modules/{}/hibernate", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{}))
{
@@ -329,17 +319,13 @@ HubWakeSubCmd::Run(const ZenCliOptions& GlobalOptions)
{
ZEN_UNUSED(GlobalOptions);
- m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName);
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve hub host specification", SubOptions().help());
- }
if (m_ModuleId.empty())
{
throw OptionParseException("moduleid is required", SubOptions().help());
}
- HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Resp =
Http.Post(fmt::format("/hub/modules/{}/wake", m_ModuleId), HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{}))
{
@@ -369,13 +355,8 @@ HubStatusSubCmd::Run(const ZenCliOptions& GlobalOptions)
{
ZEN_UNUSED(GlobalOptions);
- m_HostName = ZenCmdBase::ResolveTargetHostSpec(m_HostName);
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve hub host specification", SubOptions().help());
- }
-
- HttpClient Http = ZenCmdBase::CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = HubCommand::Name});
+ HttpClient& Http = Service.Http();
if (!m_ModuleId.empty())
{
diff --git a/src/zen/cmds/info_cmd.cpp b/src/zen/cmds/info_cmd.cpp
index 9faad5691..b147b93c3 100644
--- a/src/zen/cmds/info_cmd.cpp
+++ b/src/zen/cmds/info_cmd.cpp
@@ -2,6 +2,8 @@
#include "info_cmd.h"
+#include "zenserviceclient.h"
+
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/string.h>
@@ -31,14 +33,8 @@ InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Result = Http.Get("/admin/info", HttpClient::Accept(ZenContentType::kJSON)))
{
@@ -46,7 +42,7 @@ InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- Result.ThrowError(fmt::format("Failed getting info from {}", m_HostName));
+ Result.ThrowError(fmt::format("Failed getting info from {}", Service.HostSpec()));
}
}
diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index d31c34fd0..c6a3434f8 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -2,16 +2,21 @@
#include "projectstore_cmd.h"
+#include "zenserviceclient.h"
+
#include <zencore/basicfile.h>
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinaryutil.h>
#include <zencore/compress.h>
+#include <zencore/except_fmt.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
+#include <zencore/logging/broadcastsink.h>
#include <zencore/parallelwork.h>
#include <zencore/process.h>
#include <zencore/scopeguard.h>
+#include <zencore/session.h>
#include <zencore/stream.h>
#include <zencore/timer.h>
#include <zencore/workthreadpool.h>
@@ -21,16 +26,20 @@
#include <zenhttp/httpclientauth.h>
#include <zenhttp/httpcommon.h>
#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildstorageresolve.h>
#include <zenremotestore/builds/buildstorageutil.h>
#include <zenremotestore/builds/jupiterbuildstorage.h>
#include <zenremotestore/jupiter/jupiterhost.h>
-#include <zenremotestore/operationlogoutput.h>
#include <zenremotestore/projectstore/projectstoreoperations.h>
#include <zenremotestore/projectstore/remoteprojectstore.h>
#include <zenremotestore/transferthreadworkers.h>
+#include <zenutil/authutils.h>
+#include <zenutil/logging.h>
+#include <zenutil/progress.h>
+#include <zenutil/sessionsclient.h>
#include <zenutil/workerpools.h>
-#include "../progressbar.h"
+#include "consoleprogress.h"
ZEN_THIRD_PARTY_INCLUDES_START
#include <json11.hpp>
@@ -112,6 +121,26 @@ namespace projectstore_impl {
}
}
+ // `OplogMirrorCommand::Run` uses a latching boolean flag rather than the
+ // SignalCounter above, because it drives a worker pool that aborts on any
+ // interrupt. Kept separate from SignalCallbackHandler so neither interferes
+ // with the other when both are installed in the same process.
+ static std::atomic<bool> MirrorAbortFlag{false};
+
+ static void MirrorSignalCallbackHandler(int SigNum)
+ {
+ if (SigNum == SIGINT)
+ {
+ MirrorAbortFlag.store(true);
+ }
+#if ZEN_PLATFORM_WINDOWS
+ if (SigNum == SIGBREAK)
+ {
+ MirrorAbortFlag.store(true);
+ }
+#endif
+ }
+
void ExecuteAsyncOperation(HttpClient& Http, std::string_view Url, IoBuffer&& Payload, bool PlainProgress)
{
signal(SIGINT, SignalCallbackHandler);
@@ -131,13 +160,16 @@ namespace projectstore_impl {
throw std::runtime_error(fmt::format("invalid job id returned, received '{}'", JobIdText));
}
- ProgressBar ProgressBar(PlainProgress ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty, ""sv);
+ std::unique_ptr<ProgressBase> ProgressOwner(
+ CreateConsoleProgress(PlainProgress ? ConsoleProgressMode::Plain : ConsoleProgressMode::Pretty));
+ std::unique_ptr<ProgressBase::ProgressBar> Bar = ProgressOwner->CreateProgressBar(""sv);
+ std::string ActiveTask;
- auto OuputMessages = [&](CbObjectView StatusObject) {
+ auto OutputMessages = [&](CbObjectView StatusObject) {
CbArrayView Messages = StatusObject["Messages"sv].AsArrayView();
if (Messages.Num() > 0)
{
- ProgressBar.ForceLinebreak();
+ Bar->ForceLinebreak();
for (auto M : Messages)
{
std::string_view Message = M.AsString();
@@ -169,33 +201,36 @@ namespace projectstore_impl {
uint64_t RemainingCount = StatusObject["RemainingCount"sv].AsUInt64();
uint64_t ProgressElapsedTimeMs = StatusObject["ProgressElapsedTimeMs"sv].AsUInt64((uint64_t)-1);
- if (!ProgressBar.IsSameTask(CurrentOp))
+ if (ActiveTask != CurrentOp)
{
- ProgressBar.Finish();
+ Bar->Finish();
+ ActiveTask = "";
}
- if (!ProgressBar.HasActiveTask())
+ if (ActiveTask.empty())
{
- OuputMessages(StatusObject);
+ OutputMessages(StatusObject);
MessagesDone = true;
+ ActiveTask = std::string(CurrentOp);
}
- ProgressBar.UpdateState({.Task = std::string(CurrentOp),
- .Details = std::string(CurrentOpDetails),
- .TotalCount = TotalCount,
- .RemainingCount = RemainingCount,
- .OptionalElapsedTime = ProgressElapsedTimeMs},
- false);
+ Bar->UpdateState({.Task = std::string(CurrentOp),
+ .Details = std::string(CurrentOpDetails),
+ .TotalCount = TotalCount,
+ .RemainingCount = RemainingCount,
+ .OptionalElapsedTime = ProgressElapsedTimeMs},
+ false);
}
if ((Status == "Complete") || (Status == "Aborted"))
{
- ProgressBar.Finish();
+ Bar->Finish();
+ ActiveTask = "";
}
if (!MessagesDone)
{
- OuputMessages(StatusObject);
+ OutputMessages(StatusObject);
}
if (Status == "Complete")
@@ -246,13 +281,13 @@ namespace projectstore_impl {
#endif // ZEN_PLATFORM_WINDOWS
if (HttpClient::Response DeleteResult = Http.Delete(fmt::format("/admin/jobs/{}", JobId)))
{
- ProgressBar.ForceLinebreak();
+ Bar->ForceLinebreak();
ZEN_CONSOLE("Requested cancel...");
Cancelled = true;
}
else
{
- ProgressBar.ForceLinebreak();
+ Bar->ForceLinebreak();
ZEN_CONSOLE("Failed cancelling job {}", DeleteResult);
}
continue;
@@ -530,14 +565,8 @@ DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
m_ProjectName = ResolveProject(Http, m_ProjectName);
if (m_ProjectName.empty())
@@ -569,7 +598,7 @@ DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
m_OplogName = ResolveOplog(Http, m_ProjectName, m_OplogName);
if (m_OplogName.empty())
{
- throw std::runtime_error(fmt::format("Can't find oplog in project '{}'", m_OplogName, m_ProjectName));
+ throw zen::runtime_error("Can't find oplog '{}' in project '{}'", m_OplogName, m_ProjectName);
}
if (m_DryRun)
{
@@ -620,19 +649,13 @@ ProjectInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
if (!m_OplogName.empty() && m_ProjectName.empty())
{
throw OptionParseException("'--project' is required", m_Options.help());
}
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
std::string Url;
if (m_ProjectName.empty())
@@ -709,19 +732,13 @@ CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
if (m_ProjectId.empty())
{
throw OptionParseException("'--project' is required", m_Options.help());
}
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
std::string Url = fmt::format("/prj/{}", m_ProjectId);
@@ -779,20 +796,14 @@ CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
if (m_ProjectId.empty())
{
throw OptionParseException("'--project' is required", m_Options.help());
}
- HttpClient Http = CreateHttpClient(m_HostName);
- m_ProjectId = ResolveProject(Http, m_ProjectId);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
+ m_ProjectId = ResolveProject(Http, m_ProjectId);
if (m_ProjectId.empty())
{
throw std::runtime_error("Project can not be found");
@@ -1010,20 +1021,14 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
m_BoostWorkerMemory = true;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
if (m_ProjectName.empty())
{
throw OptionParseException("'--project' is required", m_Options.help());
}
- HttpClient Http = CreateHttpClient(m_HostName);
- m_ProjectName = ResolveProject(Http, m_ProjectName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
+ m_ProjectName = ResolveProject(Http, m_ProjectName);
if (m_ProjectName.empty())
{
throw std::runtime_error("Project can not be found");
@@ -1114,7 +1119,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
std::string TargetUrlBase = m_ZenUrl;
if (TargetUrlBase.find("://") == std::string::npos)
{
- // Assume https URL
+ // Assume http URL
TargetUrlBase = fmt::format("http://{}", TargetUrlBase);
}
@@ -1303,7 +1308,14 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
std::filesystem::path MetadataPath(m_BuildsMetadataPath);
IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten();
std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
- CbFieldIterator MetaData = LoadCompactBinaryFromJson(Json);
+ std::string JsonError;
+ CbFieldIterator MetaData = LoadCompactBinaryFromJson(Json, JsonError);
+ if (!JsonError.empty())
+ {
+ throw zen::runtime_error("builds metadata file '{}' is malformed. Reason: '{}'",
+ MetadataPath.string(),
+ JsonError);
+ }
Writer.AddBinary("metadata"sv, MetaData.GetBuffer());
}
if (!m_BuildsMetadata.empty())
@@ -1313,7 +1325,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
size_t SplitPos = Pair.find('=');
if (SplitPos == std::string::npos || SplitPos == 0)
{
- throw std::runtime_error(fmt::format("builds metadata key-value pair '{}' is malformed", Pair));
+ throw zen::runtime_error("builds metadata key-value pair '{}' is malformed", Pair);
}
MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1));
return true;
@@ -1323,7 +1335,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
}
}
Writer.EndObject(); // "builds"
- TargetDescription = fmt::format("[builds] {}/{}/{}/{}", m_CloudUrl, m_JupiterNamespace, m_JupiterBucket, m_BuildsId);
+ TargetDescription = fmt::format("[builds] {}/{}/{}/{}", m_BuildsUrl, m_JupiterNamespace, m_JupiterBucket, m_BuildsId);
}
if (!m_ZenUrl.empty())
{
@@ -1517,13 +1529,6 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
m_BoostWorkerMemory = true;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
if (m_ProjectName.empty())
{
throw OptionParseException("'--project' is required", m_Options.help());
@@ -1541,8 +1546,9 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
m_Options.help());
}
- HttpClient Http = CreateHttpClient(m_HostName);
- m_ProjectName = ResolveProject(Http, m_ProjectName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
+ m_ProjectName = ResolveProject(Http, m_ProjectName);
if (m_ProjectName.empty())
{
throw std::runtime_error("Project can not be found");
@@ -1738,7 +1744,11 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
}
}
Writer.EndObject(); // "builds"
- SourceDescription = fmt::format("[builds] {}/{}/{}/{}", m_CloudUrl, m_JupiterNamespace, m_JupiterBucket, m_BuildsId);
+ SourceDescription = fmt::format("[builds] {}/{}/{}/{}",
+ m_BuildsHost.empty() ? m_BuildsOverrideHost : m_BuildsHost,
+ m_JupiterNamespace,
+ m_JupiterBucket,
+ m_BuildsId);
}
if (!m_ZenUrl.empty())
{
@@ -1806,14 +1816,8 @@ SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (m_ProjectName.empty())
{
@@ -1869,14 +1873,8 @@ ProjectStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Result = Http.Get("/stats/prj", HttpClient::Accept(ZenContentType::kJSON)))
{
ZEN_CONSOLE("{}", Result.ToText());
@@ -1922,14 +1920,8 @@ ProjectOpDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char*
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
m_ProjectName = ResolveProject(Http, m_ProjectName);
if (m_ProjectName.empty())
@@ -2038,14 +2030,8 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
m_ProjectName = ResolveProject(Http, m_ProjectName);
if (m_ProjectName.empty())
@@ -2108,16 +2094,59 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
std::unordered_set<std::u8string> FileNames;
std::atomic<uint64_t> WrittenByteCount = 0;
- std::atomic<bool> AbortFlag(false);
+ // Install Ctrl-C handler so SIGINT aborts the worker pool rather than killing
+ // the process. Without this the local AbortFlag would shadow whatever global
+ // handler is installed elsewhere and interrupts would be dropped. RAII so
+ // the previous handler is restored when the function returns or throws.
+ MirrorAbortFlag.store(false);
+ ScopedSignalHandler SigIntGuard(SIGINT, MirrorSignalCallbackHandler);
+#if ZEN_PLATFORM_WINDOWS
+ ScopedSignalHandler SigBreakGuard(SIGBREAK, MirrorSignalCallbackHandler);
+#endif
+ std::atomic<bool>& AbortFlag = MirrorAbortFlag;
Stopwatch WriteStopWatch;
+ // Filenames come from the remote oplog, which may be compromised or untrusted.
+ // Reject anything that could escape the mirror root via an absolute path, drive
+ // letter / UNC / device path prefix, or '..' component before it is joined to
+ // RootPath. Returns nullptr when the filename is safe.
+ auto UnsafeFileNameReason = [](const std::filesystem::path& FileName) -> const char* {
+ if (FileName.empty())
+ {
+ return "filename is empty";
+ }
+ if (FileName.has_root_name())
+ {
+ return "filename has a root name (drive letter, UNC share, or device path)";
+ }
+ if (FileName.has_root_directory())
+ {
+ return "filename is absolute";
+ }
+ for (const std::filesystem::path& Component : FileName)
+ {
+ const std::u8string C = Component.u8string();
+ if (C.empty() || C == u8"..")
+ {
+ return "filename contains a '..' or empty component";
+ }
+ }
+ return nullptr;
+ };
+
auto EmitFilesForDataArray = [&](CbArrayView DataArray) {
for (auto DataIter : DataArray)
{
if (CbObjectView Data = DataIter.AsObjectView())
{
std::filesystem::path FileName(Data["filename"sv].AsU8String());
+ if (const char* Reason = UnsafeFileNameReason(FileName))
+ {
+ ZEN_CONSOLE_ERROR("Rejecting unsafe filename '{}' from remote oplog: {}", FileName.string(), Reason);
+ AbortFlag.store(true);
+ break;
+ }
if (!m_FilenameFilter.empty())
{
std::string FileNameLowerCase = ToLower(FileName.string());
@@ -2162,7 +2191,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
IoBuffer ChunkData =
m_Decompress ? TryDecompress(ChunkResponse.ResponsePayload) : ChunkResponse.ResponsePayload;
- if (!MoveToFile(TargetPath, ChunkData))
+ if (std::error_code MoveEc = MoveToFile(TargetPath, ChunkData); MoveEc)
{
WriteFile(TargetPath, ChunkData);
}
@@ -2196,17 +2225,18 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
ZEN_CONSOLE("Fetched oplog in {}", NiceTimeSpanMs(uint64_t(Response.ElapsedSeconds * 1000.0)));
if (CbObject ResponseObject = Response.AsObject())
{
- std::unique_ptr<ProgressBar> EmitProgressBar;
+ std::unique_ptr<ProgressBase> ProgressOwner2(CreateConsoleProgress(ConsoleProgressMode::Pretty));
+ std::unique_ptr<ProgressBase::ProgressBar> EmitProgressBar;
{
- ProgressBar ParseProgressBar(ProgressBar::Mode::Pretty, "");
- CbArrayView Entries = ResponseObject["entries"sv].AsArrayView();
- uint64_t Remaining = Entries.Num();
+ std::unique_ptr<ProgressBase::ProgressBar> ParseProgressBar = ProgressOwner2->CreateProgressBar("");
+ CbArrayView Entries = ResponseObject["entries"sv].AsArrayView();
+ uint64_t Remaining = Entries.Num();
for (auto EntryIter : Entries)
{
if (!AbortFlag)
{
CbObjectView Entry = EntryIter.AsObjectView();
- ParseProgressBar.UpdateState(
+ ParseProgressBar->UpdateState(
{.Task = "Parsing oplog", .Details = "", .TotalCount = Entries.Num(), .RemainingCount = Remaining},
false);
Remaining--;
@@ -2219,7 +2249,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
}
if (!EmitProgressBar)
{
- EmitProgressBar = std::make_unique<ProgressBar>(ProgressBar::Mode::Pretty, ""sv);
+ EmitProgressBar = ProgressOwner2->CreateProgressBar(""sv);
WriteStopWatch.Reset();
}
@@ -2229,7 +2259,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
++OplogEntryCount;
}
}
- ParseProgressBar.Finish();
+ ParseProgressBar->Finish();
}
WorkRemaining.CountDown();
@@ -2262,7 +2292,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
if (AbortFlag)
{
- throw std::runtime_error("Failed top mirror oplog");
+ throw std::runtime_error("Failed to mirror oplog");
}
}
else
@@ -2306,14 +2336,8 @@ OplogValidateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
m_ProjectName = ResolveProject(Http, m_ProjectName);
if (m_ProjectName.empty())
@@ -2472,7 +2496,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
};
ParseSystemOptions();
- ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
+ ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty;
auto ParseOutputOptions = [&]() {
if (m_Verbose && m_Quiet)
@@ -2494,23 +2518,19 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
if (m_LogProgress)
{
- ProgressMode = ProgressBar::Mode::Log;
+ ProgressMode = ConsoleProgressMode::Log;
}
else if (m_PlainProgress)
{
- ProgressMode = ProgressBar::Mode::Plain;
- }
- else if (m_Verbose)
- {
- ProgressMode = ProgressBar::Mode::Plain;
+ ProgressMode = ConsoleProgressMode::Plain;
}
else if (m_Quiet)
{
- ProgressMode = ProgressBar::Mode::Quiet;
+ ProgressMode = ConsoleProgressMode::Quiet;
}
else
{
- ProgressMode = ProgressBar::Mode::Pretty;
+ ProgressMode = ConsoleProgressMode::Pretty;
}
};
ParseOutputOptions();
@@ -2565,7 +2585,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
m_BoostWorkerMemory = true;
}
- std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode));
TransferThreadWorkers Workers(m_BoostWorkerCount, /*SingleThreaded*/ false);
if (!m_Quiet)
@@ -2594,7 +2614,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
/*Hidden*/ false,
m_Verbose);
- BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*OperationLogOutput,
+ BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(),
ClientSettings,
m_Host,
m_OverrideHost,
@@ -2629,6 +2649,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u},
[&AbortFlag]() { return AbortFlag.load(); });
Storage.CacheHost = ResolveRes.Cache;
+
+ Storage.SetupCacheSession(ResolveRes.Cache.Address, Name, GetSessionId());
}
if (!m_Quiet)
@@ -2677,7 +2699,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
}
ProjectStoreOperationOplogState State(
- *OperationLogOutput,
+ ConsoleLog(),
Storage,
BuildId,
{.IsQuiet = m_Quiet, .IsVerbose = m_Verbose, .ForceDownload = m_ForceDownload, .TempFolderPath = StorageTempPath});
@@ -2704,7 +2726,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
}
std::atomic<bool> PauseFlag;
- ProjectStoreOperationDownloadAttachments Op(*OperationLogOutput,
+ ProjectStoreOperationDownloadAttachments Op(ConsoleLog(),
+ *Progress,
Storage,
AbortFlag,
PauseFlag,
diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h
index 1ba98b39e..41db36139 100644
--- a/src/zen/cmds/projectstore_cmd.h
+++ b/src/zen/cmds/projectstore_cmd.h
@@ -217,7 +217,7 @@ class SnapshotOplogCommand : public ProjectStoreCommand
{
public:
static constexpr char Name[] = "oplog-snapshot";
- static constexpr char Description[] = "Snapshot project store oplog";
+ static constexpr char Description[] = "Copy oplog's loose files on disk into zenserver";
SnapshotOplogCommand();
~SnapshotOplogCommand();
diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp
deleted file mode 100644
index 3bf81a9df..000000000
--- a/src/zen/cmds/rpcreplay_cmd.cpp
+++ /dev/null
@@ -1,486 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "rpcreplay_cmd.h"
-
-#include <zencore/compactbinarybuilder.h>
-#include <zencore/filesystem.h>
-#include <zencore/fmtutils.h>
-#include <zencore/logging.h>
-#include <zencore/process.h>
-#include <zencore/scopeguard.h>
-#include <zencore/session.h>
-#include <zencore/stream.h>
-#include <zencore/timer.h>
-#include <zencore/workthreadpool.h>
-#include <zenhttp/formatters.h>
-#include <zenhttp/httpclient.h>
-#include <zenhttp/httpcommon.h>
-#include <zenhttp/packageformat.h>
-#include <zenutil/rpcrecording.h>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <fmt/format.h>
-#include <gsl/gsl-lite.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
-
-#include <memory>
-
-namespace zen {
-
-using namespace std::literals;
-
-RpcStartRecordingCommand::RpcStartRecordingCommand()
-{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
- m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), "<path>");
-
- m_Options.parse_positional("path");
-}
-
-RpcStartRecordingCommand::~RpcStartRecordingCommand() = default;
-
-void
-RpcStartRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
-{
- ZEN_UNUSED(GlobalOptions, argc, argv);
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- if (m_RecordingPath.empty())
- {
- throw OptionParseException("'--path' is required", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
- if (HttpClient::Response Response =
- Http.Post("/z$/exec$/start-recording"sv, HttpClient::KeyValueMap{}, HttpClient::KeyValueMap({{"path", m_RecordingPath}})))
- {
- ZEN_CONSOLE("{}", Response.ToText());
- }
- else
- {
- Response.ThrowError("Failed to start recording");
- }
-}
-
-////////////////////////////////////////////////////
-
-RpcStopRecordingCommand::RpcStopRecordingCommand()
-{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
-}
-
-RpcStopRecordingCommand::~RpcStopRecordingCommand() = default;
-
-void
-RpcStopRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
-{
- ZEN_UNUSED(GlobalOptions, argc, argv);
-
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
- if (HttpClient::Response Response = Http.Post("/z$/exec$/stop-recording"sv))
- {
- ZEN_CONSOLE("{}", Response.ToText());
- }
- else
- {
- Response.ThrowError("Failed to stop recording");
- }
-}
-
-////////////////////////////////////////////////////
-
-RpcReplayCommand::RpcReplayCommand()
-{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
- m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), "<path>");
- m_Options.add_option("", "", "dry", "Do a dry run", cxxopts::value(m_DryRun), "<enable>");
- m_Options.add_option("",
- "w",
- "numthreads",
- "Number of worker threads per process",
- cxxopts::value(m_ThreadCount)->default_value(fmt::format("{}", GetHardwareConcurrency())),
- "<count>");
- m_Options.add_option("", "", "onhost", "Replay on host, bypassing http/network layer", cxxopts::value(m_OnHost), "<onhost>");
- m_Options.add_option("",
- "",
- "showmethodstats",
- "Show statistics of which RPC methods are used",
- cxxopts::value(m_ShowMethodStats),
- "<showmethodstats>");
- m_Options.add_option("",
- "",
- "offset",
- "Offset into request recording to start replay",
- cxxopts::value(m_Offset)->default_value("0"),
- "<offset>");
- m_Options.add_option("",
- "",
- "stride",
- "Stride for request recording when replaying requests",
- cxxopts::value(m_Stride)->default_value("1"),
- "<stride>");
- m_Options.add_option("", "", "numproc", "Number of worker processes", cxxopts::value(m_ProcessCount)->default_value("1"), "<count>");
- m_Options.add_option("",
- "",
- "forceallowlocalrefs",
- "Force enable local refs in requests",
- cxxopts::value(m_ForceAllowLocalRefs),
- "<enable>");
- m_Options
- .add_option("", "", "disablelocalrefs", "Force disable local refs in requests", cxxopts::value(m_DisableLocalRefs), "<enable>");
- m_Options.add_option("",
- "",
- "forceallowlocalhandlerefs",
- "Force enable local refs as handles in requests",
- cxxopts::value(m_ForceAllowLocalHandleRef),
- "<enable>");
- m_Options.add_option("",
- "",
- "disablelocalhandlerefs",
- "Force disable local refs as handles in requests",
- cxxopts::value(m_DisableLocalHandleRefs),
- "<enable>");
- m_Options.add_option("",
- "",
- "forceallowpartiallocalrefs",
- "Force enable local refs for all sizes",
- cxxopts::value(m_ForceAllowPartialLocalRefs),
- "<enable>");
- m_Options.add_option("",
- "",
- "disablepartiallocalrefs",
- "Force disable local refs for all sizes",
- cxxopts::value(m_DisablePartialLocalRefs),
- "<enable>");
-
- m_Options.parse_positional("path");
-}
-
-RpcReplayCommand::~RpcReplayCommand() = default;
-
-void
-RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
-{
- ZEN_UNUSED(GlobalOptions, argc, argv);
-
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- if (m_RecordingPath.empty())
- {
- throw OptionParseException("'--path' is required", m_Options.help());
- }
-
- if (!IsDir(m_RecordingPath))
- {
- throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath));
- }
-
- m_ThreadCount = Max(m_ThreadCount, 1);
-
- ZEN_CONSOLE("Replay '{}' (start offset {}, stride {}) to '{}', {} threads",
- m_RecordingPath,
- m_Offset,
- m_Stride,
- m_HostName,
- m_ThreadCount);
-
- Stopwatch TotalTimer;
-
- if (m_OnHost)
- {
- HttpClient Http = CreateHttpClient(m_HostName);
- if (HttpClient::Response Response =
- Http.Post("/z$/exec$/replay-recording"sv,
- HttpClient::KeyValueMap{},
- HttpClient::KeyValueMap({{"path", m_RecordingPath}, {"thread-count", fmt::format("{}", m_ThreadCount)}})))
- {
- ZEN_CONSOLE("{}", Response.ToText());
-
- return;
- }
- else
- {
- Response.ThrowError("Failed to start replay");
- }
- }
-
- std::unique_ptr<cache::IRpcRequestReplayer> Replayer = cache::MakeDiskRequestReplayer(m_RecordingPath, true);
- uint64_t EntryCount = Replayer->GetRequestCount();
-
- std::atomic_uint64_t EntryOffset = m_Offset;
- std::atomic_uint64_t BytesSent = 0;
- std::atomic_uint64_t BytesReceived = 0;
-
- Stopwatch Timer;
-
- if (m_ProcessCount > 1)
- {
- std::vector<std::unique_ptr<ProcessHandle>> WorkerProcesses;
- WorkerProcesses.resize(m_ProcessCount);
-
- ProcessMonitor Monitor;
- for (int ProcessIndex = 0; ProcessIndex < m_ProcessCount; ++ProcessIndex)
- {
- std::string CommandLine =
- fmt::format("{} rpc-record-replay --hosturl {} --path \"{}\" --offset {} --stride {} --numthreads {} --numproc {}"sv,
- argv[0],
- m_HostName,
- m_RecordingPath,
- m_Stride == 1 ? 0 : m_Offset + ProcessIndex,
- m_Stride,
- m_ThreadCount,
- 1);
- CreateProcResult Result(CreateProc(std::filesystem::path(std::string(argv[0])), CommandLine));
- WorkerProcesses[ProcessIndex] = std::make_unique<ProcessHandle>();
- WorkerProcesses[ProcessIndex]->Initialize(Result);
- Monitor.AddPid(WorkerProcesses[ProcessIndex]->Pid());
- }
- while (Monitor.IsRunning())
- {
- ZEN_CONSOLE("Waiting for worker processes...");
- Sleep(1000);
- }
- return;
- }
- else
- {
- std::map<std::string, size_t> MethodTypes;
- RwLock MethodTypesLock;
-
- WorkerThreadPool WorkerPool(m_ThreadCount);
-
- Latch WorkLatch(m_ThreadCount);
- for (int WorkerIndex = 0; WorkerIndex < m_ThreadCount; ++WorkerIndex)
- {
- WorkerPool.ScheduleWork(
- [this, &WorkLatch, EntryCount, &EntryOffset, &Replayer, &BytesSent, &BytesReceived, &MethodTypes, &MethodTypesLock]() {
- auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
-
- std::map<std::string, size_t> LocalMethodTypes;
-
- auto ReduceTypes = MakeGuard([&] {
- RwLock::ExclusiveLockScope __(MethodTypesLock);
-
- for (auto& Entry : LocalMethodTypes)
- {
- MethodTypes[Entry.first] += Entry.second;
- }
- });
-
- HttpClient Http = CreateHttpClient(m_HostName);
-
- uint64_t EntryIndex = EntryOffset.fetch_add(m_Stride);
- while (EntryIndex < EntryCount)
- {
- IoBuffer Payload;
- const zen::cache::RecordedRequestInfo RequestInfo = Replayer->GetRequest(EntryIndex, /* out */ Payload);
-
- if (RequestInfo != zen::cache::RecordedRequestInfo::NullRequest)
- {
- CbPackage RequestPackage;
- CbObject Request;
-
- switch (RequestInfo.ContentType)
- {
- case ZenContentType::kCbPackage:
- {
- if (ParsePackageMessageWithLegacyFallback(Payload, RequestPackage))
- {
- Request = RequestPackage.GetObject();
- }
- }
- break;
- case ZenContentType::kCbObject:
- {
- Request = LoadCompactBinaryObject(Payload);
- }
- break;
- }
-
- RpcAcceptOptions OriginalAcceptOptions = static_cast<RpcAcceptOptions>(Request["AcceptFlags"sv].AsUInt16(0u));
- int OriginalProcessPid = Request["Pid"sv].AsInt32(0);
-
- int AdjustedPid = 0;
- RpcAcceptOptions AdjustedAcceptOptions = RpcAcceptOptions::kNone;
-
- if (!m_DisableLocalRefs)
- {
- if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowLocalReferences) ||
- m_ForceAllowLocalRefs)
- {
- AdjustedAcceptOptions |= RpcAcceptOptions::kAllowLocalReferences;
- if (!m_DisablePartialLocalRefs)
- {
- if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowPartialLocalReferences) ||
- m_ForceAllowPartialLocalRefs)
- {
- AdjustedAcceptOptions |= RpcAcceptOptions::kAllowPartialLocalReferences;
- }
- }
- if (!m_DisableLocalHandleRefs)
- {
- if (OriginalProcessPid != 0 || m_ForceAllowLocalHandleRef)
- {
- AdjustedPid = GetCurrentProcessId();
- }
- }
- }
- }
-
- if (m_ShowMethodStats)
- {
- std::string MethodName = std::string(Request["Method"sv].AsString());
- if (auto It = LocalMethodTypes.find(MethodName); It != LocalMethodTypes.end())
- {
- It->second++;
- }
- else
- {
- LocalMethodTypes[MethodName] = 1;
- }
- }
-
- if (OriginalAcceptOptions != AdjustedAcceptOptions || OriginalProcessPid != AdjustedPid)
- {
- CbObjectWriter RequestCopyWriter;
- for (const CbFieldView& Field : Request)
- {
- if (!Field.HasName())
- {
- RequestCopyWriter.AddField(Field);
- continue;
- }
- std::string_view FieldName = Field.GetName();
- if (FieldName == "Pid"sv)
- {
- continue;
- }
- if (FieldName == "AcceptFlags"sv)
- {
- continue;
- }
- RequestCopyWriter.AddField(FieldName, Field);
- }
- if (AdjustedPid != 0)
- {
- RequestCopyWriter.AddInteger("Pid"sv, AdjustedPid);
- }
- if (AdjustedAcceptOptions != RpcAcceptOptions::kNone)
- {
- RequestCopyWriter.AddInteger("AcceptFlags"sv, static_cast<uint16_t>(AdjustedAcceptOptions));
- }
-
- if (RequestInfo.ContentType == ZenContentType::kCbPackage)
- {
- RequestPackage.SetObject(RequestCopyWriter.Save());
- std::vector<IoBuffer> Buffers = FormatPackageMessage(RequestPackage);
- std::vector<SharedBuffer> SharedBuffers(Buffers.begin(), Buffers.end());
- Payload = CompositeBuffer(std::move(SharedBuffers)).Flatten().AsIoBuffer();
- }
- else
- {
- RequestCopyWriter.Finalize();
- Payload = IoBuffer(RequestCopyWriter.GetSaveSize());
- RequestCopyWriter.Save(Payload.GetMutableView());
- }
- }
-
- if (!m_DryRun)
- {
- Http.SetSessionId(RequestInfo.SessionId);
- Payload.SetContentType(RequestInfo.ContentType);
-
- HttpClient::Response Response =
- Http.Post("/z$/$rpc", Payload, {HttpClient::Accept(RequestInfo.AcceptType)});
-
- BytesSent.fetch_add(Payload.GetSize());
- if (!Response)
- {
- ZEN_CONSOLE_ERROR("{}", Response);
- break;
- }
- BytesReceived.fetch_add(Response.DownloadedBytes);
- }
- }
-
- EntryIndex = EntryOffset.fetch_add(m_Stride);
- }
- },
- WorkerThreadPool::EMode::EnableBacklog);
- }
-
- while (!WorkLatch.Wait(1000))
- {
- const uint64_t RequestsTotal = (EntryCount - m_Offset) / m_Stride;
- const uint64_t RequestsRemaining = (EntryCount - EntryOffset.load()) / m_Stride;
-
- ZEN_CONSOLE("[{:3}%] [{}] {} requests, {} remaining (sent {}, received {})",
- (RequestsTotal - RequestsRemaining) * 100 / RequestsTotal,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- RequestsTotal,
- RequestsRemaining,
- NiceBytes(BytesSent.load()),
- NiceBytes(BytesReceived.load()));
- }
-
- if (m_ShowMethodStats)
- {
- for (const auto& It : MethodTypes)
- {
- ZEN_CONSOLE("{:18}: {:10}", It.first, It.second);
- }
- }
- }
-
- const uint64_t RequestsSent = (EntryOffset.load() - m_Offset) / m_Stride;
- const uint64_t ElapsedMS = Timer.GetElapsedTimeMs();
- const uint64_t Sent = BytesSent.load();
- const uint64_t Received = BytesReceived.load();
-
- ZEN_CONSOLE("Processed requests: {} ({}), payloads sent {} ({}), payloads received {} ({}) in {}.\nTotal runtime: {}",
- RequestsSent,
- NiceRate(RequestsSent, ElapsedMS, "req"),
- NiceBytes(Sent),
- NiceByteRate(Sent, ElapsedMS),
- NiceBytes(Received),
- NiceByteRate(Received, ElapsedMS),
- NiceTimeSpanMs(ElapsedMS),
- NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
-}
-
-} // namespace zen
diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h
deleted file mode 100644
index 332a3126c..000000000
--- a/src/zen/cmds/rpcreplay_cmd.h
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include "../zen.h"
-
-namespace zen {
-
-class RpcStartRecordingCommand : public CacheStoreCommand
-{
-public:
- static constexpr char Name[] = "rpc-record-start";
- static constexpr char Description[] = "Starts recording of cache rpc requests on a host";
-
- RpcStartRecordingCommand();
- ~RpcStartRecordingCommand();
-
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
-
-private:
- cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- std::string m_RecordingPath;
-};
-
-class RpcStopRecordingCommand : public CacheStoreCommand
-{
-public:
- static constexpr char Name[] = "rpc-record-stop";
- static constexpr char Description[] = "Stops recording of cache rpc requests on a host";
-
- RpcStopRecordingCommand();
- ~RpcStopRecordingCommand();
-
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
-
-private:
- cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
-};
-
-class RpcReplayCommand : public CacheStoreCommand
-{
-public:
- static constexpr char Name[] = "rpc-record-replay";
- static constexpr char Description[] = "Replays a previously recorded session of rpc requests";
-
- RpcReplayCommand();
- ~RpcReplayCommand();
-
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
-
-private:
- cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- std::string m_RecordingPath;
- bool m_OnHost = false;
- bool m_ShowMethodStats = false;
- int m_ProcessCount = 1;
- int m_ThreadCount = 0;
- uint64_t m_Offset = 0;
- uint64_t m_Stride = 1;
- bool m_ForceAllowLocalRefs = false;
- bool m_DisableLocalRefs = false;
- bool m_ForceAllowLocalHandleRef = false;
- bool m_DisableLocalHandleRefs = false;
- bool m_ForceAllowPartialLocalRefs = false;
- bool m_DisablePartialLocalRefs = false;
- bool m_DryRun = false;
-};
-
-} // namespace zen
diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp
deleted file mode 100644
index ee47eb9f3..000000000
--- a/src/zen/cmds/run_cmd.cpp
+++ /dev/null
@@ -1,194 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "run_cmd.h"
-
-#include <zencore/filesystem.h>
-#include <zencore/fmtutils.h>
-#include <zencore/logging.h>
-#include <zencore/process.h>
-#include <zencore/string.h>
-#include <zencore/timer.h>
-
-using namespace std::literals;
-
-#define ZEN_COLOR_BLACK "\033[0;30m"
-#define ZEN_COLOR_RED "\033[0;31m"
-#define ZEN_COLOR_GREEN "\033[0;32m"
-#define ZEN_COLOR_YELLOW "\033[0;33m"
-#define ZEN_COLOR_BLUE "\033[0;34m"
-#define ZEN_COLOR_MAGENTA "\033[0;35m"
-#define ZEN_COLOR_CYAN "\033[0;36m"
-#define ZEN_COLOR_WHITE "\033[0;37m"
-
-#define ZEN_BRIGHT_COLOR_BLACK "\033[1;30m"
-#define ZEN_BRIGHT_COLOR_RED "\033[1;31m"
-#define ZEN_BRIGHT_COLOR_GREEN "\033[1;32m"
-#define ZEN_BRIGHT_COLOR_YELLOW "\033[1;33m"
-#define ZEN_BRIGHT_COLOR_BLUE "\033[1;34m"
-#define ZEN_BRIGHT_COLOR_MAGENTA "\033[1;35m"
-#define ZEN_BRIGHT_COLOR_CYAN "\033[1;36m"
-#define ZEN_BRIGHT_COLOR_WHITE "\033[1;37m"
-
-#define ZEN_COLOR_RESET "\033[0m"
-
-namespace zen {
-
-RunCommand::RunCommand()
-{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "n", "count", "Number of times to run command", cxxopts::value(m_RunCount), "<count>");
- m_Options.add_option("", "t", "time", "How long to run command(s) for", cxxopts::value(m_RunTime), "<seconds>");
- m_Options.add_option("",
- "",
- "basepath",
- "Where to run command. Each run will execute in its own numbered subdirectory below this directory. Additionally, "
- "stdout will be redirected to a file in the provided directory",
- cxxopts::value(m_BaseDirectory),
- "<path>");
- m_Options.add_option("",
- "",
- "max-dirs",
- "Number of base directories to retain on rotation",
- cxxopts::value(m_MaxBaseDirectoryCount),
- "<count>");
-}
-
-RunCommand::~RunCommand()
-{
-}
-
-void
-RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
-{
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- // Validate arguments
-
- if (GlobalOptions.PassthroughArgV.empty() || GlobalOptions.PassthroughArgV[0].empty())
- throw OptionParseException("No command specified. The command to run is passed in after a double dash ('--') on the command line",
- m_Options.help());
-
- if (m_RunCount < 0)
- throw OptionParseException(fmt::format("'--count' ('{}') is invalid", m_RunCount), m_Options.help());
-
- if (m_RunTime < -1 || m_RunTime == 0)
- throw OptionParseException(fmt::format("'--time' ('{}') is invalid", m_RunTime), m_Options.help());
-
- if (m_MaxBaseDirectoryCount < 0)
- throw OptionParseException(fmt::format("'--max-dirs' ('{}') is invalid", m_MaxBaseDirectoryCount), m_Options.help());
-
- if (m_RunTime > 0 && m_RunCount > 0)
- throw OptionParseException(fmt::format("'--time' ('{}') conflicts with '--count' ('{}') ", m_RunTime, m_RunCount),
- m_Options.help());
-
- if (m_RunCount == 0)
- m_RunCount = 1;
-
- std::filesystem::path BaseDirectory;
-
- if (!m_BaseDirectory.empty())
- {
- BaseDirectory = m_BaseDirectory;
-
- if (m_MaxBaseDirectoryCount)
- {
- RotateDirectories(BaseDirectory, m_MaxBaseDirectoryCount);
- CreateDirectories(BaseDirectory);
- }
- else
- {
- CleanDirectory(BaseDirectory, /*ForceRemoveReadOnlyFiles*/ false);
- }
- }
-
- bool TimedRun = false;
- auto CommandDeadlineTime = std::chrono::system_clock::now();
-
- if (m_RunTime > 0)
- {
- m_RunCount = 1'000'000;
- TimedRun = true;
-
- CommandDeadlineTime += std::chrono::seconds(m_RunTime);
- }
-
- struct RunResults
- {
- int ExitCode = 0;
- std::chrono::duration<long, std::milli> Duration{};
- };
-
- std::vector<RunResults> Results;
- int ErrorCount = 0;
-
- std::filesystem::path ExecutablePath = SearchPathForExecutable(GlobalOptions.PassthroughArgV[0]);
- std::string CommandArguments = GlobalOptions.PassthroughArgs;
-
- for (int i = 0; i < m_RunCount; ++i)
- {
- std::filesystem::path RunDir;
- if (!BaseDirectory.empty())
- {
- RunDir = BaseDirectory / IntNum(i + 1).c_str();
- CreateDirectories(RunDir);
- }
-
- Stopwatch Timer;
-
- CreateProcOptions ProcOptions;
-
- if (!RunDir.empty())
- {
- ProcOptions.WorkingDirectory = &RunDir;
- ProcOptions.StdoutFile = RunDir / "stdout.txt";
- }
-
- fmt::print(ZEN_BRIGHT_COLOR_WHITE "run #{}" ZEN_COLOR_RESET ": {}\n", i + 1, GlobalOptions.PassthroughCommandLine);
-
- ProcessHandle Proc;
- Proc.Initialize(CreateProc(ExecutablePath, GlobalOptions.PassthroughCommandLine, ProcOptions));
- if (!Proc.IsValid())
- {
- throw std::runtime_error(fmt::format("failed to launch '{}'", ExecutablePath));
- }
-
- int ExitCode = Proc.WaitExitCode();
-
- auto EndTime = std::chrono::system_clock::now();
-
- if (ExitCode)
- ++ErrorCount;
-
- Results.emplace_back(RunResults{.ExitCode = ExitCode, .Duration = std::chrono::milliseconds(Timer.GetElapsedTimeMs())});
-
- if (TimedRun)
- {
- if (EndTime >= CommandDeadlineTime)
- {
- m_RunCount = i + 1;
- break;
- }
- }
- }
-
- fmt::print("{:>5} {:>3} {:>6}\n", "run", "rc", "time");
- int i = 0;
- for (const auto& Entry : Results)
- {
- fmt::print("{:5} {:3} {:>6}\n", ++i, Entry.ExitCode, NiceTimeSpanMs(Entry.Duration.count()));
- }
-
- if (ErrorCount)
- {
- fmt::print("run complete ({}/{} failed)\n", ErrorCount, m_RunCount);
- }
- else
- {
- fmt::print("run complete, no error exit code\n", m_RunCount);
- }
-}
-
-} // namespace zen
diff --git a/src/zen/cmds/run_cmd.h b/src/zen/cmds/run_cmd.h
deleted file mode 100644
index 300c08c5b..000000000
--- a/src/zen/cmds/run_cmd.h
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include "../zen.h"
-
-namespace zen {
-
-class RunCommand : public ZenCmdBase
-{
-public:
- static constexpr char Name[] = "run";
- static constexpr char Description[] = "Run command with special options";
-
- RunCommand();
- ~RunCommand();
-
- virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
- virtual cxxopts::Options& Options() override { return m_Options; }
- virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
-
-private:
- cxxopts::Options m_Options{Name, Description};
- int m_RunCount = 0;
- int m_RunTime = -1;
- std::string m_BaseDirectory;
- int m_MaxBaseDirectoryCount = 10;
-};
-
-} // namespace zen
diff --git a/src/zen/cmds/service_cmd.cpp b/src/zen/cmds/service_cmd.cpp
index 37baf5483..c43c4e614 100644
--- a/src/zen/cmds/service_cmd.cpp
+++ b/src/zen/cmds/service_cmd.cpp
@@ -320,7 +320,7 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
throw OptionParseException("'verb' option is required", m_Options.help());
}
- // Parse subcommand permissively — forward unrecognised options to the parent parser.
+ // Parse subcommand permissively - forward unrecognised options to the parent parser.
std::vector<std::string> SubUnmatched;
if (!ParseOptionsPermissive(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data(), SubUnmatched))
{
@@ -500,9 +500,9 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
std::filesystem::path Destination = m_InstallPath / File.filename();
- if (!CopyFile(File, Destination, {.EnableClone = false}))
+ if (std::error_code CopyEc = CopyFile(File, Destination, {.EnableClone = false}); CopyEc)
{
- throw std::runtime_error(fmt::format("Failed to copy '{}' to '{}'", File, Destination));
+ throw std::system_error(CopyEc, fmt::format("Failed to copy '{}' to '{}'", File, Destination));
}
ZEN_INFO("Copied '{}' to '{}'", File, Destination);
diff --git a/src/zen/cmds/trace_cmd.cpp b/src/zen/cmds/trace_cmd.cpp
deleted file mode 100644
index 54c0f080d..000000000
--- a/src/zen/cmds/trace_cmd.cpp
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "trace_cmd.h"
-#include <zencore/logging.h>
-#include <zenhttp/httpclient.h>
-#include <zenhttp/httpcommon.h>
-
-using namespace std::literals;
-
-namespace zen {
-
-TraceCommand::TraceCommand()
-{
- m_Options.add_options()("h,help", "Print help");
- m_Options.add_option("", "u", "hosturl", kHostUrlHelp, cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
- m_Options.add_option("", "s", "stop", "Stop tracing", cxxopts::value(m_Stop)->default_value("false"), "<stop>");
- m_Options.add_option("", "", "host", "Start tracing to host", cxxopts::value(m_TraceHost), "<hostip>");
- m_Options.add_option("", "", "file", "Start tracing to file", cxxopts::value(m_TraceFile), "<filepath>");
-}
-
-TraceCommand::~TraceCommand() = default;
-
-void
-TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
-{
- ZEN_UNUSED(GlobalOptions);
-
- if (!ParseOptions(argc, argv))
- {
- return;
- }
-
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
-
- zen::HttpClient Http = CreateHttpClient(m_HostName);
-
- if (m_Stop)
- {
- if (zen::HttpClient::Response Response = Http.Post("/admin/trace/stop"sv))
- {
- ZEN_CONSOLE("OK: {}", Response.ToText());
- }
- else
- {
- Response.ThrowError("Trace stop failed");
- }
- return;
- }
-
- std::string StartArg;
- if (!m_TraceHost.empty())
- {
- StartArg = fmt::format("host={}", m_TraceHost);
- }
- else if (!m_TraceFile.empty())
- {
- StartArg = fmt::format("file={}", m_TraceFile);
- }
-
- if (!StartArg.empty())
- {
- if (zen::HttpClient::Response Response = Http.Post(fmt::format("/admin/trace/start?{}"sv, StartArg)))
- {
- ZEN_CONSOLE("OK: {}", Response.ToText());
- }
- else
- {
- Response.ThrowError("Trace start failed");
- }
- }
- else
- {
- if (zen::HttpClient::Response Response = Http.Get("/admin/trace"sv))
- {
- ZEN_CONSOLE("OK: {}", Response.ToText());
- }
- else
- {
- Response.ThrowError("Trace status failed");
- }
- }
-}
-
-} // namespace zen
diff --git a/src/zen/cmds/ui_cmd.cpp b/src/zen/cmds/ui_cmd.cpp
index 4846b4d18..53dbb22da 100644
--- a/src/zen/cmds/ui_cmd.cpp
+++ b/src/zen/cmds/ui_cmd.cpp
@@ -2,6 +2,9 @@
#include "ui_cmd.h"
+#include "browser_launcher.h"
+#include "zenserviceclient.h"
+
#include <zencore/except_fmt.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
@@ -9,11 +12,6 @@
#include <zenutil/consoletui.h>
#include <zenutil/zenserverprocess.h>
-#if ZEN_PLATFORM_WINDOWS
-# include <zencore/windows.h>
-# include <shellapi.h>
-#endif
-
namespace zen {
namespace {
@@ -83,40 +81,10 @@ UiCommand::OpenBrowser(std::string_view HostName)
}
}
- bool Success = false;
-
ExtendableStringBuilder<256> FullUrl;
FullUrl << HostName << m_DashboardPath;
-#if ZEN_PLATFORM_WINDOWS
- HINSTANCE Result = ShellExecuteA(nullptr, "open", FullUrl.c_str(), nullptr, nullptr, SW_SHOWNORMAL);
- Success = reinterpret_cast<intptr_t>(Result) > 32;
-#else
- // Validate URL doesn't contain shell metacharacters that could lead to command injection
- std::string_view FullUrlView = FullUrl;
- constexpr std::string_view DangerousChars = ";|&$`\\\"'<>(){}[]!#*?~\n\r";
- if (FullUrlView.find_first_of(DangerousChars) != std::string_view::npos)
- {
- throw OptionParseException(fmt::format("URL contains invalid characters: '{}'", FullUrl), m_Options.help());
- }
-
-# if ZEN_PLATFORM_MAC
- std::string Command = fmt::format("open \"{}\"", FullUrl);
-# elif ZEN_PLATFORM_LINUX
- std::string Command = fmt::format("xdg-open \"{}\"", FullUrl);
-# else
- ZEN_NOT_IMPLEMENTED("Browser launching not implemented on this platform");
-# endif
-
- Success = system(Command.c_str()) == 0;
-#endif
-
- if (!Success)
- {
- throw zen::runtime_error("Failed to launch browser for '{}'", FullUrl);
- }
-
- ZEN_CONSOLE("Web browser launched for '{}' successfully", FullUrl);
+ LaunchBrowser(std::string_view(FullUrl));
}
void
@@ -162,7 +130,7 @@ UiCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
Labels.push_back(fmt::format("(all {} instances)", Servers.size()));
const int32_t Cols = static_cast<int32_t>(TuiConsoleColumns());
- constexpr int32_t kIndicator = 3; // " ▶ " or " " prefix
+ constexpr int32_t kIndicator = 3; // " > " or " " prefix
constexpr int32_t kSeparator = 2; // " " before cmdline
constexpr int32_t kEllipsis = 3; // "..."
@@ -225,17 +193,14 @@ UiCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_HostName = ResolveTargetHostSpec(m_HostName, ServerPort);
}
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
- }
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
- if (IsUnixSocketSpec(m_HostName))
+ if (IsUnixSocketSpec(Service.HostSpec()))
{
throw std::runtime_error("Cannot open browser for a Unix domain socket connection");
}
- OpenBrowser(m_HostName);
+ OpenBrowser(Service.HostSpec());
}
} // namespace zen
diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp
index 809a41bb6..1f23e6819 100644
--- a/src/zen/cmds/up_cmd.cpp
+++ b/src/zen/cmds/up_cmd.cpp
@@ -2,6 +2,7 @@
#include "up_cmd.h"
+#include <zencore/basicfile.h>
#include <zencore/compactbinary.h>
#include <zencore/compactbinaryutil.h>
#include <zencore/filesystem.h>
@@ -12,6 +13,57 @@
namespace zen {
+namespace {
+
+ bool TryShutdownByPid(ZenServerState& Instance, uint32_t Pid, const std::filesystem::path& ProgramBaseDir, bool ForceTerminate)
+ {
+ Instance.Sweep();
+
+ uint16_t DesiredPort = 0;
+ Instance.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
+ if (Entry.Pid.load() == Pid)
+ {
+ DesiredPort = Entry.DesiredListenPort.load();
+ }
+ });
+
+ ZenServerState::ZenServerEntry* Entry = (DesiredPort != 0) ? Instance.Lookup(DesiredPort) : nullptr;
+ if (Entry && Entry->Pid.load() != Pid)
+ {
+ Entry = nullptr;
+ }
+
+ if (Entry)
+ {
+ if (ShutdownZenServer(ConsoleLog(), Instance, Entry, ProgramBaseDir))
+ {
+ return true;
+ }
+ }
+
+ std::error_code Ec;
+ ProcessHandle Proc;
+ Proc.Initialize(int(Pid), Ec);
+ if (!Ec && Proc.IsValid() && !Proc.IsRunning())
+ {
+ return true;
+ }
+
+ if (ForceTerminate && !Ec && Proc.IsValid() && Proc.IsRunning())
+ {
+ ZEN_CONSOLE_WARN("Hard terminating zen process with pid ({})", Pid);
+ if (Proc.Terminate(0))
+ {
+ ZEN_CONSOLE("Terminate complete");
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+} // namespace
+
UpCommand::UpCommand()
{
m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("0"), "<hostport>");
@@ -40,12 +92,19 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
throw OptionParseException("'--show-console' conflicts with '--show-log'", m_Options.help());
}
+ if (m_ProgramBaseDir.empty())
+ {
+ m_ProgramBaseDir = GetRunningExecutablePath().parent_path();
+ }
+ MakeSafeAbsolutePathInPlace(m_ProgramBaseDir);
+
std::optional<int> StartResult = StartupZenServer(ConsoleLog(),
- {.ProgramBaseDir = m_ProgramBaseDir,
- .Port = m_Port,
- .OpenConsole = m_ShowConsole,
- .ShowLog = m_ShowLog,
- .ExtraArgs = GlobalOptions.PassthroughCommandLine});
+ {.ProgramBaseDir = m_ProgramBaseDir,
+ .Port = m_Port,
+ .OpenConsole = m_ShowConsole,
+ .ShowLog = m_ShowLog,
+ .ExtraArgs = GlobalOptions.PassthroughCommandLine,
+ .EnableExecutionHistory = GlobalOptions.EnableExecutionHistory});
if (!StartResult.has_value())
{
ZEN_CONSOLE("Zen server already running");
@@ -80,6 +139,11 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
+ if (!m_DataDir.empty())
+ {
+ MakeSafeAbsolutePathInPlace(m_DataDir);
+ }
+
ZenServerState Instance;
Instance.Initialize();
Instance.Sweep();
@@ -87,9 +151,9 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!m_DataDir.empty())
{
- if (!IsFile(m_DataDir / ".lock"))
+ if (!LockFile::IsHeldLive(m_DataDir / ".lock", /*AttemptCleanup*/ false))
{
- throw std::runtime_error(fmt::format("Lock file does not exist in directory '{}'", m_DataDir));
+ throw std::runtime_error(fmt::format("No live zen server holding lock file in directory '{}'", m_DataDir));
}
CbValidateError ValidateResult = CbValidateError::None;
if (CbObject LockFileObject =
@@ -134,6 +198,13 @@ DownCommand::DownCommand()
m_Options.add_option("", "f", "force", "Force terminate if graceful shutdown fails", cxxopts::value(m_ForceTerminate), "<force>");
m_Options.add_option("", "b", "base-dir", "Parent folder of server executable", cxxopts::value(m_ProgramBaseDir), "<directory>");
m_Options.add_option("", "", "data-dir", "Path to data directory to inspect for running server", cxxopts::value(m_DataDir), "<file>");
+ m_Options.add_option("", "", "pid", "Shut down zen server process by PID", cxxopts::value(m_Pid)->default_value("0"), "<pid>");
+ m_Options.add_option("",
+ "",
+ "executable",
+ "Shut down all zen server processes matching executable path",
+ cxxopts::value(m_ExecutablePath),
+ "<path>");
}
DownCommand::~DownCommand() = default;
@@ -148,16 +219,77 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
+ const bool HasPid = m_Pid != 0;
+ const bool HasExecutable = !m_ExecutablePath.empty();
+ const bool HasDataDir = !m_DataDir.empty();
+ const int SelectorCount = int(m_All) + int(HasPid) + int(HasExecutable) + int(HasDataDir);
+ if (SelectorCount > 1)
+ {
+ throw OptionParseException("--all, --pid, --executable, and --data-dir are mutually exclusive", m_Options.help());
+ }
+
if (m_ProgramBaseDir.empty())
{
std::filesystem::path ExePath = GetRunningExecutablePath();
m_ProgramBaseDir = ExePath.parent_path();
}
+ MakeSafeAbsolutePathInPlace(m_ProgramBaseDir);
+ if (!m_DataDir.empty())
+ {
+ MakeSafeAbsolutePathInPlace(m_DataDir);
+ }
+ if (!m_ExecutablePath.empty() && m_ExecutablePath.has_parent_path())
+ {
+ MakeSafeAbsolutePathInPlace(m_ExecutablePath);
+ }
// Discover executing instances
ZenServerState Instance;
Instance.Initialize();
+ if (HasPid)
+ {
+ if (!TryShutdownByPid(Instance, m_Pid, m_ProgramBaseDir, m_ForceTerminate))
+ {
+ throw std::runtime_error(fmt::format("Failed to shut down zen process with pid {}, use --force to hard terminate", m_Pid));
+ }
+ ZEN_CONSOLE("Zen server with pid {} is down", m_Pid);
+ return;
+ }
+
+ if (HasExecutable)
+ {
+ int ShutdownCount = 0;
+ while (true)
+ {
+ ProcessHandle Proc;
+ std::error_code Ec = FindProcess(m_ExecutablePath, Proc, /*IncludeSelf*/ false);
+ if (Ec)
+ {
+ throw std::system_error(Ec, fmt::format("FindProcess failed for '{}'", m_ExecutablePath));
+ }
+ if (!Proc.IsValid())
+ {
+ break;
+ }
+ const uint32_t Pid = uint32_t(Proc.Pid());
+ if (!TryShutdownByPid(Instance, Pid, m_ProgramBaseDir, m_ForceTerminate))
+ {
+ throw std::runtime_error(fmt::format("Failed to shut down zen process with pid {}, use --force to hard terminate", Pid));
+ }
+ ++ShutdownCount;
+ }
+ if (ShutdownCount == 0)
+ {
+ ZEN_CONSOLE("No zen server processes matching executable '{}'", m_ExecutablePath);
+ }
+ else
+ {
+ ZEN_CONSOLE("Shut down {} zen server instance(s) matching executable '{}'", ShutdownCount, m_ExecutablePath);
+ }
+ return;
+ }
+
if (m_All)
{
struct EntryInfo
@@ -184,15 +316,10 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
int FailCount = 0;
for (const EntryInfo& Info : Entries)
{
- Instance.Sweep();
- ZenServerState::ZenServerEntry* Entry = Instance.Lookup(Info.Port);
- if (Entry && Entry->Pid.load() == Info.Pid)
+ if (!TryShutdownByPid(Instance, Info.Pid, m_ProgramBaseDir, m_ForceTerminate))
{
- if (!ShutdownZenServer(ConsoleLog(), Instance, Entry, m_ProgramBaseDir))
- {
- ZEN_CONSOLE_WARN("Failed to shutdown server on port {} (pid {})", Info.Port, Info.Pid);
- ++FailCount;
- }
+ ZEN_CONSOLE_WARN("Failed to shutdown server on port {} (pid {})", Info.Port, Info.Pid);
+ ++FailCount;
}
}
@@ -207,9 +334,10 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!m_DataDir.empty())
{
- if (!IsFile(m_DataDir / ".lock"))
+ if (!LockFile::IsHeldLive(m_DataDir / ".lock", /*AttemptCleanup*/ true))
{
- throw std::runtime_error(fmt::format("Lock file does not exist in directory '{}'", m_DataDir));
+ ZEN_CONSOLE("No live zen server holding lock file in '{}', nothing to do", m_DataDir);
+ return;
}
CbValidateError ValidateResult = CbValidateError::None;
if (CbObject LockFileObject =
diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h
index f904fe0d9..bcc7d7da4 100644
--- a/src/zen/cmds/up_cmd.h
+++ b/src/zen/cmds/up_cmd.h
@@ -64,10 +64,12 @@ public:
private:
cxxopts::Options m_Options{Name, Description};
uint16_t m_Port = 0;
+ uint32_t m_Pid = 0;
bool m_All = false;
bool m_ForceTerminate = false;
std::filesystem::path m_ProgramBaseDir;
std::filesystem::path m_DataDir;
+ std::filesystem::path m_ExecutablePath;
};
} // namespace zen
diff --git a/src/zen/cmds/version_cmd.cpp b/src/zen/cmds/version_cmd.cpp
index 0948de1bb..19951f862 100644
--- a/src/zen/cmds/version_cmd.cpp
+++ b/src/zen/cmds/version_cmd.cpp
@@ -2,6 +2,8 @@
#include "version_cmd.h"
+#include "zenserviceclient.h"
+
#include <zencore/basicfile.h>
#include <zencore/config.h>
#include <zencore/filesystem.h>
@@ -57,7 +59,9 @@ VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE("Querying host {}", m_HostName);
}
- HttpClient Client = CreateHttpClient(m_HostName, {.Timeout = std::chrono::milliseconds(5000)});
+ ZenServiceClient Service(
+ {.HostSpec = m_HostName, .CommandName = Name, .HttpSettings = {.Timeout = std::chrono::milliseconds(5000)}});
+ HttpClient& Client = Service.Http();
HttpClient::KeyValueMap Parameters;
if (m_DetailedVersion)
diff --git a/src/zen/cmds/vfs_cmd.cpp b/src/zen/cmds/vfs_cmd.cpp
index 29ad8dc7c..c07526789 100644
--- a/src/zen/cmds/vfs_cmd.cpp
+++ b/src/zen/cmds/vfs_cmd.cpp
@@ -2,6 +2,8 @@
#include "vfs_cmd.h"
+#include "zenserviceclient.h"
+
#include <zencore/compactbinarybuilder.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
@@ -40,12 +42,8 @@ VfsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
// Validate arguments
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
- if (m_HostName.empty())
- throw OptionParseException("Unable to resolve server specification", m_Options.help());
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (m_Verb == "mount"sv)
{
diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp
index 10f5ad8e1..d5344fb01 100644
--- a/src/zen/cmds/wipe_cmd.cpp
+++ b/src/zen/cmds/wipe_cmd.cpp
@@ -4,6 +4,7 @@
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
+#include <zencore/iohash.h>
#include <zencore/logging.h>
#include <zencore/parallelwork.h>
#include <zencore/string.h>
@@ -11,7 +12,7 @@
#include <zencore/trace.h>
#include <zenutil/workerpools.h>
-#include "../progressbar.h"
+#include "consoleprogress.h"
#include <signal.h>
@@ -34,13 +35,13 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
namespace wipe_impl {
- static std::atomic<bool> AbortFlag = false;
- static std::atomic<bool> PauseFlag = false;
- static bool IsVerbose = false;
- static bool Quiet = false;
- static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
- const bool SingleThreaded = false;
- bool BoostWorkerThreads = true;
+ static std::atomic<bool> AbortFlag = false;
+ static std::atomic<bool> PauseFlag = false;
+ static bool IsVerbose = false;
+ static bool Quiet = false;
+ static ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty;
+ const bool SingleThreaded = false;
+ bool BoostWorkerThreads = true;
WorkerThreadPool& GetIOWorkerPool()
{
@@ -167,7 +168,8 @@ namespace wipe_impl {
ZEN_TRACE_CPU("CleanDirectory");
Stopwatch Timer;
- ProgressBar Progress(ProgressMode, "Clean Folder");
+ std::unique_ptr<ProgressBase> ProgressOwner(CreateConsoleProgress(ProgressMode));
+ std::unique_ptr<ProgressBase::ProgressBar> Progress = ProgressOwner->CreateProgressBar("Clean Folder");
std::atomic<bool> CleanWipe = true;
std::atomic<uint64_t> DiscoveredItemCount = 0;
@@ -413,7 +415,7 @@ namespace wipe_impl {
GetIOWorkerPool(),
Work.PendingWork());
- Work.Wait(ProgressMode == ProgressBar::Mode::Pretty ? 200 : 5000, [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) {
+ Work.Wait(ProgressOwner->GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) {
ZEN_UNUSED(PendingWork);
if (Quiet)
{
@@ -424,12 +426,12 @@ namespace wipe_impl {
uint64_t Deleted = DeletedItemCount.load();
uint64_t DeletedBytes = DeletedByteCount.load();
uint64_t Discovered = DiscoveredItemCount.load();
- Progress.UpdateState({.Task = "Removing files ",
- .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
- .TotalCount = Discovered,
- .RemainingCount = Discovered - Deleted,
- .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
+ Progress->UpdateState({.Task = "Removing files ",
+ .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+ .TotalCount = Discovered,
+ .RemainingCount = Discovered - Deleted,
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
});
std::vector<std::filesystem::path> DirectoriesToDelete;
@@ -473,22 +475,22 @@ namespace wipe_impl {
}
uint64_t NowMs = Timer.GetElapsedTimeMs();
- if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode))
+ if ((NowMs - LastUpdateTimeMs) >= ProgressOwner->GetProgressUpdateDelayMS())
{
LastUpdateTimeMs = NowMs;
uint64_t Deleted = DeletedItemCount.load();
uint64_t DeletedBytes = DeletedByteCount.load();
uint64_t Discovered = DiscoveredItemCount.load();
- Progress.UpdateState({.Task = "Removing folders",
- .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
- .TotalCount = DirectoriesToDelete.size(),
- .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex},
- false);
+ Progress->UpdateState({.Task = "Removing folders",
+ .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+ .TotalCount = DirectoriesToDelete.size(),
+ .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex},
+ false);
}
}
- Progress.Finish();
+ Progress->Finish();
uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
if (!Quiet)
@@ -536,10 +538,10 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
using namespace wipe_impl;
ZEN_UNUSED(GlobalOptions);
- signal(SIGINT, SignalCallbackHandler);
+ ScopedSignalHandler SigIntGuard(SIGINT, SignalCallbackHandler);
#if ZEN_PLATFORM_WINDOWS
- signal(SIGBREAK, SignalCallbackHandler);
-#endif // ZEN_PLATFORM_WINDOWS
+ ScopedSignalHandler SigBreakGuard(SIGBREAK, SignalCallbackHandler);
+#endif
if (!ParseOptions(argc, argv))
{
@@ -548,7 +550,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
Quiet = m_Quiet;
IsVerbose = m_Verbose;
- ProgressMode = (IsVerbose || m_PlainProgress) ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty;
+ ProgressMode = m_PlainProgress ? ConsoleProgressMode::Plain : ConsoleProgressMode::Pretty;
BoostWorkerThreads = m_BoostWorkerThreads;
MakeSafeAbsolutePathInPlace(m_Directory);
diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp
index 9e49b464e..3ab3b9e04 100644
--- a/src/zen/cmds/workspaces_cmd.cpp
+++ b/src/zen/cmds/workspaces_cmd.cpp
@@ -2,6 +2,8 @@
#include "workspaces_cmd.h"
+#include "zenserviceclient.h"
+
#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
@@ -137,7 +139,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
throw OptionParseException("'verb' option is required", m_Options.help());
}
- // Parse subcommand permissively — forward unrecognised options to the parent parser.
+ // Parse subcommand permissively - forward unrecognised options to the parent parser.
std::vector<std::string> SubUnmatched;
if (!ParseOptionsPermissive(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data(), SubUnmatched))
{
@@ -159,8 +161,6 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
if (m_SystemRootDir.empty())
{
m_SystemRootDir = PickDefaultSystemRootDirectory();
@@ -199,7 +199,8 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (!m_HostName.empty())
{
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result)
{
ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
@@ -271,7 +272,8 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (!m_HostName.empty())
{
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result)
{
ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
@@ -403,7 +405,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
throw OptionParseException("'verb' option is required", m_Options.help());
}
- // Parse subcommand permissively — forward unrecognised options to the parent parser.
+ // Parse subcommand permissively - forward unrecognised options to the parent parser.
std::vector<std::string> SubUnmatched;
if (!ParseOptionsPermissive(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data(), SubUnmatched))
{
@@ -425,8 +427,6 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
return;
}
- m_HostName = ResolveTargetHostSpec(m_HostName);
-
if (m_SystemRootDir.empty())
{
m_SystemRootDir = PickDefaultSystemRootDirectory();
@@ -509,7 +509,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
{
if (!m_HostName.empty())
{
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result)
{
ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
@@ -626,7 +627,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
{
if (!m_HostName.empty())
{
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result)
{
ZEN_CONSOLE_WARN("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
@@ -674,12 +676,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
Params.Entries.insert_or_assign("refresh", ToString(m_Refresh));
}
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", SubOption->help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/files", GetShareIdentityUrl(m_FilesOptions)), {}, Params))
{
ZEN_CONSOLE("{}: {}", Result, Result.ToText());
@@ -707,12 +705,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
Params.Entries.insert_or_assign("refresh", ToString(m_Refresh));
}
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", SubOption->help());
- }
-
- HttpClient Http = CreateHttpClient(m_HostName);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/entries", GetShareIdentityUrl(m_EntriesOptions)), {}, Params))
{
ZEN_CONSOLE("{}: {}", Result, Result.ToText());
@@ -777,18 +771,14 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
if (SubOption == &m_GetChunkOptions)
{
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", SubOption->help());
- }
-
if (m_ChunkId.empty())
{
throw OptionParseException("'--chunk' is required", SubOption->help());
}
- HttpClient Http = CreateHttpClient(m_HostName);
- m_ChunkId = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, std::vector<std::string>{m_ChunkId})[0];
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
+ m_ChunkId = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, std::vector<std::string>{m_ChunkId})[0];
HttpClient::KeyValueMap Params;
if (m_Offset != 0)
@@ -813,11 +803,6 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
if (SubOption == &m_GetChunkBatchOptions)
{
- if (m_HostName.empty())
- {
- throw OptionParseException("Unable to resolve server specification", SubOption->help());
- }
-
if (m_ShareId.empty())
{
throw OptionParseException("'--share' is required", SubOption->help());
@@ -828,8 +813,9 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
throw OptionParseException("'--chunks' is required", SubOption->help());
}
- HttpClient Http = CreateHttpClient(m_HostName);
- m_ChunkIds = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, m_ChunkIds);
+ ZenServiceClient Service({.HostSpec = m_HostName, .CommandName = Name});
+ HttpClient& Http = Service.Http();
+ m_ChunkIds = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, m_ChunkIds);
std::vector<RequestChunkEntry> ChunkRequests;
ChunkRequests.resize(m_ChunkIds.size());