aboutsummaryrefslogtreecommitdiff
path: root/src/zen/cmds
diff options
context:
space:
mode:
authorLiam Mitchell <[email protected]>2026-03-09 19:06:36 -0700
committerLiam Mitchell <[email protected]>2026-03-09 19:06:36 -0700
commitd1abc50ee9d4fb72efc646e17decafea741caa34 (patch)
treee4288e00f2f7ca0391b83d986efcb69d3ba66a83 /src/zen/cmds
parentAllow requests with invalid content-types unless specified in command line or... (diff)
parentupdated chunk–block analyser (#818) (diff)
downloadarchived-zen-d1abc50ee9d4fb72efc646e17decafea741caa34.tar.xz
archived-zen-d1abc50ee9d4fb72efc646e17decafea741caa34.zip
Merge branch 'main' into lm/restrict-content-type
Diffstat (limited to 'src/zen/cmds')
-rw-r--r--src/zen/cmds/admin_cmd.h40
-rw-r--r--src/zen/cmds/bench_cmd.h5
-rw-r--r--src/zen/cmds/builds_cmd.cpp217
-rw-r--r--src/zen/cmds/builds_cmd.h2
-rw-r--r--src/zen/cmds/cache_cmd.h20
-rw-r--r--src/zen/cmds/copy_cmd.h5
-rw-r--r--src/zen/cmds/dedup_cmd.h5
-rw-r--r--src/zen/cmds/exec_cmd.cpp1374
-rw-r--r--src/zen/cmds/exec_cmd.h101
-rw-r--r--src/zen/cmds/info_cmd.h5
-rw-r--r--src/zen/cmds/print_cmd.cpp4
-rw-r--r--src/zen/cmds/print_cmd.h10
-rw-r--r--src/zen/cmds/projectstore_cmd.cpp68
-rw-r--r--src/zen/cmds/projectstore_cmd.h60
-rw-r--r--src/zen/cmds/rpcreplay_cmd.h15
-rw-r--r--src/zen/cmds/run_cmd.h5
-rw-r--r--src/zen/cmds/serve_cmd.h5
-rw-r--r--src/zen/cmds/status_cmd.h5
-rw-r--r--src/zen/cmds/top_cmd.h10
-rw-r--r--src/zen/cmds/trace_cmd.h7
-rw-r--r--src/zen/cmds/ui_cmd.cpp236
-rw-r--r--src/zen/cmds/ui_cmd.h32
-rw-r--r--src/zen/cmds/up_cmd.h15
-rw-r--r--src/zen/cmds/vfs_cmd.h5
-rw-r--r--src/zen/cmds/wipe_cmd.cpp16
-rw-r--r--src/zen/cmds/workspaces_cmd.cpp4
26 files changed, 2100 insertions, 171 deletions
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h
index 87ef8091b..83bcf8893 100644
--- a/src/zen/cmds/admin_cmd.h
+++ b/src/zen/cmds/admin_cmd.h
@@ -13,6 +13,9 @@ namespace zen {
class ScrubCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "scrub";
+ static constexpr char Description[] = "Scrub zen storage (verify data integrity)";
+
ScrubCommand();
~ScrubCommand();
@@ -20,7 +23,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"scrub", "Scrub zen storage"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_DryRun = false;
bool m_NoGc = false;
@@ -33,6 +36,9 @@ private:
class GcCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "gc";
+ static constexpr char Description[] = "Garbage collect zen storage";
+
GcCommand();
~GcCommand();
@@ -40,7 +46,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"gc", "Garbage collect zen storage"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_SmallObjects{false};
bool m_SkipCid{false};
@@ -62,6 +68,9 @@ private:
class GcStatusCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "gc-status";
+ static constexpr char Description[] = "Garbage collect zen storage status check";
+
GcStatusCommand();
~GcStatusCommand();
@@ -69,7 +78,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"gc-status", "Garbage collect zen storage status check"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_Details = false;
};
@@ -77,6 +86,9 @@ private:
class GcStopCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "gc-stop";
+ static constexpr char Description[] = "Request cancel of running garbage collection in zen storage";
+
GcStopCommand();
~GcStopCommand();
@@ -84,7 +96,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"gc-stop", "Request cancel of running garbage collection in zen storage"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
@@ -93,6 +105,9 @@ private:
class JobCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "jobs";
+ static constexpr char Description[] = "Show/cancel zen background jobs";
+
JobCommand();
~JobCommand();
@@ -100,7 +115,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"jobs", "Show/cancel zen background jobs"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::uint64_t m_JobId = 0;
bool m_Cancel = 0;
@@ -111,6 +126,9 @@ private:
class LoggingCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "logs";
+ static constexpr char Description[] = "Show/control zen logging";
+
LoggingCommand();
~LoggingCommand();
@@ -118,7 +136,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"logs", "Show/control zen logging"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_CacheWriteLog;
std::string m_CacheAccessLog;
@@ -133,6 +151,9 @@ private:
class FlushCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "flush";
+ static constexpr char Description[] = "Flush storage";
+
FlushCommand();
~FlushCommand();
@@ -140,7 +161,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"flush", "Flush zen storage"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
@@ -149,6 +170,9 @@ private:
class CopyStateCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "copy-state";
+ static constexpr char Description[] = "Copy zen server disk state";
+
CopyStateCommand();
~CopyStateCommand();
@@ -156,7 +180,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"};
+ cxxopts::Options m_Options{Name, Description};
std::filesystem::path m_DataPath;
std::filesystem::path m_TargetPath;
bool m_SkipLogs = false;
diff --git a/src/zen/cmds/bench_cmd.h b/src/zen/cmds/bench_cmd.h
index ed123be75..7fbf85340 100644
--- a/src/zen/cmds/bench_cmd.h
+++ b/src/zen/cmds/bench_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class BenchCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "bench";
+ static constexpr char Description[] = "Utility command for benchmarking";
+
BenchCommand();
~BenchCommand();
@@ -17,7 +20,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"bench", "Benchmarking utility command"};
+ cxxopts::Options m_Options{Name, Description};
bool m_PurgeStandbyLists = false;
bool m_SingleProcess = false;
};
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index f4edb65ab..b4b4df7c9 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -67,13 +67,11 @@ ZEN_THIRD_PARTY_INCLUDES_END
static const bool DoExtraContentVerify = false;
-#define ZEN_CLOUD_STORAGE "Cloud Storage"
-
namespace zen {
using namespace std::literals;
-namespace {
+namespace builds_impl {
static std::atomic<bool> AbortFlag = false;
static std::atomic<bool> PauseFlag = false;
@@ -270,10 +268,11 @@ namespace {
static bool IsQuiet = false;
static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
-#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \
- if (IsVerbose) \
- { \
- ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \
+#undef ZEN_CONSOLE_VERBOSE
+#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \
+ if (IsVerbose) \
+ { \
+ ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \
}
const std::string DefaultAccessTokenEnvVariableName(
@@ -1467,9 +1466,16 @@ namespace {
ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
}
+ Stopwatch IndexTimer;
+
const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent);
const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
+ if (!IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(Output, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
+ }
+
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount);
BuildsOperationUpdateFolder Updater(
@@ -1588,7 +1594,7 @@ namespace {
}
}
}
- if (Storage.BuildCacheStorage)
+ if (Storage.CacheStorage)
{
if (SB.Size() > 0)
{
@@ -1643,9 +1649,9 @@ namespace {
}
if (Options.PrimeCacheOnly)
{
- if (Storage.BuildCacheStorage)
+ if (Storage.CacheStorage)
{
- Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) {
+ Storage.CacheStorage->Flush(5000, [](intptr_t Remaining) {
if (!IsQuiet)
{
if (Remaining == 0)
@@ -2002,12 +2008,13 @@ namespace {
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
}
-} // namespace
+} // namespace builds_impl
//////////////////////////////////////////////////////////////////////////////////////////////////////
BuildsCommand::BuildsCommand()
{
+ using namespace builds_impl;
m_Options.add_options()("h,help", "Print help");
auto AddSystemOptions = [this](cxxopts::Options& Ops) {
@@ -2648,6 +2655,7 @@ BuildsCommand::~BuildsCommand() = default;
void
BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace builds_impl;
ZEN_UNUSED(GlobalOptions);
signal(SIGINT, SignalCallbackHandler);
@@ -2680,7 +2688,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_SystemRootDir = PickDefaultSystemRootDirectory();
}
- MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
+ MakeSafeAbsolutePathInPlace(m_SystemRootDir);
};
ParseSystemOptions();
@@ -2729,7 +2737,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help());
}
- MakeSafeAbsolutePathÍnPlace(m_StoragePath);
+ MakeSafeAbsolutePathInPlace(m_StoragePath);
};
auto ParseOutputOptions = [&]() {
@@ -2800,8 +2808,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
.Verbose = m_VerboseHttp,
.MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)};
- std::unique_ptr<AuthMgr> Auth;
-
std::string StorageDescription;
std::string CacheDescription;
@@ -2820,44 +2826,47 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BuildStorageResolveResult ResolveRes =
ResolveBuildStorage(*Output, ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose);
- if (!ResolveRes.HostUrl.empty())
+ if (!ResolveRes.Cloud.Address.empty())
{
- ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2;
+ ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2;
Result.BuildStorageHttp =
- std::make_unique<HttpClient>(ResolveRes.HostUrl, ClientSettings, []() { return AbortFlag.load(); });
+ std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); });
- Result.BuildStorage = CreateJupiterBuildStorage(Log(),
+ Result.BuildStorage = CreateJupiterBuildStorage(Log(),
*Result.BuildStorageHttp,
StorageStats,
m_Namespace,
m_Bucket,
m_AllowRedirect,
TempPath / "storage");
- Result.StorageName = ResolveRes.HostName;
+ Result.BuildStorageHost = ResolveRes.Cloud;
+
+ uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0;
- StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
- ResolveRes.HostName,
- (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl),
- Result.BuildStorageHttp->GetSessionId(),
- m_Namespace,
- m_Bucket);
- ;
+ StorageDescription =
+ fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}",
+ ResolveRes.Cloud.Name,
+ (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address),
+ Result.BuildStorageHttp->GetSessionId(),
+ m_Namespace,
+ m_Bucket,
+ NiceLatencyNs(HostLatencyNs));
- if (!ResolveRes.CacheUrl.empty())
+ if (!ResolveRes.Cache.Address.empty())
{
Result.CacheHttp = std::make_unique<HttpClient>(
- ResolveRes.CacheUrl,
+ ResolveRes.Cache.Address,
HttpClientSettings{
.LogCategory = "httpcacheclient",
.ConnectTimeout = std::chrono::milliseconds{3000},
.Timeout = std::chrono::milliseconds{30000},
- .AssumeHttp2 = ResolveRes.CacheAssumeHttp2,
+ .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2,
.AllowResume = true,
.RetryCount = 0,
.Verbose = m_VerboseHttp,
.MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
[]() { return AbortFlag.load(); });
- Result.BuildCacheStorage =
+ Result.CacheStorage =
CreateZenBuildStorageCache(*Result.CacheHttp,
StorageCacheStats,
m_Namespace,
@@ -2865,14 +2874,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
TempPath / "zencache",
BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
: GetTinyWorkerPool(EWorkloadType::Background));
- Result.CacheName = ResolveRes.CacheName;
+ Result.CacheHost = ResolveRes.Cache;
+
+ uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0;
CacheDescription =
- fmt::format("Zen {}{}. SessionId: '{}'",
- ResolveRes.CacheName,
- (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl),
- Result.CacheHttp->GetSessionId());
- ;
+ fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}",
+ ResolveRes.Cache.Name,
+ (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address),
+ Result.CacheHttp->GetSessionId(),
+ NiceLatencyNs(CacheLatencyNs));
+
if (!m_Namespace.empty())
{
CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
@@ -2888,41 +2900,56 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
StorageDescription = fmt::format("folder {}", m_StoragePath);
Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
- Result.StorageName = fmt::format("Disk {}", m_StoragePath.stem());
+
+ Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(),
+ .Name = "Disk",
+ .LatencySec = 1.0 / 100000, // 1 us
+ .Caps = {.MaxRangeCountPerRequest = 2048u}};
if (!m_ZenCacheHost.empty())
{
- Result.CacheHttp = std::make_unique<HttpClient>(
- m_ZenCacheHost,
- HttpClientSettings{
- .LogCategory = "httpcacheclient",
- .ConnectTimeout = std::chrono::milliseconds{3000},
- .Timeout = std::chrono::milliseconds{30000},
- .AssumeHttp2 = m_AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 0,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
- []() { return AbortFlag.load(); });
- Result.BuildCacheStorage =
- CreateZenBuildStorageCache(*Result.CacheHttp,
- StorageCacheStats,
- m_Namespace,
- m_Bucket,
- TempPath / "zencache",
- BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
- : GetTinyWorkerPool(EWorkloadType::Background));
- Result.CacheName = m_ZenCacheHost;
-
- CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'", Result.CacheName, "", Result.CacheHttp->GetSessionId());
- ;
- if (!m_Namespace.empty())
- {
- CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
- }
- if (!m_Bucket.empty())
+ ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp);
+
+ if (TestResult.Success)
{
- CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
+ Result.CacheHttp = std::make_unique<HttpClient>(
+ m_ZenCacheHost,
+ HttpClientSettings{
+ .LogCategory = "httpcacheclient",
+ .ConnectTimeout = std::chrono::milliseconds{3000},
+ .Timeout = std::chrono::milliseconds{30000},
+ .AssumeHttp2 = m_AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 0,
+ .Verbose = m_VerboseHttp,
+ .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
+ []() { return AbortFlag.load(); });
+
+ Result.CacheStorage =
+ CreateZenBuildStorageCache(*Result.CacheHttp,
+ StorageCacheStats,
+ m_Namespace,
+ m_Bucket,
+ TempPath / "zencache",
+ BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
+ : GetTinyWorkerPool(EWorkloadType::Background));
+ Result.CacheHost =
+ BuildStorageResolveResult::Host{.Address = m_ZenCacheHost,
+ .Name = m_ZenCacheHost,
+ .AssumeHttp2 = m_AssumeHttp2,
+ .LatencySec = TestResult.LatencySeconds,
+ .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}};
+
+ CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId());
+
+ if (!m_Namespace.empty())
+ {
+ CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
+ }
+ if (!m_Bucket.empty())
+ {
+ CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
+ }
}
}
}
@@ -2934,7 +2961,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!IsQuiet)
{
ZEN_CONSOLE("Remote: {}", StorageDescription);
- if (!Result.CacheName.empty())
+ if (!Result.CacheHost.Name.empty())
{
ZEN_CONSOLE("Cache : {}", CacheDescription);
}
@@ -2947,7 +2974,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
throw OptionParseException("'--local-path' is required", SubOption->help());
}
- MakeSafeAbsolutePathÍnPlace(m_Path);
+ MakeSafeAbsolutePathInPlace(m_Path);
};
auto ParseFileFilters = [&](std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) {
@@ -3004,7 +3031,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
throw OptionParseException("'--compare-path' is required", SubOption->help());
}
- MakeSafeAbsolutePathÍnPlace(m_DiffPath);
+ MakeSafeAbsolutePathInPlace(m_DiffPath);
};
auto ParseBlobHash = [&]() -> IoHash {
@@ -3016,7 +3043,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_BlobHash.length() != IoHash::StringLength)
{
throw OptionParseException(
- fmt::format("'--blob-hash' ('{}') is malfomed, it must be {} characters long", m_BlobHash, IoHash::StringLength),
+ fmt::format("'--blob-hash' ('{}') is malformed, it must be {} characters long", m_BlobHash, IoHash::StringLength),
SubOption->help());
}
@@ -3033,7 +3060,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_BuildId.length() != Oid::StringLength)
{
throw OptionParseException(
- fmt::format("'--build-id' ('{}') is malfomed, it must be {} characters long", m_BuildId, Oid::StringLength),
+ fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildId, Oid::StringLength),
SubOption->help());
}
else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero)
@@ -3105,7 +3132,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!m_BuildMetadataPath.empty())
{
- MakeSafeAbsolutePathÍnPlace(m_BuildMetadataPath);
+ MakeSafeAbsolutePathInPlace(m_BuildMetadataPath);
IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten();
std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
std::string JsonError;
@@ -3202,8 +3229,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_ListOptions)
{
- MakeSafeAbsolutePathÍnPlace(m_ListQueryPath);
- MakeSafeAbsolutePathÍnPlace(m_ListResultPath);
+ MakeSafeAbsolutePathInPlace(m_ListQueryPath);
+ MakeSafeAbsolutePathInPlace(m_ListResultPath);
if (!m_ListResultPath.empty())
{
@@ -3255,7 +3282,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
CreateDirectories(m_ZenFolderPath);
auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); });
@@ -3294,7 +3321,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_ListBlocksOptions)
{
- MakeSafeAbsolutePathÍnPlace(m_ListResultPath);
+ MakeSafeAbsolutePathInPlace(m_ListResultPath);
if (!m_ListResultPath.empty())
{
@@ -3316,7 +3343,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
CreateDirectories(m_ZenFolderPath);
auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); });
@@ -3387,8 +3414,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
- MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ChunkingCachePath);
CreateDirectories(m_ZenFolderPath);
auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });
@@ -3475,7 +3502,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
"Requests: {}\n"
"Avg Request Time: {}\n"
"Avg I/O Time: {}",
- Storage.StorageName,
+ Storage.BuildStorageHost.Name,
NiceBytes(StorageStats.TotalBytesRead.load()),
NiceBytes(StorageStats.TotalBytesWritten.load()),
StorageStats.TotalRequestCount.load(),
@@ -3532,7 +3559,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = m_Path / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
BuildStorageBase::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
@@ -3632,7 +3659,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = m_Path / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
BuildStorageBase::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
@@ -3652,7 +3679,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::unique_ptr<CbObjectWriter> StructuredOutput;
if (!m_LsResultPath.empty())
{
- MakeSafeAbsolutePathÍnPlace(m_LsResultPath);
+ MakeSafeAbsolutePathInPlace(m_LsResultPath);
StructuredOutput = std::make_unique<CbObjectWriter>();
}
@@ -3696,7 +3723,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ParsePath();
ParseDiffPath();
- MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath);
+ MakeSafeAbsolutePathInPlace(m_ChunkingCachePath);
std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
@@ -3745,7 +3772,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
CreateDirectories(m_ZenFolderPath);
auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });
@@ -3796,12 +3823,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!IsQuiet)
{
- if (Storage.BuildCacheStorage)
+ if (Storage.CacheStorage)
{
- ZEN_CONSOLE("Uploaded {} ({}) blobs",
+ ZEN_CONSOLE("Uploaded {} ({}) blobs to {}",
StorageCacheStats.PutBlobCount.load(),
NiceBytes(StorageCacheStats.PutBlobByteCount),
- Storage.CacheName);
+ Storage.CacheHost.Name);
}
}
@@ -3828,7 +3855,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
CreateDirectories(m_ZenFolderPath);
auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });
@@ -3883,7 +3910,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
CreateDirectories(m_ZenFolderPath);
auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); });
@@ -3933,7 +3960,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = m_Path / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests();
@@ -4083,8 +4110,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_ZenFolderPath = m_Path / ZenFolderName;
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
- MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath);
+ MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ MakeSafeAbsolutePathInPlace(m_ChunkingCachePath);
StorageInstance Storage = CreateBuildStorage(StorageStats,
StorageCacheStats,
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index f5c44ab55..5c80beed5 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -71,7 +71,7 @@ private:
bool m_AppendNewContent = false;
uint8_t m_BlockReuseMinPercentLimit = 85;
bool m_AllowMultiparts = true;
- std::string m_AllowPartialBlockRequests = "mixed";
+ std::string m_AllowPartialBlockRequests = "true";
AuthCommandLineOptions m_AuthOptions;
diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h
index 4dc05bbdc..4f5b90f4d 100644
--- a/src/zen/cmds/cache_cmd.h
+++ b/src/zen/cmds/cache_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class DropCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "drop";
+ static constexpr char Description[] = "Drop cache namespace or bucket";
+
DropCommand();
~DropCommand();
@@ -16,7 +19,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"drop", "Drop cache namespace or bucket"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_NamespaceName;
std::string m_BucketName;
@@ -25,13 +28,16 @@ private:
class CacheInfoCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "cache-info";
+ static constexpr char Description[] = "Info on cache, namespace or bucket";
+
CacheInfoCommand();
~CacheInfoCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"cache-info", "Info on cache, namespace or bucket"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_NamespaceName;
std::string m_SizeInfoBucketNames;
@@ -42,26 +48,32 @@ private:
class CacheStatsCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "cache-stats";
+ static constexpr char Description[] = "Stats on cache";
+
CacheStatsCommand();
~CacheStatsCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"cache-stats", "Stats info on cache"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
class CacheDetailsCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "cache-details";
+ static constexpr char Description[] = "Details on cache";
+
CacheDetailsCommand();
~CacheDetailsCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"cache-details", "Detailed info on cache"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_CSV = false;
bool m_Details = false;
diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h
index e1a5dcb82..757a8e691 100644
--- a/src/zen/cmds/copy_cmd.h
+++ b/src/zen/cmds/copy_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class CopyCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "copy";
+ static constexpr char Description[] = "Copy file(s)";
+
CopyCommand();
~CopyCommand();
@@ -19,7 +22,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"copy", "Copy files efficiently"};
+ cxxopts::Options m_Options{Name, Description};
std::filesystem::path m_CopySource;
std::filesystem::path m_CopyTarget;
bool m_NoClone = false;
diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h
index 5b8387dd2..835b35e92 100644
--- a/src/zen/cmds/dedup_cmd.h
+++ b/src/zen/cmds/dedup_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class DedupCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "dedup";
+ static constexpr char Description[] = "Dedup files";
+
DedupCommand();
~DedupCommand();
@@ -19,7 +22,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"dedup", "Deduplicate files"};
+ cxxopts::Options m_Options{Name, Description};
std::vector<std::string> m_Positional;
std::filesystem::path m_DedupSource;
std::filesystem::path m_DedupTarget;
diff --git a/src/zen/cmds/exec_cmd.cpp b/src/zen/cmds/exec_cmd.cpp
new file mode 100644
index 000000000..42c7119e7
--- /dev/null
+++ b/src/zen/cmds/exec_cmd.cpp
@@ -0,0 +1,1374 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "exec_cmd.h"
+
+#include <zencompute/computeservice.h>
+#include <zencompute/recordingreader.h>
+#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryfile.h>
+#include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryvalue.h>
+#include <zencore/compress.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/scopeguard.h>
+#include <zencore/session.h>
+#include <zencore/stream.h>
+#include <zencore/string.h>
+#include <zencore/system.h>
+#include <zencore/timer.h>
+#include <zenhttp/httpclient.h>
+#include <zenhttp/packageformat.h>
+
+#include <EASTL/hash_map.h>
+#include <EASTL/hash_set.h>
+#include <EASTL/map.h>
+
+using namespace std::literals;
+
+namespace eastl {
+
+template<>
+struct hash<zen::IoHash> : public zen::IoHash::Hasher
+{
+};
+
+} // namespace eastl
+
+#if ZEN_WITH_COMPUTE_SERVICES
+
+namespace zen {
+
+ExecCommand::ExecCommand()
+{
+ m_Options.add_options()("h,help", "Print help");
+ m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName), "<hosturl>");
+ m_Options.add_option("", "", "log", "Action log directory", cxxopts::value(m_RecordingLogPath), "<path>");
+ m_Options.add_option("", "p", "path", "Recording path (directory or .actionlog file)", cxxopts::value(m_RecordingPath), "<path>");
+ m_Options.add_option("", "", "offset", "Recording replay start offset", cxxopts::value(m_Offset), "<offset>");
+ m_Options.add_option("", "", "stride", "Recording replay stride", cxxopts::value(m_Stride), "<stride>");
+ m_Options.add_option("", "", "limit", "Recording replay limit", cxxopts::value(m_Limit), "<limit>");
+ m_Options.add_option("", "", "beacon", "Beacon path", cxxopts::value(m_BeaconPath), "<path>");
+ m_Options.add_option("", "", "orch", "Orchestrator URL for worker discovery", cxxopts::value(m_OrchestratorUrl), "<url>");
+ m_Options.add_option("",
+ "",
+ "mode",
+ "Select execution mode (http,inproc,dump,direct,beacon,buildlog)",
+ cxxopts::value(m_Mode)->default_value("http"),
+ "<string>");
+ m_Options
+ .add_option("", "", "dump-actions", "Dump each action to console as it is dispatched", cxxopts::value(m_DumpActions), "<bool>");
+ m_Options.add_option("", "o", "output", "Save action results to directory", cxxopts::value(m_OutputPath), "<path>");
+ m_Options.add_option("", "", "binary", "Write output as binary packages instead of YAML", cxxopts::value(m_Binary), "<bool>");
+ m_Options.add_option("", "", "quiet", "Quiet mode (less logging)", cxxopts::value(m_Quiet), "<bool>");
+ m_Options.parse_positional("mode");
+}
+
+ExecCommand::~ExecCommand()
+{
+}
+
+void
+ExecCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+ // Configure
+
+ if (!ParseOptions(argc, argv))
+ {
+ return;
+ }
+
+ m_HostName = ResolveTargetHostSpec(m_HostName);
+
+ if (m_RecordingPath.empty())
+ {
+ throw OptionParseException("replay path is required!", m_Options.help());
+ }
+
+ m_VerboseLogging = GlobalOptions.IsVerbose;
+ m_QuietLogging = m_Quiet && !m_VerboseLogging;
+
+ enum ExecMode
+ {
+ kHttp,
+ kDirect,
+ kInproc,
+ kDump,
+ kBeacon,
+ kBuildLog
+ } Mode;
+
+ if (m_Mode == "http"sv)
+ {
+ Mode = kHttp;
+ }
+ else if (m_Mode == "direct"sv)
+ {
+ Mode = kDirect;
+ }
+ else if (m_Mode == "inproc"sv)
+ {
+ Mode = kInproc;
+ }
+ else if (m_Mode == "dump"sv)
+ {
+ Mode = kDump;
+ }
+ else if (m_Mode == "beacon"sv)
+ {
+ Mode = kBeacon;
+ }
+ else if (m_Mode == "buildlog"sv)
+ {
+ Mode = kBuildLog;
+ }
+ else
+ {
+ throw OptionParseException("invalid mode specified!", m_Options.help());
+ }
+
+ // Gather information from recording path
+
+ std::unique_ptr<zen::compute::RecordingReader> Reader;
+ std::unique_ptr<zen::compute::UeRecordingReader> UeReader;
+
+ std::filesystem::path RecordingPath{m_RecordingPath};
+
+ if (!std::filesystem::is_directory(RecordingPath))
+ {
+ throw OptionParseException("replay path should be a directory path!", m_Options.help());
+ }
+ else
+ {
+ if (std::filesystem::is_directory(RecordingPath / "cid"))
+ {
+ Reader = std::make_unique<zen::compute::RecordingReader>(RecordingPath);
+ m_WorkerMap = Reader->ReadWorkers();
+ m_ChunkResolver = Reader.get();
+ m_RecordingReader = Reader.get();
+ }
+ else
+ {
+ UeReader = std::make_unique<zen::compute::UeRecordingReader>(RecordingPath);
+ m_WorkerMap = UeReader->ReadWorkers();
+ m_ChunkResolver = UeReader.get();
+ m_RecordingReader = UeReader.get();
+ }
+ }
+
+ ZEN_CONSOLE("found {} workers, {} action items", m_WorkerMap.size(), m_RecordingReader->GetActionCount());
+
+ for (auto& Kv : m_WorkerMap)
+ {
+ CbObject WorkerDesc = Kv.second.GetObject();
+ const IoHash& WorkerId = Kv.first;
+
+ RegisterWorkerFunctionsFromDescription(WorkerDesc, WorkerId);
+
+ if (m_VerboseLogging)
+ {
+ zen::ExtendableStringBuilder<1024> ObjStr;
+# if 0
+ zen::CompactBinaryToJson(WorkerDesc, ObjStr);
+ ZEN_CONSOLE("worker {}: {}", WorkerId, ObjStr);
+# else
+ zen::CompactBinaryToYaml(WorkerDesc, ObjStr);
+ ZEN_CONSOLE("worker {}:\n{}", WorkerId, ObjStr);
+# endif
+ }
+ }
+
+ if (m_VerboseLogging)
+ {
+ EmitFunctionList(m_FunctionList);
+ }
+
+ // Iterate over work items and dispatch or log them
+
+ int ReturnValue = 0;
+
+ Stopwatch ExecTimer;
+
+ switch (Mode)
+ {
+ case kHttp:
+ // Forward requests to HTTP function service
+ ReturnValue = HttpExecute();
+ break;
+
+ case kDirect:
+ // Not currently supported
+ ReturnValue = LocalMessagingExecute();
+ break;
+
+ case kInproc:
+ // Handle execution in-core (by spawning child processes)
+ ReturnValue = InProcessExecute();
+ break;
+
+ case kDump:
+ // Dump high level information about actions to console
+ ReturnValue = DumpWorkItems();
+ break;
+
+ case kBeacon:
+ ReturnValue = BeaconExecute();
+ break;
+
+ case kBuildLog:
+ ReturnValue = BuildActionsLog();
+ break;
+
+ default:
+ ZEN_ERROR("Unknown operating mode! No work submitted");
+
+ ReturnValue = 1;
+ }
+
+ ZEN_CONSOLE("complete - took {}", NiceTimeSpanMs(ExecTimer.GetElapsedTimeMs()));
+
+ if (!ReturnValue)
+ {
+ ZEN_CONSOLE("all work items completed successfully");
+ }
+ else
+ {
+ ZEN_CONSOLE("some work items failed (code {})", ReturnValue);
+ }
+}
+
+int
+ExecCommand::InProcessExecute()
+{
+ ZEN_ASSERT(m_ChunkResolver);
+ ChunkResolver& Resolver = *m_ChunkResolver;
+
+ zen::compute::ComputeServiceSession ComputeSession(Resolver);
+
+ std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp");
+ ComputeSession.AddLocalRunner(Resolver, TempPath);
+
+ return ExecUsingSession(ComputeSession);
+}
+
+int
+ExecCommand::ExecUsingSession(zen::compute::ComputeServiceSession& ComputeSession)
+{
+ struct JobTracker
+ {
+ public:
+ inline void Insert(int LsnField)
+ {
+ RwLock::ExclusiveLockScope _(Lock);
+ PendingJobs.insert(LsnField);
+ }
+
+ inline bool IsEmpty() const
+ {
+ RwLock::SharedLockScope _(Lock);
+ return PendingJobs.empty();
+ }
+
+ inline void Remove(int CompleteLsn)
+ {
+ RwLock::ExclusiveLockScope _(Lock);
+ PendingJobs.erase(CompleteLsn);
+ }
+
+ inline size_t GetSize() const
+ {
+ RwLock::SharedLockScope _(Lock);
+ return PendingJobs.size();
+ }
+
+ private:
+ mutable RwLock Lock;
+ std::unordered_set<int> PendingJobs;
+ };
+
+ JobTracker PendingJobs;
+
+ struct ActionSummaryEntry
+ {
+ int32_t Lsn = 0;
+ int RecordingIndex = 0;
+ IoHash ActionId;
+ std::string FunctionName;
+ int InputAttachments = 0;
+ uint64_t InputBytes = 0;
+ int OutputAttachments = 0;
+ uint64_t OutputBytes = 0;
+ float WallSeconds = 0.0f;
+ float CpuSeconds = 0.0f;
+ uint64_t SubmittedTicks = 0;
+ uint64_t StartedTicks = 0;
+ std::string ExecutionLocation;
+ };
+
+ std::mutex SummaryLock;
+ std::unordered_map<int32_t, ActionSummaryEntry> SummaryEntries;
+
+ ComputeSession.WaitUntilReady();
+
+ // Register as a client with the orchestrator (best-effort)
+
+ std::string OrchestratorClientId;
+
+ if (!m_OrchestratorUrl.empty())
+ {
+ try
+ {
+ HttpClient OrchestratorClient(m_OrchestratorUrl);
+
+ CbObjectWriter Ann;
+ Ann << "session_id"sv << GetSessionId();
+ Ann << "hostname"sv << std::string_view(GetMachineName());
+
+ CbObjectWriter Meta;
+ Meta << "source"sv
+ << "zen-exec"sv;
+ Ann << "metadata"sv << Meta.Save();
+
+ auto Resp = OrchestratorClient.Post("/orch/clients", Ann.Save());
+ if (Resp.IsSuccess())
+ {
+ OrchestratorClientId = std::string(Resp.AsObject()["id"].AsString());
+ ZEN_CONSOLE_INFO("registered with orchestrator as {}", OrchestratorClientId);
+ }
+ else
+ {
+ ZEN_WARN("failed to register with orchestrator (status {})", static_cast<int>(Resp.StatusCode));
+ }
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_WARN("failed to register with orchestrator: {}", Ex.what());
+ }
+ }
+
+ Stopwatch OrchestratorHeartbeatTimer;
+
+ auto SendOrchestratorHeartbeat = [&] {
+ if (OrchestratorClientId.empty() || OrchestratorHeartbeatTimer.GetElapsedTimeMs() < 30'000)
+ {
+ return;
+ }
+ OrchestratorHeartbeatTimer.Reset();
+ try
+ {
+ HttpClient OrchestratorClient(m_OrchestratorUrl);
+ std::ignore = OrchestratorClient.Post(fmt::format("/orch/clients/{}/update", OrchestratorClientId));
+ }
+ catch (...)
+ {
+ }
+ };
+
+ auto ClientCleanup = MakeGuard([&] {
+ if (!OrchestratorClientId.empty())
+ {
+ try
+ {
+ HttpClient OrchestratorClient(m_OrchestratorUrl);
+ std::ignore = OrchestratorClient.Post(fmt::format("/orch/clients/{}/complete", OrchestratorClientId));
+ }
+ catch (...)
+ {
+ }
+ }
+ });
+
+ // Create a queue to group all actions from this exec session
+
+ CbObjectWriter Metadata;
+ Metadata << "source"sv
+ << "zen-exec"sv;
+
+ auto QueueResult = ComputeSession.CreateQueue("zen-exec", Metadata.Save());
+ const int QueueId = QueueResult.QueueId;
+ if (!QueueId)
+ {
+ ZEN_ERROR("failed to create compute queue");
+ return 1;
+ }
+
+ auto QueueCleanup = MakeGuard([&] { ComputeSession.DeleteQueue(QueueId); });
+
+ if (!m_OutputPath.empty())
+ {
+ zen::CreateDirectories(m_OutputPath);
+ }
+
+ std::atomic<int> IsDraining{0};
+
+ auto DrainCompletedJobs = [&] {
+ if (IsDraining.exchange(1))
+ {
+ return;
+ }
+
+ auto _ = MakeGuard([&] { IsDraining.store(0, std::memory_order_release); });
+
+ CbObjectWriter Cbo;
+ ComputeSession.GetQueueCompleted(QueueId, Cbo);
+
+ if (CbObject Completed = Cbo.Save())
+ {
+ for (auto& It : Completed["completed"sv])
+ {
+ int32_t CompleteLsn = It.AsInt32();
+
+ CbPackage ResultPackage;
+ HttpResponseCode Response = ComputeSession.GetActionResult(CompleteLsn, /* out */ ResultPackage);
+
+ if (Response == HttpResponseCode::OK)
+ {
+ if (!m_OutputPath.empty() && ResultPackage)
+ {
+ int OutputAttachments = 0;
+ uint64_t OutputBytes = 0;
+
+ if (!m_Binary)
+ {
+ // Write the root object as YAML
+ ExtendableStringBuilder<4096> YamlStr;
+ CompactBinaryToYaml(ResultPackage.GetObject(), YamlStr);
+
+ std::string_view Yaml = YamlStr;
+ zen::WriteFile(m_OutputPath / fmt::format("{}.result.yaml", CompleteLsn),
+ IoBuffer(IoBuffer::Clone, Yaml.data(), Yaml.size()));
+
+ // Write decompressed attachments
+ auto Attachments = ResultPackage.GetAttachments();
+
+ if (!Attachments.empty())
+ {
+ std::filesystem::path AttDir = m_OutputPath / fmt::format("{}.result.attachments", CompleteLsn);
+ zen::CreateDirectories(AttDir);
+
+ for (const CbAttachment& Att : Attachments)
+ {
+ ++OutputAttachments;
+
+ IoHash AttHash = Att.GetHash();
+
+ if (Att.IsCompressedBinary())
+ {
+ SharedBuffer Decompressed = Att.AsCompressedBinary().Decompress();
+ OutputBytes += Decompressed.GetSize();
+ zen::WriteFile(AttDir / AttHash.ToHexString(),
+ IoBuffer(IoBuffer::Clone, Decompressed.GetData(), Decompressed.GetSize()));
+ }
+ else
+ {
+ SharedBuffer Binary = Att.AsBinary();
+ OutputBytes += Binary.GetSize();
+ zen::WriteFile(AttDir / AttHash.ToHexString(),
+ IoBuffer(IoBuffer::Clone, Binary.GetData(), Binary.GetSize()));
+ }
+ }
+ }
+
+ if (!m_QuietLogging)
+ {
+ ZEN_CONSOLE("saved result: {}/{}.result.yaml ({} attachments)",
+ m_OutputPath.string(),
+ CompleteLsn,
+ OutputAttachments);
+ }
+ }
+ else
+ {
+ CompositeBuffer Serialized = FormatPackageMessageBuffer(ResultPackage);
+ zen::WriteFile(m_OutputPath / fmt::format("{}.result.pkg", CompleteLsn), std::move(Serialized));
+
+ for (const CbAttachment& Att : ResultPackage.GetAttachments())
+ {
+ ++OutputAttachments;
+ OutputBytes += Att.AsBinary().GetSize();
+ }
+
+ if (!m_QuietLogging)
+ {
+ ZEN_CONSOLE("saved result: {}/{}.result.pkg", m_OutputPath.string(), CompleteLsn);
+ }
+ }
+
+ std::lock_guard Lock(SummaryLock);
+ if (auto It2 = SummaryEntries.find(CompleteLsn); It2 != SummaryEntries.end())
+ {
+ It2->second.OutputAttachments = OutputAttachments;
+ It2->second.OutputBytes = OutputBytes;
+ }
+ }
+
+ PendingJobs.Remove(CompleteLsn);
+
+ ZEN_CONSOLE("completed: LSN {} ({} still pending)", CompleteLsn, PendingJobs.GetSize());
+ }
+ }
+ }
+ };
+
+ // Describe workers
+
+ ZEN_CONSOLE("describing {} workers", m_WorkerMap.size());
+
+ for (auto Kv : m_WorkerMap)
+ {
+ CbPackage WorkerDesc = Kv.second;
+
+ ComputeSession.RegisterWorker(WorkerDesc);
+ }
+
+ // Then submit work items
+
+ int FailedWorkCounter = 0;
+ size_t RemainingWorkItems = m_RecordingReader->GetActionCount();
+ int SubmittedWorkItems = 0;
+
+ ZEN_CONSOLE("submitting {} work items", RemainingWorkItems);
+
+ int OffsetCounter = m_Offset;
+ int StrideCounter = m_Stride;
+
+ auto ShouldSchedule = [&]() -> bool {
+ if (m_Limit && SubmittedWorkItems >= m_Limit)
+ {
+ // Limit reached, ignore
+
+ return false;
+ }
+
+ if (OffsetCounter && OffsetCounter--)
+ {
+ // Still in offset, ignore
+
+ return false;
+ }
+
+ if (--StrideCounter == 0)
+ {
+ StrideCounter = m_Stride;
+
+ return true;
+ }
+
+ return false;
+ };
+
+ int TargetParallelism = 8;
+
+ if (OffsetCounter || StrideCounter || m_Limit)
+ {
+ TargetParallelism = 1;
+ }
+
+ std::atomic<int> RecordingIndex{0};
+
+ m_RecordingReader->IterateActions(
+ [&](CbObject ActionObject, const IoHash& ActionId) {
+ // Enqueue job
+
+ const int CurrentRecordingIndex = RecordingIndex++;
+
+ Stopwatch SubmitTimer;
+
+ const int Priority = 0;
+
+ if (ShouldSchedule())
+ {
+ if (m_VerboseLogging)
+ {
+ int AttachmentCount = 0;
+ uint64_t AttachmentBytes = 0;
+ eastl::hash_set<IoHash> ReferencedChunks;
+
+ ActionObject.IterateAttachments([&](CbFieldView Field) {
+ IoHash AttachData = Field.AsAttachment();
+
+ ReferencedChunks.insert(AttachData);
+ ++AttachmentCount;
+
+ if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachData))
+ {
+ AttachmentBytes += ChunkData.GetSize();
+ }
+ });
+
+ zen::ExtendableStringBuilder<1024> ObjStr;
+ zen::CompactBinaryToJson(ActionObject, ObjStr);
+ ZEN_CONSOLE("work item {} ({} attachments, {} bytes): {}",
+ ActionId,
+ AttachmentCount,
+ NiceBytes(AttachmentBytes),
+ ObjStr);
+ }
+
+ if (m_DumpActions)
+ {
+ int AttachmentCount = 0;
+ uint64_t AttachmentBytes = 0;
+
+ ActionObject.IterateAttachments([&](CbFieldView Field) {
+ IoHash AttachData = Field.AsAttachment();
+
+ ++AttachmentCount;
+
+ if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachData))
+ {
+ AttachmentBytes += ChunkData.GetSize();
+ }
+ });
+
+ zen::ExtendableStringBuilder<1024> ObjStr;
+ zen::CompactBinaryToYaml(ActionObject, ObjStr);
+ ZEN_CONSOLE("action {} ({} attachments, {}):\n{}", ActionId, AttachmentCount, NiceBytes(AttachmentBytes), ObjStr);
+ }
+
+ if (zen::compute::ComputeServiceSession::EnqueueResult EnqueueResult =
+ ComputeSession.EnqueueActionToQueue(QueueId, ActionObject, Priority))
+ {
+ const int32_t LsnField = EnqueueResult.Lsn;
+
+ --RemainingWorkItems;
+ ++SubmittedWorkItems;
+
+ if (!m_QuietLogging)
+ {
+ ZEN_CONSOLE("submitted work item #{} - LSN {} - {}. {} remaining",
+ SubmittedWorkItems,
+ LsnField,
+ NiceTimeSpanMs(SubmitTimer.GetElapsedTimeMs()),
+ RemainingWorkItems);
+ }
+
+ if (!m_OutputPath.empty())
+ {
+ ActionSummaryEntry Entry;
+ Entry.Lsn = LsnField;
+ Entry.RecordingIndex = CurrentRecordingIndex;
+ Entry.ActionId = ActionId;
+ Entry.FunctionName = std::string(ActionObject["Function"sv].AsString());
+
+ if (!m_Binary)
+ {
+ // Write action object as YAML
+ ExtendableStringBuilder<4096> YamlStr;
+ CompactBinaryToYaml(ActionObject, YamlStr);
+
+ std::string_view Yaml = YamlStr;
+ zen::WriteFile(m_OutputPath / fmt::format("{}.action.yaml", LsnField),
+ IoBuffer(IoBuffer::Clone, Yaml.data(), Yaml.size()));
+
+ // Write decompressed input attachments
+ std::filesystem::path AttDir = m_OutputPath / fmt::format("{}.action.attachments", LsnField);
+ bool AttDirCreated = false;
+
+ ActionObject.IterateAttachments([&](CbFieldView Field) {
+ IoHash AttachCid = Field.AsAttachment();
+ ++Entry.InputAttachments;
+
+ if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachCid))
+ {
+ IoHash RawHash;
+ uint64_t RawSize = 0;
+ CompressedBuffer Compressed =
+ CompressedBuffer::FromCompressed(SharedBuffer(ChunkData), RawHash, RawSize);
+ SharedBuffer Decompressed = Compressed.Decompress();
+
+ Entry.InputBytes += Decompressed.GetSize();
+
+ if (!AttDirCreated)
+ {
+ zen::CreateDirectories(AttDir);
+ AttDirCreated = true;
+ }
+
+ zen::WriteFile(AttDir / AttachCid.ToHexString(),
+ IoBuffer(IoBuffer::Clone, Decompressed.GetData(), Decompressed.GetSize()));
+ }
+ });
+
+ if (!m_QuietLogging)
+ {
+ ZEN_CONSOLE("saved action: {}/{}.action.yaml ({} attachments)",
+ m_OutputPath.string(),
+ LsnField,
+ Entry.InputAttachments);
+ }
+ }
+ else
+ {
+ // Build a CbPackage from the action and write as .pkg
+ CbPackage ActionPackage;
+ ActionPackage.SetObject(ActionObject);
+
+ ActionObject.IterateAttachments([&](CbFieldView Field) {
+ IoHash AttachCid = Field.AsAttachment();
+ ++Entry.InputAttachments;
+
+ if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachCid))
+ {
+ IoHash RawHash;
+ uint64_t RawSize = 0;
+ CompressedBuffer Compressed =
+ CompressedBuffer::FromCompressed(SharedBuffer(ChunkData), RawHash, RawSize);
+
+ Entry.InputBytes += ChunkData.GetSize();
+ ActionPackage.AddAttachment(CbAttachment(std::move(Compressed), RawHash));
+ }
+ });
+
+ CompositeBuffer Serialized = FormatPackageMessageBuffer(ActionPackage);
+ zen::WriteFile(m_OutputPath / fmt::format("{}.action.pkg", LsnField), std::move(Serialized));
+
+ if (!m_QuietLogging)
+ {
+ ZEN_CONSOLE("saved action: {}/{}.action.pkg", m_OutputPath.string(), LsnField);
+ }
+ }
+
+ std::lock_guard Lock(SummaryLock);
+ SummaryEntries.emplace(LsnField, std::move(Entry));
+ }
+
+ PendingJobs.Insert(LsnField);
+ }
+ else
+ {
+ if (!m_QuietLogging)
+ {
+ std::string_view FunctionName = ActionObject["Function"sv].AsString();
+ const Guid FunctionVersion = ActionObject["FunctionVersion"sv].AsUuid();
+ const Guid BuildSystemVersion = ActionObject["BuildSystemVersion"sv].AsUuid();
+
+ ZEN_ERROR(
+ "failed to resolve function for work with (Function:{},FunctionVersion:{},BuildSystemVersion:{}). Work "
+ "descriptor "
+ "at: 'file://{}'",
+ std::string(FunctionName),
+ FunctionVersion,
+ BuildSystemVersion,
+ "<null>");
+
+ EmitFunctionListOnce(m_FunctionList);
+ }
+
+ ++FailedWorkCounter;
+ }
+ }
+
+ // Check for completed work
+
+ DrainCompletedJobs();
+ SendOrchestratorHeartbeat();
+ },
+ TargetParallelism);
+
+ // Wait until all pending work is complete
+
+ while (!PendingJobs.IsEmpty())
+ {
+ // TODO: improve this logic
+ zen::Sleep(500);
+
+ DrainCompletedJobs();
+ SendOrchestratorHeartbeat();
+ }
+
+ // Merge timing data from queue history into summary entries
+
+ if (!SummaryEntries.empty())
+ {
+ // RunnerAction::State indices (can't include functionrunner.h from here)
+ constexpr int kStateNew = 0;
+ constexpr int kStatePending = 1;
+ constexpr int kStateRunning = 3;
+ constexpr int kStateCompleted = 4; // first terminal state
+ constexpr int kStateCount = 8;
+
+ for (const auto& HistEntry : ComputeSession.GetQueueHistory(QueueId, 0))
+ {
+ std::lock_guard Lock(SummaryLock);
+ if (auto It = SummaryEntries.find(HistEntry.Lsn); It != SummaryEntries.end())
+ {
+ // Find terminal state timestamp (Completed, Failed, Abandoned, or Cancelled)
+ uint64_t EndTick = 0;
+ for (int S = kStateCompleted; S < kStateCount; ++S)
+ {
+ if (HistEntry.Timestamps[S] != 0)
+ {
+ EndTick = HistEntry.Timestamps[S];
+ break;
+ }
+ }
+ uint64_t StartTick = HistEntry.Timestamps[kStateNew];
+ if (EndTick > StartTick)
+ {
+ It->second.WallSeconds = float(double(EndTick - StartTick) / double(TimeSpan::TicksPerSecond));
+ }
+ It->second.CpuSeconds = HistEntry.CpuSeconds;
+ It->second.SubmittedTicks = HistEntry.Timestamps[kStatePending];
+ It->second.StartedTicks = HistEntry.Timestamps[kStateRunning];
+ It->second.ExecutionLocation = HistEntry.ExecutionLocation;
+ }
+ }
+ }
+
+ // Write summary file if output path is set
+
+ if (!m_OutputPath.empty() && !SummaryEntries.empty())
+ {
+ std::vector<ActionSummaryEntry> Sorted;
+ Sorted.reserve(SummaryEntries.size());
+ for (auto& [_, Entry] : SummaryEntries)
+ {
+ Sorted.push_back(std::move(Entry));
+ }
+
+ std::sort(Sorted.begin(), Sorted.end(), [](const ActionSummaryEntry& A, const ActionSummaryEntry& B) {
+ return A.RecordingIndex < B.RecordingIndex;
+ });
+
+ auto FormatTimestamp = [](uint64_t Ticks) -> std::string {
+ if (Ticks == 0)
+ {
+ return "-";
+ }
+ return DateTime(Ticks).ToString("%H:%M:%S.%s");
+ };
+
+ ExtendableStringBuilder<4096> Summary;
+ Summary.Append(fmt::format("{:<8} {:<8} {:<40} {:<40} {:>8} {:>12} {:>8} {:>12} {:>8} {:>8} {:>12} {:>12} {:<24}\n",
+ "LSN",
+ "Index",
+ "ActionId",
+ "Function",
+ "InAtt",
+ "InBytes",
+ "OutAtt",
+ "OutBytes",
+ "Wall(s)",
+ "CPU(s)",
+ "Submitted",
+ "Started",
+ "Location"));
+ Summary.Append(fmt::format("{:-<8} {:-<8} {:-<40} {:-<40} {:-<8} {:-<12} {:-<8} {:-<12} {:-<8} {:-<8} {:-<12} {:-<12} {:-<24}\n",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ "",
+ ""));
+
+ for (const ActionSummaryEntry& Entry : Sorted)
+ {
+ Summary.Append(fmt::format("{:<8} {:<8} {:<40} {:<40} {:>8} {:>12} {:>8} {:>12} {:>8.2f} {:>8.2f} {:>12} {:>12} {:<24}\n",
+ Entry.Lsn,
+ Entry.RecordingIndex,
+ Entry.ActionId,
+ Entry.FunctionName,
+ Entry.InputAttachments,
+ NiceBytes(Entry.InputBytes),
+ Entry.OutputAttachments,
+ NiceBytes(Entry.OutputBytes),
+ Entry.WallSeconds,
+ Entry.CpuSeconds,
+ FormatTimestamp(Entry.SubmittedTicks),
+ FormatTimestamp(Entry.StartedTicks),
+ Entry.ExecutionLocation));
+ }
+
+ std::filesystem::path SummaryPath = m_OutputPath / "summary.txt";
+ std::string_view SummaryStr = Summary;
+ zen::WriteFile(SummaryPath, IoBuffer(IoBuffer::Clone, SummaryStr.data(), SummaryStr.size()));
+
+ ZEN_CONSOLE("wrote summary to {}", SummaryPath.string());
+
+ if (!m_Binary)
+ {
+ auto EscapeHtml = [](std::string_view Input) -> std::string {
+ std::string Out;
+ Out.reserve(Input.size());
+ for (char C : Input)
+ {
+ switch (C)
+ {
+ case '&':
+ Out += "&amp;";
+ break;
+ case '<':
+ Out += "&lt;";
+ break;
+ case '>':
+ Out += "&gt;";
+ break;
+ case '"':
+ Out += "&quot;";
+ break;
+ case '\'':
+ Out += "&#39;";
+ break;
+ default:
+ Out += C;
+ }
+ }
+ return Out;
+ };
+
+ auto EscapeJson = [](std::string_view Input) -> std::string {
+ std::string Out;
+ Out.reserve(Input.size());
+ for (char C : Input)
+ {
+ switch (C)
+ {
+ case '"':
+ Out += "\\\"";
+ break;
+ case '\\':
+ Out += "\\\\";
+ break;
+ case '\n':
+ Out += "\\n";
+ break;
+ case '\r':
+ Out += "\\r";
+ break;
+ case '\t':
+ Out += "\\t";
+ break;
+ default:
+ if (static_cast<unsigned char>(C) < 0x20)
+ {
+ Out += fmt::format("\\u{:04x}", static_cast<unsigned>(static_cast<unsigned char>(C)));
+ }
+ else
+ {
+ Out += C;
+ }
+ }
+ }
+ return Out;
+ };
+
+ ExtendableStringBuilder<8192> Html;
+
+ Html.Append(std::string_view(R"(<!DOCTYPE html>
+<html><head><meta charset="utf-8"><title>Exec Summary</title>
+<style>
+body{font-family:system-ui,sans-serif;margin:20px;background:#fafafa}
+#container{overflow-y:auto;height:calc(100vh - 120px)}
+table{border-collapse:collapse;width:100%}
+th,td{border:1px solid #ddd;padding:6px 10px;text-align:left;white-space:nowrap}
+th{background:#f0f0f0;cursor:pointer;user-select:none;position:sticky;top:0;z-index:1}
+th:hover{background:#e0e0e0}
+th .arrow{font-size:0.7em;margin-left:4px}
+tr:hover{background:#e8f0fe}
+input{padding:6px 10px;margin-bottom:12px;width:300px;border:1px solid #ccc;border-radius:4px}
+button{padding:6px 14px;margin-left:8px;margin-bottom:12px;border:1px solid #ccc;border-radius:4px;background:#f0f0f0;cursor:pointer}
+button:hover{background:#e0e0e0}
+a{color:#1a73e8;text-decoration:none}
+a:hover{text-decoration:underline}
+.num{text-align:right}
+</style></head><body>
+<h2>Exec Summary</h2>
+<input type="text" id="filter" placeholder="Filter by function name..."><button id="csvBtn">Export CSV</button>
+<div id="container">
+<table><thead><tr>
+<th data-col="0">LSN <span class="arrow"></span></th>
+<th data-col="1">Index <span class="arrow"></span></th>
+<th data-col="2">Action ID <span class="arrow"></span></th>
+<th data-col="3">Function <span class="arrow"></span></th>
+<th data-col="4">In Attachments <span class="arrow"></span></th>
+<th data-col="5">In Bytes <span class="arrow"></span></th>
+<th data-col="6">Out Attachments <span class="arrow"></span></th>
+<th data-col="7">Out Bytes <span class="arrow"></span></th>
+<th data-col="8">Wall(s) <span class="arrow"></span></th>
+<th data-col="9">CPU(s) <span class="arrow"></span></th>
+<th data-col="10">Submitted <span class="arrow"></span></th>
+<th data-col="11">Started <span class="arrow"></span></th>
+<th data-col="12">Location <span class="arrow"></span></th>
+</tr></thead><tbody>
+<tr id="spacerTop"><td colspan="13"></td></tr>
+<tr id="spacerBot"><td colspan="13"></td></tr>
+</tbody></table></div>
+<script>
+const DATA=[
+)"));
+
+ std::string_view ResultExt = ".result.yaml";
+ std::string_view ActionExt = ".action.yaml";
+
+ for (const ActionSummaryEntry& Entry : Sorted)
+ {
+ std::string SafeName = EscapeJson(EscapeHtml(Entry.FunctionName));
+ std::string ActionIdStr = Entry.ActionId.ToHexString();
+ std::string ActionLink;
+ if (!ActionExt.empty())
+ {
+ ActionLink = EscapeJson(fmt::format(" <a href=\"{}{}\">[action]</a>", Entry.Lsn, ActionExt));
+ }
+
+ // Indices: 0=lsn, 1=idx, 2=actionId, 3=fn, 4=inAtt, 5=inBytes, 6=outAtt, 7=outBytes,
+ // 8=wall, 9=cpu, 10=niceBytesIn, 11=niceBytesOut, 12=actionLink,
+ // 13=submittedTicks, 14=startedTicks, 15=submittedDisplay, 16=startedDisplay,
+ // 17=location
+ Html.Append(
+ fmt::format("[{},{},\"{}\",\"{}\",{},{},{},{},{:.6f},{:.6f},\"{}\",\"{}\",\"{}\",{},{},\"{}\",\"{}\",\"{}\"],\n",
+ Entry.Lsn,
+ Entry.RecordingIndex,
+ ActionIdStr,
+ SafeName,
+ Entry.InputAttachments,
+ Entry.InputBytes,
+ Entry.OutputAttachments,
+ Entry.OutputBytes,
+ Entry.WallSeconds,
+ Entry.CpuSeconds,
+ EscapeJson(NiceBytes(Entry.InputBytes)),
+ EscapeJson(NiceBytes(Entry.OutputBytes)),
+ ActionLink,
+ Entry.SubmittedTicks,
+ Entry.StartedTicks,
+ FormatTimestamp(Entry.SubmittedTicks),
+ FormatTimestamp(Entry.StartedTicks),
+ EscapeJson(EscapeHtml(Entry.ExecutionLocation))));
+ }
+
+ Html.Append(fmt::format(R"(];
+const RESULT_EXT="{}";
+)",
+ ResultExt));
+
+ Html.Append(std::string_view(R"JS((function(){
+const ROW_H=33,BUF=20;
+const container=document.getElementById("container");
+const tbody=container.querySelector("tbody");
+const headers=container.querySelectorAll("th");
+const filterInput=document.getElementById("filter");
+const spacerTop=document.getElementById("spacerTop");
+const spacerBot=document.getElementById("spacerBot");
+let view=[...DATA.keys()];
+let sortCol=-1,sortAsc=true;
+const COLS=[
+ {f:0,t:"n"},{f:1,t:"n"},{f:2,t:"s"},{f:3,t:"s"},
+ {f:4,t:"n"},{f:5,t:"n"},{f:6,t:"n"},{f:7,t:"n"},
+ {f:8,t:"n"},{f:9,t:"n"},{f:13,t:"n"},{f:14,t:"n"},{f:17,t:"s"}
+];
+function rowHtml(i){
+ const d=DATA[view[i]];
+ const bg=i%2?' style="background:#f9f9f9"':'';
+ return '<tr'+bg+'>'+
+ '<td class="num"><a href="'+d[0]+RESULT_EXT+'">'+d[0]+'</a></td>'+
+ '<td class="num">'+d[1]+'</td>'+
+ '<td><code>'+d[2]+'</code></td>'+
+ '<td>'+d[3]+d[12]+'</td>'+
+ '<td class="num">'+d[4]+'</td>'+
+ '<td class="num">'+d[10]+'</td>'+
+ '<td class="num">'+d[6]+'</td>'+
+ '<td class="num">'+d[11]+'</td>'+
+ '<td class="num">'+d[8].toFixed(2)+'</td>'+
+ '<td class="num">'+d[9].toFixed(2)+'</td>'+
+ '<td class="num">'+d[15]+'</td>'+
+ '<td class="num">'+d[16]+'</td>'+
+ '<td>'+d[17]+'</td></tr>';
+}
+let lastFirst=-1,lastLast=-1;
+function render(){
+ const scrollTop=container.scrollTop;
+ const viewH=container.clientHeight;
+ let first=Math.floor(scrollTop/ROW_H)-BUF;
+ let last=Math.ceil((scrollTop+viewH)/ROW_H)+BUF;
+ if(first<0) first=0;
+ if(last>=view.length) last=view.length-1;
+ if(first===lastFirst&&last===lastLast) return;
+ lastFirst=first;lastLast=last;
+ const rows=[];
+ for(let i=first;i<=last;i++) rows.push(rowHtml(i));
+ spacerTop.style.height=(first*ROW_H)+'px';
+ spacerBot.style.height=((view.length-1-last)*ROW_H)+'px';
+ const mid=rows.join('');
+ const topTr='<tr id="spacerTop"><td colspan="13" style="border:0;padding:0;height:'+spacerTop.style.height+'"></td></tr>';
+ const botTr='<tr id="spacerBot"><td colspan="13" style="border:0;padding:0;height:'+spacerBot.style.height+'"></td></tr>';
+ tbody.innerHTML=topTr+mid+botTr;
+}
+function applySort(){
+ if(sortCol<0) return;
+ const c=COLS[sortCol];
+ view.sort((a,b)=>{
+ const va=DATA[a][c.f],vb=DATA[b][c.f];
+ if(c.t==="n") return sortAsc?va-vb:vb-va;
+ return sortAsc?(va<vb?-1:va>vb?1:0):(va>vb?-1:va<vb?1:0);
+ });
+}
+function rebuild(){
+ const q=filterInput.value.toLowerCase();
+ view=[];
+ for(let i=0;i<DATA.length;i++){
+ if(!q||DATA[i][3].toLowerCase().includes(q)) view.push(i);
+ }
+ applySort();
+ lastFirst=lastLast=-1;
+ render();
+}
+headers.forEach(th=>{
+ th.addEventListener("click",()=>{
+ const col=parseInt(th.dataset.col);
+ if(sortCol===col){sortAsc=!sortAsc}else{sortCol=col;sortAsc=true}
+ headers.forEach(h=>h.querySelector(".arrow").textContent="");
+ th.querySelector(".arrow").textContent=sortAsc?"\u25B2":"\u25BC";
+ applySort();
+ lastFirst=lastLast=-1;
+ render();
+ });
+});
+filterInput.addEventListener("input",()=>rebuild());
+let ticking=false;
+container.addEventListener("scroll",()=>{
+ if(!ticking){ticking=true;requestAnimationFrame(()=>{render();ticking=false})}
+});
+rebuild();
+document.getElementById("csvBtn").addEventListener("click",()=>{
+ const H=["LSN","Index","Action ID","Function","In Attachments","In Bytes","Out Attachments","Out Bytes","Wall(s)","CPU(s)","Submitted","Started","Location"];
+ const esc=v=>{const s=String(v);return s.includes(',')||s.includes('"')||s.includes('\n')?'"'+s.replace(/"/g,'""')+'"':s};
+ const rows=[H.join(",")];
+ for(let i=0;i<view.length;i++){
+ const d=DATA[view[i]];
+ rows.push([d[0],d[1],d[2],d[3],d[4],d[5],d[6],d[7],d[8],d[9],d[15],d[16],d[17]].map(esc).join(","));
+ }
+ const blob=new Blob([rows.join("\n")],{type:"text/csv"});
+ const a=document.createElement("a");
+ a.href=URL.createObjectURL(blob);
+ a.download="summary.csv";
+ a.click();
+ URL.revokeObjectURL(a.href);
+});
+})();
+</script></body></html>
+)JS"));
+
+ std::filesystem::path HtmlPath = m_OutputPath / "summary.html";
+ std::string_view HtmlStr = Html;
+ zen::WriteFile(HtmlPath, IoBuffer(IoBuffer::Clone, HtmlStr.data(), HtmlStr.size()));
+
+ ZEN_CONSOLE("wrote HTML summary to {}", HtmlPath.string());
+ }
+ }
+
+ if (FailedWorkCounter)
+ {
+ return 1;
+ }
+
+ return 0;
+}
+
+int
+ExecCommand::LocalMessagingExecute()
+{
+ // Non-HTTP work submission path
+
+ // To be reimplemented using final transport
+
+ return 0;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+int
+ExecCommand::HttpExecute()
+{
+ ZEN_ASSERT(m_ChunkResolver);
+ ChunkResolver& Resolver = *m_ChunkResolver;
+
+ std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp");
+
+ zen::compute::ComputeServiceSession ComputeSession(Resolver);
+ ComputeSession.AddRemoteRunner(Resolver, TempPath, m_HostName);
+
+ return ExecUsingSession(ComputeSession);
+}
+
+int
+ExecCommand::BeaconExecute()
+{
+ ZEN_ASSERT(m_ChunkResolver);
+ ChunkResolver& Resolver = *m_ChunkResolver;
+ std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp");
+
+ zen::compute::ComputeServiceSession ComputeSession(Resolver);
+
+ if (!m_OrchestratorUrl.empty())
+ {
+ ZEN_CONSOLE_INFO("using orchestrator at {}", m_OrchestratorUrl);
+ ComputeSession.SetOrchestratorEndpoint(m_OrchestratorUrl);
+ ComputeSession.SetOrchestratorBasePath(TempPath);
+ }
+ else
+ {
+ ZEN_CONSOLE_INFO("note: using hard-coded local worker path");
+ ComputeSession.AddRemoteRunner(Resolver, TempPath, "http://localhost:8558");
+ }
+
+ return ExecUsingSession(ComputeSession);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+ExecCommand::RegisterWorkerFunctionsFromDescription(const CbObject& WorkerDesc, const IoHash& WorkerId)
+{
+ const Guid WorkerBuildSystemVersion = WorkerDesc["buildsystem_version"sv].AsUuid();
+
+ for (auto& Item : WorkerDesc["functions"sv])
+ {
+ CbObjectView Function = Item.AsObjectView();
+
+ std::string_view FunctionName = Function["name"sv].AsString();
+ const Guid FunctionVersion = Function["version"sv].AsUuid();
+
+ m_FunctionList.emplace_back(FunctionDefinition{.FunctionName = std::string{FunctionName},
+ .FunctionVersion = FunctionVersion,
+ .BuildSystemVersion = WorkerBuildSystemVersion,
+ .WorkerId = WorkerId});
+ }
+}
+
+void
+ExecCommand::EmitFunctionListOnce(const std::vector<FunctionDefinition>& FunctionList)
+{
+ if (m_FunctionListEmittedOnce == false)
+ {
+ EmitFunctionList(FunctionList);
+
+ m_FunctionListEmittedOnce = true;
+ }
+}
+
+int
+ExecCommand::DumpWorkItems()
+{
+ std::atomic<int> EmittedCount{0};
+
+ eastl::hash_map<IoHash, uint64_t> SeenAttachments; // Attachment CID -> count of references
+
+ m_RecordingReader->IterateActions(
+ [&](CbObject ActionObject, const IoHash& ActionId) {
+ eastl::hash_map<IoHash, CompressedBuffer> Attachments;
+
+ uint64_t AttachmentBytes = 0;
+ uint64_t UncompressedAttachmentBytes = 0;
+
+ ActionObject.IterateAttachments([&](const zen::CbFieldView AttachmentField) {
+ const IoHash AttachmentCid = AttachmentField.GetValue().AsHash();
+ IoBuffer AttachmentData = m_ChunkResolver->FindChunkByCid(AttachmentCid);
+ IoHash RawHash;
+ uint64_t RawSize = 0;
+ CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize);
+ Attachments[AttachmentCid] = CompressedData;
+
+ AttachmentBytes += CompressedData.GetCompressedSize();
+ UncompressedAttachmentBytes += CompressedData.DecodeRawSize();
+
+ if (auto [Iter, Inserted] = SeenAttachments.insert({AttachmentCid, 1}); !Inserted)
+ {
+ ++Iter->second;
+ }
+ });
+
+ zen::ExtendableStringBuilder<1024> ObjStr;
+
+# if 0
+ zen::CompactBinaryToJson(ActionObject, ObjStr);
+ ZEN_CONSOLE("work item {} ({} attachments): {}", ActionId, Attachments.size(), ObjStr);
+# else
+ zen::CompactBinaryToYaml(ActionObject, ObjStr);
+ ZEN_CONSOLE("work item {} ({} attachments, {}->{} bytes):\n{}",
+ ActionId,
+ Attachments.size(),
+ AttachmentBytes,
+ UncompressedAttachmentBytes,
+ ObjStr);
+# endif
+
+ ++EmittedCount;
+ },
+ 1);
+
+ ZEN_CONSOLE("emitted: {} actions", EmittedCount.load());
+
+ eastl::map<uint64_t, std::vector<IoHash>> ReferenceHistogram;
+
+ for (const auto& [K, V] : SeenAttachments)
+ {
+ if (V > 1)
+ {
+ ReferenceHistogram[V].push_back(K);
+ }
+ }
+
+ for (const auto& [RefCount, Cids] : ReferenceHistogram)
+ {
+ ZEN_CONSOLE("{} attachments with {} references", Cids.size(), RefCount);
+ }
+
+ return 0;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+int
+ExecCommand::BuildActionsLog()
+{
+ ZEN_ASSERT(m_ChunkResolver);
+ ChunkResolver& Resolver = *m_ChunkResolver;
+
+ if (m_RecordingPath.empty())
+ {
+ throw OptionParseException("need to specify recording path", m_Options.help());
+ }
+
+ if (std::filesystem::exists(m_RecordingLogPath))
+ {
+ throw OptionParseException(fmt::format("recording log directory '{}' already exists!", m_RecordingLogPath), m_Options.help());
+ }
+
+ ZEN_NOT_IMPLEMENTED("build log generation not implemented yet!");
+
+ std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp");
+
+ zen::compute::ComputeServiceSession ComputeSession(Resolver);
+ ComputeSession.StartRecording(Resolver, m_RecordingLogPath);
+
+ return ExecUsingSession(ComputeSession);
+}
+
+void
+ExecCommand::EmitFunctionList(const std::vector<FunctionDefinition>& FunctionList)
+{
+ ZEN_CONSOLE("=== Known functions:\n===========================");
+
+ ZEN_CONSOLE("{:30} {:36} {:36} {}", "function", "version", "build system", "worker id");
+
+ for (const FunctionDefinition& Func : FunctionList)
+ {
+ ZEN_CONSOLE("{:30} {:36} {:36} {}", Func.FunctionName, Func.FunctionVersion, Func.BuildSystemVersion, Func.WorkerId);
+ }
+
+ ZEN_CONSOLE("===========================");
+}
+
+} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/src/zen/cmds/exec_cmd.h b/src/zen/cmds/exec_cmd.h
new file mode 100644
index 000000000..6311354c0
--- /dev/null
+++ b/src/zen/cmds/exec_cmd.h
@@ -0,0 +1,101 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+#include <zencompute/recordingreader.h>
+#include <zencore/compactbinarypackage.h>
+#include <zencore/guid.h>
+#include <zencore/iohash.h>
+
+#include <filesystem>
+#include <functional>
+#include <unordered_map>
+
+namespace zen {
+class CbPackage;
+class CbObject;
+struct IoHash;
+class ChunkResolver;
+} // namespace zen
+
+#if ZEN_WITH_COMPUTE_SERVICES
+
+namespace zen::compute {
+class ComputeServiceSession;
+}
+
+namespace zen {
+
+/**
+ * Zen CLI command for executing functions from a recording
+ *
+ * Mostly for testing and debugging purposes
+ */
+
+class ExecCommand : public ZenCmdBase
+{
+public:
+ ExecCommand();
+ ~ExecCommand();
+
+ static constexpr char Name[] = "exec";
+ static constexpr char Description[] = "Execute functions from a recording";
+
+ virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+ virtual cxxopts::Options& Options() override { return m_Options; }
+
+private:
+ cxxopts::Options m_Options{Name, Description};
+ std::string m_HostName;
+ std::string m_OrchestratorUrl;
+ std::filesystem::path m_BeaconPath;
+ std::filesystem::path m_RecordingPath;
+ std::filesystem::path m_RecordingLogPath;
+ int m_Offset = 0;
+ int m_Stride = 1;
+ int m_Limit = 0;
+ bool m_Quiet = false;
+ std::string m_Mode{"http"};
+ std::filesystem::path m_OutputPath;
+ bool m_Binary = false;
+
+ struct FunctionDefinition
+ {
+ std::string FunctionName;
+ zen::Guid FunctionVersion;
+ zen::Guid BuildSystemVersion;
+ zen::IoHash WorkerId;
+ };
+
+ bool m_FunctionListEmittedOnce = false;
+ void EmitFunctionListOnce(const std::vector<FunctionDefinition>& FunctionList);
+ void EmitFunctionList(const std::vector<FunctionDefinition>& FunctionList);
+
+ std::unordered_map<zen::IoHash, zen::CbPackage> m_WorkerMap;
+ std::vector<FunctionDefinition> m_FunctionList;
+ bool m_VerboseLogging = false;
+ bool m_QuietLogging = false;
+ bool m_DumpActions = false;
+
+ zen::ChunkResolver* m_ChunkResolver = nullptr;
+ zen::compute::RecordingReaderBase* m_RecordingReader = nullptr;
+
+ void RegisterWorkerFunctionsFromDescription(const zen::CbObject& WorkerDesc, const zen::IoHash& WorkerId);
+
+ int ExecUsingSession(zen::compute::ComputeServiceSession& ComputeSession);
+
+ // Execution modes
+
+ int DumpWorkItems();
+ int HttpExecute();
+ int InProcessExecute();
+ int LocalMessagingExecute();
+ int BeaconExecute();
+ int BuildActionsLog();
+};
+
+} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/src/zen/cmds/info_cmd.h b/src/zen/cmds/info_cmd.h
index 231565bfd..dc108b8a2 100644
--- a/src/zen/cmds/info_cmd.h
+++ b/src/zen/cmds/info_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class InfoCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "info";
+ static constexpr char Description[] = "Show high level Zen server information";
+
InfoCommand();
~InfoCommand();
@@ -17,7 +20,7 @@ public:
// virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"info", "Show high level zen store information"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp
index 030cc8b66..c6b250fdf 100644
--- a/src/zen/cmds/print_cmd.cpp
+++ b/src/zen/cmds/print_cmd.cpp
@@ -84,7 +84,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- MakeSafeAbsolutePathÍnPlace(m_Filename);
+ MakeSafeAbsolutePathInPlace(m_Filename);
Fc = ReadFile(m_Filename);
}
@@ -244,7 +244,7 @@ PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
if (m_Filename.empty())
throw OptionParseException("'--source' is required", m_Options.help());
- MakeSafeAbsolutePathÍnPlace(m_Filename);
+ MakeSafeAbsolutePathInPlace(m_Filename);
FileContents Fc = ReadFile(m_Filename);
IoBuffer Data = Fc.Flatten();
CbPackage Package;
diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h
index 6c1529b7c..f4a97e218 100644
--- a/src/zen/cmds/print_cmd.h
+++ b/src/zen/cmds/print_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class PrintCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "print";
+ static constexpr char Description[] = "Print compact binary object";
+
PrintCommand();
~PrintCommand();
@@ -19,7 +22,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"print", "Print compact binary object"};
+ cxxopts::Options m_Options{Name, Description};
std::filesystem::path m_Filename;
bool m_ShowCbObjectTypeInfo = false;
};
@@ -29,6 +32,9 @@ private:
class PrintPackageCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "printpackage";
+ static constexpr char Description[] = "Print compact binary package";
+
PrintPackageCommand();
~PrintPackageCommand();
@@ -37,7 +43,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"printpkg", "Print compact binary package"};
+ cxxopts::Options m_Options{Name, Description};
std::filesystem::path m_Filename;
bool m_ShowCbObjectTypeInfo = false;
};
diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index 519b68126..db931e49a 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -41,12 +41,10 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
-namespace {
+namespace projectstore_impl {
using namespace std::literals;
-#define ZEN_CLOUD_STORAGE "Cloud Storage"
-
void WriteAuthOptions(CbObjectWriter& Writer,
std::string_view JupiterOpenIdProvider,
std::string_view JupiterAccessToken,
@@ -500,7 +498,7 @@ namespace {
return {};
}
-} // namespace
+} // namespace projectstore_impl
///////////////////////////////////////
@@ -522,6 +520,7 @@ DropProjectCommand::~DropProjectCommand()
void
DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -611,6 +610,7 @@ ProjectInfoCommand::~ProjectInfoCommand()
void
ProjectInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -697,6 +697,7 @@ CreateProjectCommand::~CreateProjectCommand() = default;
void
CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
using namespace std::literals;
@@ -766,6 +767,7 @@ CreateOplogCommand::~CreateOplogCommand() = default;
void
CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
using namespace std::literals;
@@ -990,6 +992,7 @@ ExportOplogCommand::~ExportOplogCommand()
void
ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
using namespace std::literals;
ZEN_UNUSED(GlobalOptions);
@@ -1470,6 +1473,20 @@ ImportOplogCommand::ImportOplogCommand()
"Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive",
cxxopts::value(m_BoostWorkers),
"<boostworkermemory>");
+ m_Options.add_option(
+ "",
+ "",
+ "allow-partial-block-requests",
+ "Allow request for partial chunk blocks.\n"
+ " false = only full block requests allowed\n"
+ " mixed = multiple partial block ranges requests per block allowed to zen cache, single partial block range "
+ "request per block to host\n"
+ " zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests "
+ "allowed to host\n"
+ " true = multiple partial block ranges requests per block allowed to zen cache and host\n"
+ "Defaults to 'mixed'.",
+ cxxopts::value(m_AllowPartialBlockRequests),
+ "<allowpartialblockrequests>");
m_Options.parse_positional({"project", "oplog", "gcpath"});
m_Options.positional_help("[<projectid> <oplogid> [<gcpath>]]");
@@ -1482,6 +1499,7 @@ ImportOplogCommand::~ImportOplogCommand()
void
ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
using namespace std::literals;
ZEN_UNUSED(GlobalOptions);
@@ -1514,6 +1532,13 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
throw OptionParseException("'--oplog' is required", m_Options.help());
}
+ EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests);
+ if (Mode == EPartialBlockRequestMode::Invalid)
+ {
+ throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests),
+ m_Options.help());
+ }
+
HttpClient Http(m_HostName);
m_ProjectName = ResolveProject(Http, m_ProjectName);
if (m_ProjectName.empty())
@@ -1651,6 +1676,9 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
{
Writer.AddBool("boostworkermemory"sv, true);
}
+
+ Writer.AddString("partialblockrequestmode", m_AllowPartialBlockRequests);
+
if (!m_FileDirectoryPath.empty())
{
Writer.BeginObject("file"sv);
@@ -1766,6 +1794,7 @@ SnapshotOplogCommand::~SnapshotOplogCommand()
void
SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
using namespace std::literals;
ZEN_UNUSED(GlobalOptions);
@@ -1830,6 +1859,7 @@ ProjectStatsCommand::~ProjectStatsCommand()
void
ProjectStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -1882,6 +1912,7 @@ ProjectOpDetailsCommand::~ProjectOpDetailsCommand()
void
ProjectOpDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -1997,6 +2028,7 @@ OplogMirrorCommand::~OplogMirrorCommand()
void
OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -2264,6 +2296,7 @@ OplogValidateCommand::~OplogValidateCommand()
void
OplogValidateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -2415,6 +2448,7 @@ OplogDownloadCommand::~OplogDownloadCommand()
void
OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -2432,7 +2466,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
{
m_SystemRootDir = PickDefaultSystemRootDirectory();
}
- MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
+ MakeSafeAbsolutePathInPlace(m_SystemRootDir);
};
ParseSystemOptions();
@@ -2570,36 +2604,37 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
StorageInstance Storage;
- ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2;
+ ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2;
ClientSettings.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u;
- Storage.BuildStorageHttp = std::make_unique<HttpClient>(ResolveRes.HostUrl, ClientSettings);
+ Storage.BuildStorageHttp = std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings);
+ Storage.BuildStorageHost = ResolveRes.Cloud;
BuildStorageCache::Statistics StorageCacheStats;
std::atomic<bool> AbortFlag(false);
- if (!ResolveRes.CacheUrl.empty())
+ if (!ResolveRes.Cache.Address.empty())
{
Storage.CacheHttp = std::make_unique<HttpClient>(
- ResolveRes.CacheUrl,
+ ResolveRes.Cache.Address,
HttpClientSettings{
.LogCategory = "httpcacheclient",
.ConnectTimeout = std::chrono::milliseconds{3000},
.Timeout = std::chrono::milliseconds{30000},
- .AssumeHttp2 = ResolveRes.CacheAssumeHttp2,
+ .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2,
.AllowResume = true,
.RetryCount = 0,
.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u},
[&AbortFlag]() { return AbortFlag.load(); });
- Storage.CacheName = ResolveRes.CacheName;
+ Storage.CacheHost = ResolveRes.Cache;
}
if (!m_Quiet)
{
std::string StorageDescription =
fmt::format("Cloud {}{}. SessionId {}. Namespace '{}', Bucket '{}'",
- ResolveRes.HostName,
- (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl),
+ ResolveRes.Cloud.Name,
+ (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address),
Storage.BuildStorageHttp->GetSessionId(),
m_Namespace,
m_Bucket);
@@ -2610,8 +2645,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
{
std::string CacheDescription =
fmt::format("Zen {}{}. SessionId {}. Namespace '{}', Bucket '{}'",
- ResolveRes.CacheName,
- (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl),
+ ResolveRes.Cache.Name,
+ (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address),
Storage.CacheHttp->GetSessionId(),
m_Namespace,
m_Bucket);
@@ -2627,11 +2662,10 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
Storage.BuildStorage =
CreateJupiterBuildStorage(Log(), *Storage.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, m_AllowRedirect, StorageTempPath);
- Storage.StorageName = ResolveRes.HostName;
if (Storage.CacheHttp)
{
- Storage.BuildCacheStorage = CreateZenBuildStorageCache(
+ Storage.CacheStorage = CreateZenBuildStorageCache(
*Storage.CacheHttp,
StorageCacheStats,
m_Namespace,
diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h
index 56ef858f5..1ba98b39e 100644
--- a/src/zen/cmds/projectstore_cmd.h
+++ b/src/zen/cmds/projectstore_cmd.h
@@ -16,6 +16,9 @@ class ProjectStoreCommand : public ZenCmdBase
class DropProjectCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-drop";
+ static constexpr char Description[] = "Drop project or project oplog";
+
DropProjectCommand();
~DropProjectCommand();
@@ -23,7 +26,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-drop", "Drop project or project oplog"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -33,13 +36,16 @@ private:
class ProjectInfoCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-info";
+ static constexpr char Description[] = "Info on project or project oplog";
+
ProjectInfoCommand();
~ProjectInfoCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-info", "Info on project or project oplog"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -48,6 +54,9 @@ private:
class CreateProjectCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-create";
+ static constexpr char Description[] = "Create a project";
+
CreateProjectCommand();
~CreateProjectCommand();
@@ -55,7 +64,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-create", "Create project, the project must not already exist."};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectId;
std::string m_RootDir;
@@ -68,6 +77,9 @@ private:
class CreateOplogCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-create";
+ static constexpr char Description[] = "Create a project oplog";
+
CreateOplogCommand();
~CreateOplogCommand();
@@ -75,7 +87,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-create", "Create oplog in an existing project, the oplog must not already exist."};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectId;
std::string m_OplogId;
@@ -86,6 +98,9 @@ private:
class ExportOplogCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-export";
+ static constexpr char Description[] = "Export project store oplog";
+
ExportOplogCommand();
~ExportOplogCommand();
@@ -93,8 +108,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-export",
- "Export project store oplog to cloud (--cloud), file system (--file) or other Zen instance (--zen)"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -145,6 +159,9 @@ private:
class ImportOplogCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-import";
+ static constexpr char Description[] = "Import project store oplog";
+
ImportOplogCommand();
~ImportOplogCommand();
@@ -152,8 +169,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-import",
- "Import project store oplog from cloud (--cloud), file system (--file) or other Zen instance (--zen)"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -193,19 +209,23 @@ private:
bool m_BoostWorkerCount = false;
bool m_BoostWorkerMemory = false;
bool m_BoostWorkers = false;
+
+ std::string m_AllowPartialBlockRequests = "true";
};
class SnapshotOplogCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-snapshot";
+ static constexpr char Description[] = "Snapshot project store oplog";
+
SnapshotOplogCommand();
~SnapshotOplogCommand();
-
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-snapshot", "Snapshot external file references in project store oplog into zen"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -214,26 +234,32 @@ private:
class ProjectStatsCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-stats";
+ static constexpr char Description[] = "Stats on project store";
+
ProjectStatsCommand();
~ProjectStatsCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-stats", "Stats info on project store"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
class ProjectOpDetailsCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-op-details";
+ static constexpr char Description[] = "Detail info on ops inside a project store oplog";
+
ProjectOpDetailsCommand();
~ProjectOpDetailsCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-op-details", "Detail info on ops inside a project store oplog"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_Details = false;
bool m_OpDetails = false;
@@ -247,13 +273,16 @@ private:
class OplogMirrorCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-mirror";
+ static constexpr char Description[] = "Mirror project store oplog to file system";
+
OplogMirrorCommand();
~OplogMirrorCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-mirror", "Mirror oplog to file system"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -268,13 +297,16 @@ private:
class OplogValidateCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-validate";
+ static constexpr char Description[] = "Validate oplog for missing references";
+
OplogValidateCommand();
~OplogValidateCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-validate", "Validate oplog for missing references"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h
index a6363b614..332a3126c 100644
--- a/src/zen/cmds/rpcreplay_cmd.h
+++ b/src/zen/cmds/rpcreplay_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class RpcStartRecordingCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "rpc-record-start";
+ static constexpr char Description[] = "Starts recording of cache rpc requests on a host";
+
RpcStartRecordingCommand();
~RpcStartRecordingCommand();
@@ -16,7 +19,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"rpc-record-start", "Starts recording of cache rpc requests on a host"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_RecordingPath;
};
@@ -24,6 +27,9 @@ private:
class RpcStopRecordingCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "rpc-record-stop";
+ static constexpr char Description[] = "Stops recording of cache rpc requests on a host";
+
RpcStopRecordingCommand();
~RpcStopRecordingCommand();
@@ -31,13 +37,16 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"rpc-record-stop", "Stops recording of cache rpc requests on a host"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
class RpcReplayCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "rpc-record-replay";
+ static constexpr char Description[] = "Replays a previously recorded session of rpc requests";
+
RpcReplayCommand();
~RpcReplayCommand();
@@ -45,7 +54,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"rpc-record-replay", "Replays a previously recorded session of cache rpc requests to a target host"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_RecordingPath;
bool m_OnHost = false;
diff --git a/src/zen/cmds/run_cmd.h b/src/zen/cmds/run_cmd.h
index 570a2e63a..300c08c5b 100644
--- a/src/zen/cmds/run_cmd.h
+++ b/src/zen/cmds/run_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class RunCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "run";
+ static constexpr char Description[] = "Run command with special options";
+
RunCommand();
~RunCommand();
@@ -17,7 +20,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"run", "Run executable"};
+ cxxopts::Options m_Options{Name, Description};
int m_RunCount = 0;
int m_RunTime = -1;
std::string m_BaseDirectory;
diff --git a/src/zen/cmds/serve_cmd.h b/src/zen/cmds/serve_cmd.h
index ac74981f2..22f430948 100644
--- a/src/zen/cmds/serve_cmd.h
+++ b/src/zen/cmds/serve_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class ServeCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "serve";
+ static constexpr char Description[] = "Serve files from a directory";
+
ServeCommand();
~ServeCommand();
@@ -18,7 +21,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"serve", "Serve files from a tree"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h
index dc103a196..df5df3066 100644
--- a/src/zen/cmds/status_cmd.h
+++ b/src/zen/cmds/status_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class StatusCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "status";
+ static constexpr char Description[] = "Show zen status";
+
StatusCommand();
~StatusCommand();
@@ -20,7 +23,7 @@ public:
private:
int GetLockFileEffectivePort() const;
- cxxopts::Options m_Options{"status", "Show zen status"};
+ cxxopts::Options m_Options{Name, Description};
uint16_t m_Port = 0;
std::filesystem::path m_DataDir;
};
diff --git a/src/zen/cmds/top_cmd.h b/src/zen/cmds/top_cmd.h
index 74167ecfd..aeb196558 100644
--- a/src/zen/cmds/top_cmd.h
+++ b/src/zen/cmds/top_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class TopCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "top";
+ static constexpr char Description[] = "Monitor zen server activity";
+
TopCommand();
~TopCommand();
@@ -16,12 +19,15 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"top", "Show dev UI"};
+ cxxopts::Options m_Options{Name, Description};
};
class PsCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "ps";
+ static constexpr char Description[] = "Enumerate running zen server instances";
+
PsCommand();
~PsCommand();
@@ -29,7 +35,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"};
+ cxxopts::Options m_Options{Name, Description};
};
} // namespace zen
diff --git a/src/zen/cmds/trace_cmd.h b/src/zen/cmds/trace_cmd.h
index a6c9742b7..6eb0ba22b 100644
--- a/src/zen/cmds/trace_cmd.h
+++ b/src/zen/cmds/trace_cmd.h
@@ -6,11 +6,12 @@
namespace zen {
-/** Scrub storage
- */
class TraceCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "trace";
+ static constexpr char Description[] = "Control zen realtime tracing";
+
TraceCommand();
~TraceCommand();
@@ -18,7 +19,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"trace", "Control zen realtime tracing"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_Stop = false;
std::string m_TraceHost;
diff --git a/src/zen/cmds/ui_cmd.cpp b/src/zen/cmds/ui_cmd.cpp
new file mode 100644
index 000000000..da06ce305
--- /dev/null
+++ b/src/zen/cmds/ui_cmd.cpp
@@ -0,0 +1,236 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "ui_cmd.h"
+
+#include <zencore/except_fmt.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/process.h>
+#include <zenutil/consoletui.h>
+#include <zenutil/zenserverprocess.h>
+
+#if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+# include <shellapi.h>
+#endif
+
+namespace zen {
+
+namespace {
+
+ struct RunningServerInfo
+ {
+ uint16_t Port;
+ uint32_t Pid;
+ std::string SessionId;
+ std::string CmdLine;
+ };
+
+ static std::vector<RunningServerInfo> CollectRunningServers()
+ {
+ std::vector<RunningServerInfo> Servers;
+ ZenServerState State;
+ if (!State.InitializeReadOnly())
+ return Servers;
+
+ State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
+ StringBuilder<25> SessionSB;
+ Entry.GetSessionId().ToString(SessionSB);
+ std::error_code CmdLineEc;
+ std::string CmdLine = GetProcessCommandLine(static_cast<int>(Entry.Pid.load()), CmdLineEc);
+ Servers.push_back({Entry.EffectiveListenPort.load(), Entry.Pid.load(), std::string(SessionSB.c_str()), std::move(CmdLine)});
+ });
+
+ return Servers;
+ }
+
+} // namespace
+
+UiCommand::UiCommand()
+{
+ m_Options.add_options()("h,help", "Print help");
+ m_Options.add_options()("a,all", "Open dashboard for all running instances", cxxopts::value(m_All)->default_value("false"));
+ m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+ m_Options.add_option("",
+ "p",
+ "path",
+ "Dashboard path (default: /dashboard/)",
+ cxxopts::value(m_DashboardPath)->default_value("/dashboard/"),
+ "<path>");
+ m_Options.parse_positional("path");
+}
+
+UiCommand::~UiCommand()
+{
+}
+
+void
+UiCommand::OpenBrowser(std::string_view HostName)
+{
+ // Allow shortcuts for specifying dashboard path, and ensure it is in a format we expect
+ // (leading slash, trailing slash if no file extension)
+
+ if (!m_DashboardPath.empty())
+ {
+ if (m_DashboardPath[0] != '/')
+ {
+ m_DashboardPath = "/dashboard/" + m_DashboardPath;
+ }
+
+ if (m_DashboardPath.find_last_of('.') == std::string::npos && m_DashboardPath.back() != '/')
+ {
+ m_DashboardPath += '/';
+ }
+ }
+
+ bool Success = false;
+
+ ExtendableStringBuilder<256> FullUrl;
+ FullUrl << HostName << m_DashboardPath;
+
+#if ZEN_PLATFORM_WINDOWS
+ HINSTANCE Result = ShellExecuteA(nullptr, "open", FullUrl.c_str(), nullptr, nullptr, SW_SHOWNORMAL);
+ Success = reinterpret_cast<intptr_t>(Result) > 32;
+#else
+ // Validate URL doesn't contain shell metacharacters that could lead to command injection
+ std::string_view FullUrlView = FullUrl;
+ constexpr std::string_view DangerousChars = ";|&$`\\\"'<>(){}[]!#*?~\n\r";
+ if (FullUrlView.find_first_of(DangerousChars) != std::string_view::npos)
+ {
+ throw OptionParseException(fmt::format("URL contains invalid characters: '{}'", FullUrl), m_Options.help());
+ }
+
+# if ZEN_PLATFORM_MAC
+ std::string Command = fmt::format("open \"{}\"", FullUrl);
+# elif ZEN_PLATFORM_LINUX
+ std::string Command = fmt::format("xdg-open \"{}\"", FullUrl);
+# else
+ ZEN_NOT_IMPLEMENTED("Browser launching not implemented on this platform");
+# endif
+
+ Success = system(Command.c_str()) == 0;
+#endif
+
+ if (!Success)
+ {
+ throw zen::runtime_error("Failed to launch browser for '{}'", FullUrl);
+ }
+
+ ZEN_CONSOLE("Web browser launched for '{}' successfully", FullUrl);
+}
+
+void
+UiCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+ using namespace std::literals;
+
+ ZEN_UNUSED(GlobalOptions);
+
+ if (!ParseOptions(argc, argv))
+ {
+ return;
+ }
+
+ // Resolve target server
+ uint16_t ServerPort = 0;
+
+ if (m_HostName.empty())
+ {
+ // Auto-discover running instances.
+ std::vector<RunningServerInfo> Servers = CollectRunningServers();
+
+ if (m_All)
+ {
+ if (Servers.empty())
+ {
+ throw OptionParseException("No running Zen server instances found", m_Options.help());
+ }
+
+ for (const auto& Server : Servers)
+ {
+ OpenBrowser(fmt::format("http://localhost:{}", Server.Port));
+ }
+ return;
+ }
+
+ // If multiple are found and we have an interactive terminal, present a picker
+ // instead of silently using the first one.
+ if (Servers.size() > 1 && IsTuiAvailable())
+ {
+ std::vector<std::string> Labels;
+ Labels.reserve(Servers.size() + 1);
+ Labels.push_back(fmt::format("(all {} instances)", Servers.size()));
+
+ const int32_t Cols = static_cast<int32_t>(TuiConsoleColumns());
+ constexpr int32_t kIndicator = 3; // " ▶ " or " " prefix
+ constexpr int32_t kSeparator = 2; // " " before cmdline
+ constexpr int32_t kEllipsis = 3; // "..."
+
+ for (const auto& Server : Servers)
+ {
+ std::string Label = fmt::format("port {:<5} pid {:<7} session {}", Server.Port, Server.Pid, Server.SessionId);
+
+ if (!Server.CmdLine.empty())
+ {
+ int32_t Available = Cols - kIndicator - kSeparator - static_cast<int32_t>(Label.size());
+ if (Available > kEllipsis)
+ {
+ Label += " ";
+ if (static_cast<int32_t>(Server.CmdLine.size()) <= Available)
+ {
+ Label += Server.CmdLine;
+ }
+ else
+ {
+ Label.append(Server.CmdLine, 0, static_cast<size_t>(Available - kEllipsis));
+ Label += "...";
+ }
+ }
+ }
+
+ Labels.push_back(std::move(Label));
+ }
+
+ int SelectedIdx = TuiPickOne("Multiple Zen server instances found. Select one to open:", Labels);
+ if (SelectedIdx < 0)
+ return; // User cancelled
+
+ if (SelectedIdx == 0)
+ {
+ // "All" selected
+ for (const auto& Server : Servers)
+ {
+ OpenBrowser(fmt::format("http://localhost:{}", Server.Port));
+ }
+ return;
+ }
+
+ ServerPort = Servers[SelectedIdx - 1].Port;
+ m_HostName = fmt::format("http://localhost:{}", ServerPort);
+ }
+
+ if (m_HostName.empty())
+ {
+ // Single or zero instances, or not an interactive terminal:
+ // fall back to default resolution (picks first instance or returns empty)
+ m_HostName = ResolveTargetHostSpec("", ServerPort);
+ }
+ }
+ else
+ {
+ if (m_All)
+ {
+ throw OptionParseException("--all cannot be used together with --hosturl", m_Options.help());
+ }
+ m_HostName = ResolveTargetHostSpec(m_HostName, ServerPort);
+ }
+
+ if (m_HostName.empty())
+ {
+ throw OptionParseException("Unable to resolve server specification", m_Options.help());
+ }
+
+ OpenBrowser(m_HostName);
+}
+
+} // namespace zen
diff --git a/src/zen/cmds/ui_cmd.h b/src/zen/cmds/ui_cmd.h
new file mode 100644
index 000000000..c74cdbbd0
--- /dev/null
+++ b/src/zen/cmds/ui_cmd.h
@@ -0,0 +1,32 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+#include <filesystem>
+
+namespace zen {
+
+class UiCommand : public ZenCmdBase
+{
+public:
+ UiCommand();
+ ~UiCommand();
+
+ static constexpr char Name[] = "ui";
+ static constexpr char Description[] = "Launch web browser with zen server UI";
+
+ virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+ virtual cxxopts::Options& Options() override { return m_Options; }
+
+private:
+ void OpenBrowser(std::string_view HostName);
+
+ cxxopts::Options m_Options{Name, Description};
+ std::string m_HostName;
+ std::string m_DashboardPath = "/dashboard/";
+ bool m_All = false;
+};
+
+} // namespace zen
diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h
index 2e822d5fc..270db7f88 100644
--- a/src/zen/cmds/up_cmd.h
+++ b/src/zen/cmds/up_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class UpCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "up";
+ static constexpr char Description[] = "Bring zen server up";
+
UpCommand();
~UpCommand();
@@ -18,7 +21,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"up", "Bring up zen service"};
+ cxxopts::Options m_Options{Name, Description};
uint16_t m_Port = 0;
bool m_ShowConsole = false;
bool m_ShowLog = false;
@@ -28,6 +31,9 @@ private:
class AttachCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "attach";
+ static constexpr char Description[] = "Add a sponsor process to a running zen service";
+
AttachCommand();
~AttachCommand();
@@ -35,7 +41,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"};
+ cxxopts::Options m_Options{Name, Description};
uint16_t m_Port = 0;
int m_OwnerPid = 0;
std::filesystem::path m_DataDir;
@@ -44,6 +50,9 @@ private:
class DownCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "down";
+ static constexpr char Description[] = "Bring zen server down";
+
DownCommand();
~DownCommand();
@@ -51,7 +60,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"down", "Bring down zen service"};
+ cxxopts::Options m_Options{Name, Description};
uint16_t m_Port = 0;
bool m_ForceTerminate = false;
std::filesystem::path m_ProgramBaseDir;
diff --git a/src/zen/cmds/vfs_cmd.h b/src/zen/cmds/vfs_cmd.h
index 5deaa02fa..9009c774b 100644
--- a/src/zen/cmds/vfs_cmd.h
+++ b/src/zen/cmds/vfs_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class VfsCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "vfs";
+ static constexpr char Description[] = "Manage virtual file system";
+
VfsCommand();
~VfsCommand();
@@ -16,7 +19,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"vfs", "Manage virtual file system"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_Verb;
std::string m_HostName;
diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp
index adf0e61f0..10f5ad8e1 100644
--- a/src/zen/cmds/wipe_cmd.cpp
+++ b/src/zen/cmds/wipe_cmd.cpp
@@ -33,7 +33,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
-namespace {
+namespace wipe_impl {
static std::atomic<bool> AbortFlag = false;
static std::atomic<bool> PauseFlag = false;
static bool IsVerbose = false;
@@ -49,10 +49,11 @@ namespace {
: GetMediumWorkerPool(EWorkloadType::Burst);
}
-#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \
- if (IsVerbose) \
- { \
- ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \
+#undef ZEN_CONSOLE_VERBOSE
+#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \
+ if (IsVerbose) \
+ { \
+ ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \
}
static void SignalCallbackHandler(int SigNum)
@@ -505,7 +506,7 @@ namespace {
}
return CleanWipe;
}
-} // namespace
+} // namespace wipe_impl
WipeCommand::WipeCommand()
{
@@ -532,6 +533,7 @@ WipeCommand::~WipeCommand() = default;
void
WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace wipe_impl;
ZEN_UNUSED(GlobalOptions);
signal(SIGINT, SignalCallbackHandler);
@@ -549,7 +551,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ProgressMode = (IsVerbose || m_PlainProgress) ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty;
BoostWorkerThreads = m_BoostWorkerThreads;
- MakeSafeAbsolutePathÍnPlace(m_Directory);
+ MakeSafeAbsolutePathInPlace(m_Directory);
if (!IsDir(m_Directory))
{
diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp
index 6e6f5d863..af265d898 100644
--- a/src/zen/cmds/workspaces_cmd.cpp
+++ b/src/zen/cmds/workspaces_cmd.cpp
@@ -398,7 +398,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
}
else
{
- MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
+ MakeSafeAbsolutePathInPlace(m_SystemRootDir);
}
std::filesystem::path StatePath = m_SystemRootDir / "workspaces";
@@ -815,7 +815,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
if (Results.size() != m_ChunkIds.size())
{
throw std::runtime_error(
- fmt::format("failed to get workspace share batch - invalid result count recevied (expected: {}, received: {}",
+ fmt::format("failed to get workspace share batch - invalid result count received (expected: {}, received: {}",
m_ChunkIds.size(),
Results.size()));
}