diff options
| author | Liam Mitchell <[email protected]> | 2026-03-09 19:06:36 -0700 |
|---|---|---|
| committer | Liam Mitchell <[email protected]> | 2026-03-09 19:06:36 -0700 |
| commit | d1abc50ee9d4fb72efc646e17decafea741caa34 (patch) | |
| tree | e4288e00f2f7ca0391b83d986efcb69d3ba66a83 /src/zen/cmds | |
| parent | Allow requests with invalid content-types unless specified in command line or... (diff) | |
| parent | updated chunk–block analyser (#818) (diff) | |
| download | archived-zen-d1abc50ee9d4fb72efc646e17decafea741caa34.tar.xz archived-zen-d1abc50ee9d4fb72efc646e17decafea741caa34.zip | |
Merge branch 'main' into lm/restrict-content-type
Diffstat (limited to 'src/zen/cmds')
| -rw-r--r-- | src/zen/cmds/admin_cmd.h | 40 | ||||
| -rw-r--r-- | src/zen/cmds/bench_cmd.h | 5 | ||||
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 217 | ||||
| -rw-r--r-- | src/zen/cmds/builds_cmd.h | 2 | ||||
| -rw-r--r-- | src/zen/cmds/cache_cmd.h | 20 | ||||
| -rw-r--r-- | src/zen/cmds/copy_cmd.h | 5 | ||||
| -rw-r--r-- | src/zen/cmds/dedup_cmd.h | 5 | ||||
| -rw-r--r-- | src/zen/cmds/exec_cmd.cpp | 1374 | ||||
| -rw-r--r-- | src/zen/cmds/exec_cmd.h | 101 | ||||
| -rw-r--r-- | src/zen/cmds/info_cmd.h | 5 | ||||
| -rw-r--r-- | src/zen/cmds/print_cmd.cpp | 4 | ||||
| -rw-r--r-- | src/zen/cmds/print_cmd.h | 10 | ||||
| -rw-r--r-- | src/zen/cmds/projectstore_cmd.cpp | 68 | ||||
| -rw-r--r-- | src/zen/cmds/projectstore_cmd.h | 60 | ||||
| -rw-r--r-- | src/zen/cmds/rpcreplay_cmd.h | 15 | ||||
| -rw-r--r-- | src/zen/cmds/run_cmd.h | 5 | ||||
| -rw-r--r-- | src/zen/cmds/serve_cmd.h | 5 | ||||
| -rw-r--r-- | src/zen/cmds/status_cmd.h | 5 | ||||
| -rw-r--r-- | src/zen/cmds/top_cmd.h | 10 | ||||
| -rw-r--r-- | src/zen/cmds/trace_cmd.h | 7 | ||||
| -rw-r--r-- | src/zen/cmds/ui_cmd.cpp | 236 | ||||
| -rw-r--r-- | src/zen/cmds/ui_cmd.h | 32 | ||||
| -rw-r--r-- | src/zen/cmds/up_cmd.h | 15 | ||||
| -rw-r--r-- | src/zen/cmds/vfs_cmd.h | 5 | ||||
| -rw-r--r-- | src/zen/cmds/wipe_cmd.cpp | 16 | ||||
| -rw-r--r-- | src/zen/cmds/workspaces_cmd.cpp | 4 |
26 files changed, 2100 insertions, 171 deletions
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h index 87ef8091b..83bcf8893 100644 --- a/src/zen/cmds/admin_cmd.h +++ b/src/zen/cmds/admin_cmd.h @@ -13,6 +13,9 @@ namespace zen { class ScrubCommand : public StorageCommand { public: + static constexpr char Name[] = "scrub"; + static constexpr char Description[] = "Scrub zen storage (verify data integrity)"; + ScrubCommand(); ~ScrubCommand(); @@ -20,7 +23,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"scrub", "Scrub zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_DryRun = false; bool m_NoGc = false; @@ -33,6 +36,9 @@ private: class GcCommand : public StorageCommand { public: + static constexpr char Name[] = "gc"; + static constexpr char Description[] = "Garbage collect zen storage"; + GcCommand(); ~GcCommand(); @@ -40,7 +46,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"gc", "Garbage collect zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_SmallObjects{false}; bool m_SkipCid{false}; @@ -62,6 +68,9 @@ private: class GcStatusCommand : public StorageCommand { public: + static constexpr char Name[] = "gc-status"; + static constexpr char Description[] = "Garbage collect zen storage status check"; + GcStatusCommand(); ~GcStatusCommand(); @@ -69,7 +78,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"gc-status", "Garbage collect zen storage status check"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_Details = false; }; @@ -77,6 +86,9 @@ private: class GcStopCommand : public StorageCommand { public: + static constexpr char Name[] = "gc-stop"; + static constexpr char Description[] = "Request cancel of running garbage collection in zen storage"; + GcStopCommand(); ~GcStopCommand(); @@ -84,7 +96,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"gc-stop", "Request cancel of running garbage collection in zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; @@ -93,6 +105,9 @@ private: class JobCommand : public ZenCmdBase { public: + static constexpr char Name[] = "jobs"; + static constexpr char Description[] = "Show/cancel zen background jobs"; + JobCommand(); ~JobCommand(); @@ -100,7 +115,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"jobs", "Show/cancel zen background jobs"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::uint64_t m_JobId = 0; bool m_Cancel = 0; @@ -111,6 +126,9 @@ private: class LoggingCommand : public ZenCmdBase { public: + static constexpr char Name[] = "logs"; + static constexpr char Description[] = "Show/control zen logging"; + LoggingCommand(); ~LoggingCommand(); @@ -118,7 +136,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"logs", "Show/control zen logging"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_CacheWriteLog; std::string m_CacheAccessLog; @@ -133,6 +151,9 @@ private: class FlushCommand : public StorageCommand { public: + static constexpr char Name[] = "flush"; + static constexpr char Description[] = "Flush storage"; + FlushCommand(); ~FlushCommand(); @@ -140,7 +161,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"flush", "Flush zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; @@ -149,6 +170,9 @@ private: class CopyStateCommand : public StorageCommand { public: + static constexpr char Name[] = "copy-state"; + static constexpr char Description[] = "Copy zen server disk state"; + CopyStateCommand(); ~CopyStateCommand(); @@ -156,7 +180,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_DataPath; std::filesystem::path m_TargetPath; bool m_SkipLogs = false; diff --git a/src/zen/cmds/bench_cmd.h b/src/zen/cmds/bench_cmd.h index ed123be75..7fbf85340 100644 --- a/src/zen/cmds/bench_cmd.h +++ b/src/zen/cmds/bench_cmd.h @@ -9,6 +9,9 @@ namespace zen { class BenchCommand : public ZenCmdBase { public: + static constexpr char Name[] = "bench"; + static constexpr char Description[] = "Utility command for benchmarking"; + BenchCommand(); ~BenchCommand(); @@ -17,7 +20,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"bench", "Benchmarking utility command"}; + cxxopts::Options m_Options{Name, Description}; bool m_PurgeStandbyLists = false; bool m_SingleProcess = false; }; diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index f4edb65ab..b4b4df7c9 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -67,13 +67,11 @@ ZEN_THIRD_PARTY_INCLUDES_END static const bool DoExtraContentVerify = false; -#define ZEN_CLOUD_STORAGE "Cloud Storage" - namespace zen { using namespace std::literals; -namespace { +namespace builds_impl { static std::atomic<bool> AbortFlag = false; static std::atomic<bool> PauseFlag = false; @@ -270,10 +268,11 @@ namespace { static bool IsQuiet = false; static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \ +#undef ZEN_CONSOLE_VERBOSE +#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ + if (IsVerbose) \ + { \ + ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ } const std::string DefaultAccessTokenEnvVariableName( @@ -1467,9 +1466,16 @@ namespace { ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); } + Stopwatch IndexTimer; + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent); const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); + if (!IsQuiet) + { + ZEN_OPERATION_LOG_INFO(Output, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + } + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); BuildsOperationUpdateFolder Updater( @@ -1588,7 +1594,7 @@ namespace { } } } - if (Storage.BuildCacheStorage) + if (Storage.CacheStorage) { if (SB.Size() > 0) { @@ -1643,9 +1649,9 @@ namespace { } if (Options.PrimeCacheOnly) { - if (Storage.BuildCacheStorage) + if (Storage.CacheStorage) { - Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) { + Storage.CacheStorage->Flush(5000, [](intptr_t Remaining) { if (!IsQuiet) { if (Remaining == 0) @@ -2002,12 +2008,13 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } -} // namespace +} // namespace builds_impl ////////////////////////////////////////////////////////////////////////////////////////////////////// BuildsCommand::BuildsCommand() { + using namespace builds_impl; m_Options.add_options()("h,help", "Print help"); auto AddSystemOptions = [this](cxxopts::Options& Ops) { @@ -2648,6 +2655,7 @@ BuildsCommand::~BuildsCommand() = default; void BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace builds_impl; ZEN_UNUSED(GlobalOptions); signal(SIGINT, SignalCallbackHandler); @@ -2680,7 +2688,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + MakeSafeAbsolutePathInPlace(m_SystemRootDir); }; ParseSystemOptions(); @@ -2729,7 +2737,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help()); } - MakeSafeAbsolutePathÍnPlace(m_StoragePath); + MakeSafeAbsolutePathInPlace(m_StoragePath); }; auto ParseOutputOptions = [&]() { @@ -2800,8 +2808,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) .Verbose = m_VerboseHttp, .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; - std::unique_ptr<AuthMgr> Auth; - std::string StorageDescription; std::string CacheDescription; @@ -2820,44 +2826,47 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*Output, ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose); - if (!ResolveRes.HostUrl.empty()) + if (!ResolveRes.Cloud.Address.empty()) { - ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2; + ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; Result.BuildStorageHttp = - std::make_unique<HttpClient>(ResolveRes.HostUrl, ClientSettings, []() { return AbortFlag.load(); }); + std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); - Result.BuildStorage = CreateJupiterBuildStorage(Log(), + Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, m_AllowRedirect, TempPath / "storage"); - Result.StorageName = ResolveRes.HostName; + Result.BuildStorageHost = ResolveRes.Cloud; + + uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0; - StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - ResolveRes.HostName, - (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), - Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket); - ; + StorageDescription = + fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}", + ResolveRes.Cloud.Name, + (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket, + NiceLatencyNs(HostLatencyNs)); - if (!ResolveRes.CacheUrl.empty()) + if (!ResolveRes.Cache.Address.empty()) { Result.CacheHttp = std::make_unique<HttpClient>( - ResolveRes.CacheUrl, + ResolveRes.Cache.Address, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = ResolveRes.CacheAssumeHttp2, + .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, .Verbose = m_VerboseHttp, .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, []() { return AbortFlag.load(); }); - Result.BuildCacheStorage = + Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, m_Namespace, @@ -2865,14 +2874,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) TempPath / "zencache", BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheName = ResolveRes.CacheName; + Result.CacheHost = ResolveRes.Cache; + + uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; CacheDescription = - fmt::format("Zen {}{}. SessionId: '{}'", - ResolveRes.CacheName, - (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl), - Result.CacheHttp->GetSessionId()); - ; + fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}", + ResolveRes.Cache.Name, + (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address), + Result.CacheHttp->GetSessionId(), + NiceLatencyNs(CacheLatencyNs)); + if (!m_Namespace.empty()) { CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); @@ -2888,41 +2900,56 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { StorageDescription = fmt::format("folder {}", m_StoragePath); Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + + Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), + .Name = "Disk", + .LatencySec = 1.0 / 100000, // 1 us + .Caps = {.MaxRangeCountPerRequest = 2048u}}; if (!m_ZenCacheHost.empty()) { - Result.CacheHttp = std::make_unique<HttpClient>( - m_ZenCacheHost, - HttpClientSettings{ - .LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{3000}, - .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); - Result.BuildCacheStorage = - CreateZenBuildStorageCache(*Result.CacheHttp, - StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheName = m_ZenCacheHost; - - CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'", Result.CacheName, "", Result.CacheHttp->GetSessionId()); - ; - if (!m_Namespace.empty()) - { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); - } - if (!m_Bucket.empty()) + ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); + + if (TestResult.Success) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + Result.CacheHttp = std::make_unique<HttpClient>( + m_ZenCacheHost, + HttpClientSettings{ + .LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 0, + .Verbose = m_VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, + []() { return AbortFlag.load(); }); + + Result.CacheStorage = + CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = + BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, + .Name = m_ZenCacheHost, + .AssumeHttp2 = m_AssumeHttp2, + .LatencySec = TestResult.LatencySeconds, + .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; + + CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); + + if (!m_Namespace.empty()) + { + CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + } + if (!m_Bucket.empty()) + { + CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + } } } } @@ -2934,7 +2961,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!IsQuiet) { ZEN_CONSOLE("Remote: {}", StorageDescription); - if (!Result.CacheName.empty()) + if (!Result.CacheHost.Name.empty()) { ZEN_CONSOLE("Cache : {}", CacheDescription); } @@ -2947,7 +2974,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--local-path' is required", SubOption->help()); } - MakeSafeAbsolutePathÍnPlace(m_Path); + MakeSafeAbsolutePathInPlace(m_Path); }; auto ParseFileFilters = [&](std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) { @@ -3004,7 +3031,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--compare-path' is required", SubOption->help()); } - MakeSafeAbsolutePathÍnPlace(m_DiffPath); + MakeSafeAbsolutePathInPlace(m_DiffPath); }; auto ParseBlobHash = [&]() -> IoHash { @@ -3016,7 +3043,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BlobHash.length() != IoHash::StringLength) { throw OptionParseException( - fmt::format("'--blob-hash' ('{}') is malfomed, it must be {} characters long", m_BlobHash, IoHash::StringLength), + fmt::format("'--blob-hash' ('{}') is malformed, it must be {} characters long", m_BlobHash, IoHash::StringLength), SubOption->help()); } @@ -3033,7 +3060,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BuildId.length() != Oid::StringLength) { throw OptionParseException( - fmt::format("'--build-id' ('{}') is malfomed, it must be {} characters long", m_BuildId, Oid::StringLength), + fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildId, Oid::StringLength), SubOption->help()); } else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero) @@ -3105,7 +3132,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_BuildMetadataPath.empty()) { - MakeSafeAbsolutePathÍnPlace(m_BuildMetadataPath); + MakeSafeAbsolutePathInPlace(m_BuildMetadataPath); IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten(); std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -3202,8 +3229,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ListOptions) { - MakeSafeAbsolutePathÍnPlace(m_ListQueryPath); - MakeSafeAbsolutePathÍnPlace(m_ListResultPath); + MakeSafeAbsolutePathInPlace(m_ListQueryPath); + MakeSafeAbsolutePathInPlace(m_ListResultPath); if (!m_ListResultPath.empty()) { @@ -3255,7 +3282,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); }); @@ -3294,7 +3321,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ListBlocksOptions) { - MakeSafeAbsolutePathÍnPlace(m_ListResultPath); + MakeSafeAbsolutePathInPlace(m_ListResultPath); if (!m_ListResultPath.empty()) { @@ -3316,7 +3343,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); }); @@ -3387,8 +3414,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3475,7 +3502,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) "Requests: {}\n" "Avg Request Time: {}\n" "Avg I/O Time: {}", - Storage.StorageName, + Storage.BuildStorageHost.Name, NiceBytes(StorageStats.TotalBytesRead.load()), NiceBytes(StorageStats.TotalBytesWritten.load()), StorageStats.TotalRequestCount.load(), @@ -3532,7 +3559,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -3632,7 +3659,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -3652,7 +3679,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::unique_ptr<CbObjectWriter> StructuredOutput; if (!m_LsResultPath.empty()) { - MakeSafeAbsolutePathÍnPlace(m_LsResultPath); + MakeSafeAbsolutePathInPlace(m_LsResultPath); StructuredOutput = std::make_unique<CbObjectWriter>(); } @@ -3696,7 +3723,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ParsePath(); ParseDiffPath(); - MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; @@ -3745,7 +3772,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3796,12 +3823,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!IsQuiet) { - if (Storage.BuildCacheStorage) + if (Storage.CacheStorage) { - ZEN_CONSOLE("Uploaded {} ({}) blobs", + ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", StorageCacheStats.PutBlobCount.load(), NiceBytes(StorageCacheStats.PutBlobByteCount), - Storage.CacheName); + Storage.CacheHost.Name); } } @@ -3828,7 +3855,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3883,7 +3910,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3933,7 +3960,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(); @@ -4083,8 +4110,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index f5c44ab55..5c80beed5 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -71,7 +71,7 @@ private: bool m_AppendNewContent = false; uint8_t m_BlockReuseMinPercentLimit = 85; bool m_AllowMultiparts = true; - std::string m_AllowPartialBlockRequests = "mixed"; + std::string m_AllowPartialBlockRequests = "true"; AuthCommandLineOptions m_AuthOptions; diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h index 4dc05bbdc..4f5b90f4d 100644 --- a/src/zen/cmds/cache_cmd.h +++ b/src/zen/cmds/cache_cmd.h @@ -9,6 +9,9 @@ namespace zen { class DropCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "drop"; + static constexpr char Description[] = "Drop cache namespace or bucket"; + DropCommand(); ~DropCommand(); @@ -16,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"drop", "Drop cache namespace or bucket"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_NamespaceName; std::string m_BucketName; @@ -25,13 +28,16 @@ private: class CacheInfoCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "cache-info"; + static constexpr char Description[] = "Info on cache, namespace or bucket"; + CacheInfoCommand(); ~CacheInfoCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"cache-info", "Info on cache, namespace or bucket"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_NamespaceName; std::string m_SizeInfoBucketNames; @@ -42,26 +48,32 @@ private: class CacheStatsCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "cache-stats"; + static constexpr char Description[] = "Stats on cache"; + CacheStatsCommand(); ~CacheStatsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"cache-stats", "Stats info on cache"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; class CacheDetailsCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "cache-details"; + static constexpr char Description[] = "Details on cache"; + CacheDetailsCommand(); ~CacheDetailsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"cache-details", "Detailed info on cache"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_CSV = false; bool m_Details = false; diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h index e1a5dcb82..757a8e691 100644 --- a/src/zen/cmds/copy_cmd.h +++ b/src/zen/cmds/copy_cmd.h @@ -11,6 +11,9 @@ namespace zen { class CopyCommand : public ZenCmdBase { public: + static constexpr char Name[] = "copy"; + static constexpr char Description[] = "Copy file(s)"; + CopyCommand(); ~CopyCommand(); @@ -19,7 +22,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"copy", "Copy files efficiently"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_CopySource; std::filesystem::path m_CopyTarget; bool m_NoClone = false; diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h index 5b8387dd2..835b35e92 100644 --- a/src/zen/cmds/dedup_cmd.h +++ b/src/zen/cmds/dedup_cmd.h @@ -11,6 +11,9 @@ namespace zen { class DedupCommand : public ZenCmdBase { public: + static constexpr char Name[] = "dedup"; + static constexpr char Description[] = "Dedup files"; + DedupCommand(); ~DedupCommand(); @@ -19,7 +22,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"dedup", "Deduplicate files"}; + cxxopts::Options m_Options{Name, Description}; std::vector<std::string> m_Positional; std::filesystem::path m_DedupSource; std::filesystem::path m_DedupTarget; diff --git a/src/zen/cmds/exec_cmd.cpp b/src/zen/cmds/exec_cmd.cpp new file mode 100644 index 000000000..42c7119e7 --- /dev/null +++ b/src/zen/cmds/exec_cmd.cpp @@ -0,0 +1,1374 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "exec_cmd.h" + +#include <zencompute/computeservice.h> +#include <zencompute/recordingreader.h> +#include <zencore/compactbinary.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryfile.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryvalue.h> +#include <zencore/compress.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/scopeguard.h> +#include <zencore/session.h> +#include <zencore/stream.h> +#include <zencore/string.h> +#include <zencore/system.h> +#include <zencore/timer.h> +#include <zenhttp/httpclient.h> +#include <zenhttp/packageformat.h> + +#include <EASTL/hash_map.h> +#include <EASTL/hash_set.h> +#include <EASTL/map.h> + +using namespace std::literals; + +namespace eastl { + +template<> +struct hash<zen::IoHash> : public zen::IoHash::Hasher +{ +}; + +} // namespace eastl + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen { + +ExecCommand::ExecCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName), "<hosturl>"); + m_Options.add_option("", "", "log", "Action log directory", cxxopts::value(m_RecordingLogPath), "<path>"); + m_Options.add_option("", "p", "path", "Recording path (directory or .actionlog file)", cxxopts::value(m_RecordingPath), "<path>"); + m_Options.add_option("", "", "offset", "Recording replay start offset", cxxopts::value(m_Offset), "<offset>"); + m_Options.add_option("", "", "stride", "Recording replay stride", cxxopts::value(m_Stride), "<stride>"); + m_Options.add_option("", "", "limit", "Recording replay limit", cxxopts::value(m_Limit), "<limit>"); + m_Options.add_option("", "", "beacon", "Beacon path", cxxopts::value(m_BeaconPath), "<path>"); + m_Options.add_option("", "", "orch", "Orchestrator URL for worker discovery", cxxopts::value(m_OrchestratorUrl), "<url>"); + m_Options.add_option("", + "", + "mode", + "Select execution mode (http,inproc,dump,direct,beacon,buildlog)", + cxxopts::value(m_Mode)->default_value("http"), + "<string>"); + m_Options + .add_option("", "", "dump-actions", "Dump each action to console as it is dispatched", cxxopts::value(m_DumpActions), "<bool>"); + m_Options.add_option("", "o", "output", "Save action results to directory", cxxopts::value(m_OutputPath), "<path>"); + m_Options.add_option("", "", "binary", "Write output as binary packages instead of YAML", cxxopts::value(m_Binary), "<bool>"); + m_Options.add_option("", "", "quiet", "Quiet mode (less logging)", cxxopts::value(m_Quiet), "<bool>"); + m_Options.parse_positional("mode"); +} + +ExecCommand::~ExecCommand() +{ +} + +void +ExecCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + // Configure + + if (!ParseOptions(argc, argv)) + { + return; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_RecordingPath.empty()) + { + throw OptionParseException("replay path is required!", m_Options.help()); + } + + m_VerboseLogging = GlobalOptions.IsVerbose; + m_QuietLogging = m_Quiet && !m_VerboseLogging; + + enum ExecMode + { + kHttp, + kDirect, + kInproc, + kDump, + kBeacon, + kBuildLog + } Mode; + + if (m_Mode == "http"sv) + { + Mode = kHttp; + } + else if (m_Mode == "direct"sv) + { + Mode = kDirect; + } + else if (m_Mode == "inproc"sv) + { + Mode = kInproc; + } + else if (m_Mode == "dump"sv) + { + Mode = kDump; + } + else if (m_Mode == "beacon"sv) + { + Mode = kBeacon; + } + else if (m_Mode == "buildlog"sv) + { + Mode = kBuildLog; + } + else + { + throw OptionParseException("invalid mode specified!", m_Options.help()); + } + + // Gather information from recording path + + std::unique_ptr<zen::compute::RecordingReader> Reader; + std::unique_ptr<zen::compute::UeRecordingReader> UeReader; + + std::filesystem::path RecordingPath{m_RecordingPath}; + + if (!std::filesystem::is_directory(RecordingPath)) + { + throw OptionParseException("replay path should be a directory path!", m_Options.help()); + } + else + { + if (std::filesystem::is_directory(RecordingPath / "cid")) + { + Reader = std::make_unique<zen::compute::RecordingReader>(RecordingPath); + m_WorkerMap = Reader->ReadWorkers(); + m_ChunkResolver = Reader.get(); + m_RecordingReader = Reader.get(); + } + else + { + UeReader = std::make_unique<zen::compute::UeRecordingReader>(RecordingPath); + m_WorkerMap = UeReader->ReadWorkers(); + m_ChunkResolver = UeReader.get(); + m_RecordingReader = UeReader.get(); + } + } + + ZEN_CONSOLE("found {} workers, {} action items", m_WorkerMap.size(), m_RecordingReader->GetActionCount()); + + for (auto& Kv : m_WorkerMap) + { + CbObject WorkerDesc = Kv.second.GetObject(); + const IoHash& WorkerId = Kv.first; + + RegisterWorkerFunctionsFromDescription(WorkerDesc, WorkerId); + + if (m_VerboseLogging) + { + zen::ExtendableStringBuilder<1024> ObjStr; +# if 0 + zen::CompactBinaryToJson(WorkerDesc, ObjStr); + ZEN_CONSOLE("worker {}: {}", WorkerId, ObjStr); +# else + zen::CompactBinaryToYaml(WorkerDesc, ObjStr); + ZEN_CONSOLE("worker {}:\n{}", WorkerId, ObjStr); +# endif + } + } + + if (m_VerboseLogging) + { + EmitFunctionList(m_FunctionList); + } + + // Iterate over work items and dispatch or log them + + int ReturnValue = 0; + + Stopwatch ExecTimer; + + switch (Mode) + { + case kHttp: + // Forward requests to HTTP function service + ReturnValue = HttpExecute(); + break; + + case kDirect: + // Not currently supported + ReturnValue = LocalMessagingExecute(); + break; + + case kInproc: + // Handle execution in-core (by spawning child processes) + ReturnValue = InProcessExecute(); + break; + + case kDump: + // Dump high level information about actions to console + ReturnValue = DumpWorkItems(); + break; + + case kBeacon: + ReturnValue = BeaconExecute(); + break; + + case kBuildLog: + ReturnValue = BuildActionsLog(); + break; + + default: + ZEN_ERROR("Unknown operating mode! No work submitted"); + + ReturnValue = 1; + } + + ZEN_CONSOLE("complete - took {}", NiceTimeSpanMs(ExecTimer.GetElapsedTimeMs())); + + if (!ReturnValue) + { + ZEN_CONSOLE("all work items completed successfully"); + } + else + { + ZEN_CONSOLE("some work items failed (code {})", ReturnValue); + } +} + +int +ExecCommand::InProcessExecute() +{ + ZEN_ASSERT(m_ChunkResolver); + ChunkResolver& Resolver = *m_ChunkResolver; + + zen::compute::ComputeServiceSession ComputeSession(Resolver); + + std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); + ComputeSession.AddLocalRunner(Resolver, TempPath); + + return ExecUsingSession(ComputeSession); +} + +int +ExecCommand::ExecUsingSession(zen::compute::ComputeServiceSession& ComputeSession) +{ + struct JobTracker + { + public: + inline void Insert(int LsnField) + { + RwLock::ExclusiveLockScope _(Lock); + PendingJobs.insert(LsnField); + } + + inline bool IsEmpty() const + { + RwLock::SharedLockScope _(Lock); + return PendingJobs.empty(); + } + + inline void Remove(int CompleteLsn) + { + RwLock::ExclusiveLockScope _(Lock); + PendingJobs.erase(CompleteLsn); + } + + inline size_t GetSize() const + { + RwLock::SharedLockScope _(Lock); + return PendingJobs.size(); + } + + private: + mutable RwLock Lock; + std::unordered_set<int> PendingJobs; + }; + + JobTracker PendingJobs; + + struct ActionSummaryEntry + { + int32_t Lsn = 0; + int RecordingIndex = 0; + IoHash ActionId; + std::string FunctionName; + int InputAttachments = 0; + uint64_t InputBytes = 0; + int OutputAttachments = 0; + uint64_t OutputBytes = 0; + float WallSeconds = 0.0f; + float CpuSeconds = 0.0f; + uint64_t SubmittedTicks = 0; + uint64_t StartedTicks = 0; + std::string ExecutionLocation; + }; + + std::mutex SummaryLock; + std::unordered_map<int32_t, ActionSummaryEntry> SummaryEntries; + + ComputeSession.WaitUntilReady(); + + // Register as a client with the orchestrator (best-effort) + + std::string OrchestratorClientId; + + if (!m_OrchestratorUrl.empty()) + { + try + { + HttpClient OrchestratorClient(m_OrchestratorUrl); + + CbObjectWriter Ann; + Ann << "session_id"sv << GetSessionId(); + Ann << "hostname"sv << std::string_view(GetMachineName()); + + CbObjectWriter Meta; + Meta << "source"sv + << "zen-exec"sv; + Ann << "metadata"sv << Meta.Save(); + + auto Resp = OrchestratorClient.Post("/orch/clients", Ann.Save()); + if (Resp.IsSuccess()) + { + OrchestratorClientId = std::string(Resp.AsObject()["id"].AsString()); + ZEN_CONSOLE_INFO("registered with orchestrator as {}", OrchestratorClientId); + } + else + { + ZEN_WARN("failed to register with orchestrator (status {})", static_cast<int>(Resp.StatusCode)); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("failed to register with orchestrator: {}", Ex.what()); + } + } + + Stopwatch OrchestratorHeartbeatTimer; + + auto SendOrchestratorHeartbeat = [&] { + if (OrchestratorClientId.empty() || OrchestratorHeartbeatTimer.GetElapsedTimeMs() < 30'000) + { + return; + } + OrchestratorHeartbeatTimer.Reset(); + try + { + HttpClient OrchestratorClient(m_OrchestratorUrl); + std::ignore = OrchestratorClient.Post(fmt::format("/orch/clients/{}/update", OrchestratorClientId)); + } + catch (...) + { + } + }; + + auto ClientCleanup = MakeGuard([&] { + if (!OrchestratorClientId.empty()) + { + try + { + HttpClient OrchestratorClient(m_OrchestratorUrl); + std::ignore = OrchestratorClient.Post(fmt::format("/orch/clients/{}/complete", OrchestratorClientId)); + } + catch (...) + { + } + } + }); + + // Create a queue to group all actions from this exec session + + CbObjectWriter Metadata; + Metadata << "source"sv + << "zen-exec"sv; + + auto QueueResult = ComputeSession.CreateQueue("zen-exec", Metadata.Save()); + const int QueueId = QueueResult.QueueId; + if (!QueueId) + { + ZEN_ERROR("failed to create compute queue"); + return 1; + } + + auto QueueCleanup = MakeGuard([&] { ComputeSession.DeleteQueue(QueueId); }); + + if (!m_OutputPath.empty()) + { + zen::CreateDirectories(m_OutputPath); + } + + std::atomic<int> IsDraining{0}; + + auto DrainCompletedJobs = [&] { + if (IsDraining.exchange(1)) + { + return; + } + + auto _ = MakeGuard([&] { IsDraining.store(0, std::memory_order_release); }); + + CbObjectWriter Cbo; + ComputeSession.GetQueueCompleted(QueueId, Cbo); + + if (CbObject Completed = Cbo.Save()) + { + for (auto& It : Completed["completed"sv]) + { + int32_t CompleteLsn = It.AsInt32(); + + CbPackage ResultPackage; + HttpResponseCode Response = ComputeSession.GetActionResult(CompleteLsn, /* out */ ResultPackage); + + if (Response == HttpResponseCode::OK) + { + if (!m_OutputPath.empty() && ResultPackage) + { + int OutputAttachments = 0; + uint64_t OutputBytes = 0; + + if (!m_Binary) + { + // Write the root object as YAML + ExtendableStringBuilder<4096> YamlStr; + CompactBinaryToYaml(ResultPackage.GetObject(), YamlStr); + + std::string_view Yaml = YamlStr; + zen::WriteFile(m_OutputPath / fmt::format("{}.result.yaml", CompleteLsn), + IoBuffer(IoBuffer::Clone, Yaml.data(), Yaml.size())); + + // Write decompressed attachments + auto Attachments = ResultPackage.GetAttachments(); + + if (!Attachments.empty()) + { + std::filesystem::path AttDir = m_OutputPath / fmt::format("{}.result.attachments", CompleteLsn); + zen::CreateDirectories(AttDir); + + for (const CbAttachment& Att : Attachments) + { + ++OutputAttachments; + + IoHash AttHash = Att.GetHash(); + + if (Att.IsCompressedBinary()) + { + SharedBuffer Decompressed = Att.AsCompressedBinary().Decompress(); + OutputBytes += Decompressed.GetSize(); + zen::WriteFile(AttDir / AttHash.ToHexString(), + IoBuffer(IoBuffer::Clone, Decompressed.GetData(), Decompressed.GetSize())); + } + else + { + SharedBuffer Binary = Att.AsBinary(); + OutputBytes += Binary.GetSize(); + zen::WriteFile(AttDir / AttHash.ToHexString(), + IoBuffer(IoBuffer::Clone, Binary.GetData(), Binary.GetSize())); + } + } + } + + if (!m_QuietLogging) + { + ZEN_CONSOLE("saved result: {}/{}.result.yaml ({} attachments)", + m_OutputPath.string(), + CompleteLsn, + OutputAttachments); + } + } + else + { + CompositeBuffer Serialized = FormatPackageMessageBuffer(ResultPackage); + zen::WriteFile(m_OutputPath / fmt::format("{}.result.pkg", CompleteLsn), std::move(Serialized)); + + for (const CbAttachment& Att : ResultPackage.GetAttachments()) + { + ++OutputAttachments; + OutputBytes += Att.AsBinary().GetSize(); + } + + if (!m_QuietLogging) + { + ZEN_CONSOLE("saved result: {}/{}.result.pkg", m_OutputPath.string(), CompleteLsn); + } + } + + std::lock_guard Lock(SummaryLock); + if (auto It2 = SummaryEntries.find(CompleteLsn); It2 != SummaryEntries.end()) + { + It2->second.OutputAttachments = OutputAttachments; + It2->second.OutputBytes = OutputBytes; + } + } + + PendingJobs.Remove(CompleteLsn); + + ZEN_CONSOLE("completed: LSN {} ({} still pending)", CompleteLsn, PendingJobs.GetSize()); + } + } + } + }; + + // Describe workers + + ZEN_CONSOLE("describing {} workers", m_WorkerMap.size()); + + for (auto Kv : m_WorkerMap) + { + CbPackage WorkerDesc = Kv.second; + + ComputeSession.RegisterWorker(WorkerDesc); + } + + // Then submit work items + + int FailedWorkCounter = 0; + size_t RemainingWorkItems = m_RecordingReader->GetActionCount(); + int SubmittedWorkItems = 0; + + ZEN_CONSOLE("submitting {} work items", RemainingWorkItems); + + int OffsetCounter = m_Offset; + int StrideCounter = m_Stride; + + auto ShouldSchedule = [&]() -> bool { + if (m_Limit && SubmittedWorkItems >= m_Limit) + { + // Limit reached, ignore + + return false; + } + + if (OffsetCounter && OffsetCounter--) + { + // Still in offset, ignore + + return false; + } + + if (--StrideCounter == 0) + { + StrideCounter = m_Stride; + + return true; + } + + return false; + }; + + int TargetParallelism = 8; + + if (OffsetCounter || StrideCounter || m_Limit) + { + TargetParallelism = 1; + } + + std::atomic<int> RecordingIndex{0}; + + m_RecordingReader->IterateActions( + [&](CbObject ActionObject, const IoHash& ActionId) { + // Enqueue job + + const int CurrentRecordingIndex = RecordingIndex++; + + Stopwatch SubmitTimer; + + const int Priority = 0; + + if (ShouldSchedule()) + { + if (m_VerboseLogging) + { + int AttachmentCount = 0; + uint64_t AttachmentBytes = 0; + eastl::hash_set<IoHash> ReferencedChunks; + + ActionObject.IterateAttachments([&](CbFieldView Field) { + IoHash AttachData = Field.AsAttachment(); + + ReferencedChunks.insert(AttachData); + ++AttachmentCount; + + if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachData)) + { + AttachmentBytes += ChunkData.GetSize(); + } + }); + + zen::ExtendableStringBuilder<1024> ObjStr; + zen::CompactBinaryToJson(ActionObject, ObjStr); + ZEN_CONSOLE("work item {} ({} attachments, {} bytes): {}", + ActionId, + AttachmentCount, + NiceBytes(AttachmentBytes), + ObjStr); + } + + if (m_DumpActions) + { + int AttachmentCount = 0; + uint64_t AttachmentBytes = 0; + + ActionObject.IterateAttachments([&](CbFieldView Field) { + IoHash AttachData = Field.AsAttachment(); + + ++AttachmentCount; + + if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachData)) + { + AttachmentBytes += ChunkData.GetSize(); + } + }); + + zen::ExtendableStringBuilder<1024> ObjStr; + zen::CompactBinaryToYaml(ActionObject, ObjStr); + ZEN_CONSOLE("action {} ({} attachments, {}):\n{}", ActionId, AttachmentCount, NiceBytes(AttachmentBytes), ObjStr); + } + + if (zen::compute::ComputeServiceSession::EnqueueResult EnqueueResult = + ComputeSession.EnqueueActionToQueue(QueueId, ActionObject, Priority)) + { + const int32_t LsnField = EnqueueResult.Lsn; + + --RemainingWorkItems; + ++SubmittedWorkItems; + + if (!m_QuietLogging) + { + ZEN_CONSOLE("submitted work item #{} - LSN {} - {}. {} remaining", + SubmittedWorkItems, + LsnField, + NiceTimeSpanMs(SubmitTimer.GetElapsedTimeMs()), + RemainingWorkItems); + } + + if (!m_OutputPath.empty()) + { + ActionSummaryEntry Entry; + Entry.Lsn = LsnField; + Entry.RecordingIndex = CurrentRecordingIndex; + Entry.ActionId = ActionId; + Entry.FunctionName = std::string(ActionObject["Function"sv].AsString()); + + if (!m_Binary) + { + // Write action object as YAML + ExtendableStringBuilder<4096> YamlStr; + CompactBinaryToYaml(ActionObject, YamlStr); + + std::string_view Yaml = YamlStr; + zen::WriteFile(m_OutputPath / fmt::format("{}.action.yaml", LsnField), + IoBuffer(IoBuffer::Clone, Yaml.data(), Yaml.size())); + + // Write decompressed input attachments + std::filesystem::path AttDir = m_OutputPath / fmt::format("{}.action.attachments", LsnField); + bool AttDirCreated = false; + + ActionObject.IterateAttachments([&](CbFieldView Field) { + IoHash AttachCid = Field.AsAttachment(); + ++Entry.InputAttachments; + + if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachCid)) + { + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer(ChunkData), RawHash, RawSize); + SharedBuffer Decompressed = Compressed.Decompress(); + + Entry.InputBytes += Decompressed.GetSize(); + + if (!AttDirCreated) + { + zen::CreateDirectories(AttDir); + AttDirCreated = true; + } + + zen::WriteFile(AttDir / AttachCid.ToHexString(), + IoBuffer(IoBuffer::Clone, Decompressed.GetData(), Decompressed.GetSize())); + } + }); + + if (!m_QuietLogging) + { + ZEN_CONSOLE("saved action: {}/{}.action.yaml ({} attachments)", + m_OutputPath.string(), + LsnField, + Entry.InputAttachments); + } + } + else + { + // Build a CbPackage from the action and write as .pkg + CbPackage ActionPackage; + ActionPackage.SetObject(ActionObject); + + ActionObject.IterateAttachments([&](CbFieldView Field) { + IoHash AttachCid = Field.AsAttachment(); + ++Entry.InputAttachments; + + if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachCid)) + { + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer(ChunkData), RawHash, RawSize); + + Entry.InputBytes += ChunkData.GetSize(); + ActionPackage.AddAttachment(CbAttachment(std::move(Compressed), RawHash)); + } + }); + + CompositeBuffer Serialized = FormatPackageMessageBuffer(ActionPackage); + zen::WriteFile(m_OutputPath / fmt::format("{}.action.pkg", LsnField), std::move(Serialized)); + + if (!m_QuietLogging) + { + ZEN_CONSOLE("saved action: {}/{}.action.pkg", m_OutputPath.string(), LsnField); + } + } + + std::lock_guard Lock(SummaryLock); + SummaryEntries.emplace(LsnField, std::move(Entry)); + } + + PendingJobs.Insert(LsnField); + } + else + { + if (!m_QuietLogging) + { + std::string_view FunctionName = ActionObject["Function"sv].AsString(); + const Guid FunctionVersion = ActionObject["FunctionVersion"sv].AsUuid(); + const Guid BuildSystemVersion = ActionObject["BuildSystemVersion"sv].AsUuid(); + + ZEN_ERROR( + "failed to resolve function for work with (Function:{},FunctionVersion:{},BuildSystemVersion:{}). Work " + "descriptor " + "at: 'file://{}'", + std::string(FunctionName), + FunctionVersion, + BuildSystemVersion, + "<null>"); + + EmitFunctionListOnce(m_FunctionList); + } + + ++FailedWorkCounter; + } + } + + // Check for completed work + + DrainCompletedJobs(); + SendOrchestratorHeartbeat(); + }, + TargetParallelism); + + // Wait until all pending work is complete + + while (!PendingJobs.IsEmpty()) + { + // TODO: improve this logic + zen::Sleep(500); + + DrainCompletedJobs(); + SendOrchestratorHeartbeat(); + } + + // Merge timing data from queue history into summary entries + + if (!SummaryEntries.empty()) + { + // RunnerAction::State indices (can't include functionrunner.h from here) + constexpr int kStateNew = 0; + constexpr int kStatePending = 1; + constexpr int kStateRunning = 3; + constexpr int kStateCompleted = 4; // first terminal state + constexpr int kStateCount = 8; + + for (const auto& HistEntry : ComputeSession.GetQueueHistory(QueueId, 0)) + { + std::lock_guard Lock(SummaryLock); + if (auto It = SummaryEntries.find(HistEntry.Lsn); It != SummaryEntries.end()) + { + // Find terminal state timestamp (Completed, Failed, Abandoned, or Cancelled) + uint64_t EndTick = 0; + for (int S = kStateCompleted; S < kStateCount; ++S) + { + if (HistEntry.Timestamps[S] != 0) + { + EndTick = HistEntry.Timestamps[S]; + break; + } + } + uint64_t StartTick = HistEntry.Timestamps[kStateNew]; + if (EndTick > StartTick) + { + It->second.WallSeconds = float(double(EndTick - StartTick) / double(TimeSpan::TicksPerSecond)); + } + It->second.CpuSeconds = HistEntry.CpuSeconds; + It->second.SubmittedTicks = HistEntry.Timestamps[kStatePending]; + It->second.StartedTicks = HistEntry.Timestamps[kStateRunning]; + It->second.ExecutionLocation = HistEntry.ExecutionLocation; + } + } + } + + // Write summary file if output path is set + + if (!m_OutputPath.empty() && !SummaryEntries.empty()) + { + std::vector<ActionSummaryEntry> Sorted; + Sorted.reserve(SummaryEntries.size()); + for (auto& [_, Entry] : SummaryEntries) + { + Sorted.push_back(std::move(Entry)); + } + + std::sort(Sorted.begin(), Sorted.end(), [](const ActionSummaryEntry& A, const ActionSummaryEntry& B) { + return A.RecordingIndex < B.RecordingIndex; + }); + + auto FormatTimestamp = [](uint64_t Ticks) -> std::string { + if (Ticks == 0) + { + return "-"; + } + return DateTime(Ticks).ToString("%H:%M:%S.%s"); + }; + + ExtendableStringBuilder<4096> Summary; + Summary.Append(fmt::format("{:<8} {:<8} {:<40} {:<40} {:>8} {:>12} {:>8} {:>12} {:>8} {:>8} {:>12} {:>12} {:<24}\n", + "LSN", + "Index", + "ActionId", + "Function", + "InAtt", + "InBytes", + "OutAtt", + "OutBytes", + "Wall(s)", + "CPU(s)", + "Submitted", + "Started", + "Location")); + Summary.Append(fmt::format("{:-<8} {:-<8} {:-<40} {:-<40} {:-<8} {:-<12} {:-<8} {:-<12} {:-<8} {:-<8} {:-<12} {:-<12} {:-<24}\n", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "")); + + for (const ActionSummaryEntry& Entry : Sorted) + { + Summary.Append(fmt::format("{:<8} {:<8} {:<40} {:<40} {:>8} {:>12} {:>8} {:>12} {:>8.2f} {:>8.2f} {:>12} {:>12} {:<24}\n", + Entry.Lsn, + Entry.RecordingIndex, + Entry.ActionId, + Entry.FunctionName, + Entry.InputAttachments, + NiceBytes(Entry.InputBytes), + Entry.OutputAttachments, + NiceBytes(Entry.OutputBytes), + Entry.WallSeconds, + Entry.CpuSeconds, + FormatTimestamp(Entry.SubmittedTicks), + FormatTimestamp(Entry.StartedTicks), + Entry.ExecutionLocation)); + } + + std::filesystem::path SummaryPath = m_OutputPath / "summary.txt"; + std::string_view SummaryStr = Summary; + zen::WriteFile(SummaryPath, IoBuffer(IoBuffer::Clone, SummaryStr.data(), SummaryStr.size())); + + ZEN_CONSOLE("wrote summary to {}", SummaryPath.string()); + + if (!m_Binary) + { + auto EscapeHtml = [](std::string_view Input) -> std::string { + std::string Out; + Out.reserve(Input.size()); + for (char C : Input) + { + switch (C) + { + case '&': + Out += "&"; + break; + case '<': + Out += "<"; + break; + case '>': + Out += ">"; + break; + case '"': + Out += """; + break; + case '\'': + Out += "'"; + break; + default: + Out += C; + } + } + return Out; + }; + + auto EscapeJson = [](std::string_view Input) -> std::string { + std::string Out; + Out.reserve(Input.size()); + for (char C : Input) + { + switch (C) + { + case '"': + Out += "\\\""; + break; + case '\\': + Out += "\\\\"; + break; + case '\n': + Out += "\\n"; + break; + case '\r': + Out += "\\r"; + break; + case '\t': + Out += "\\t"; + break; + default: + if (static_cast<unsigned char>(C) < 0x20) + { + Out += fmt::format("\\u{:04x}", static_cast<unsigned>(static_cast<unsigned char>(C))); + } + else + { + Out += C; + } + } + } + return Out; + }; + + ExtendableStringBuilder<8192> Html; + + Html.Append(std::string_view(R"(<!DOCTYPE html> +<html><head><meta charset="utf-8"><title>Exec Summary</title> +<style> +body{font-family:system-ui,sans-serif;margin:20px;background:#fafafa} +#container{overflow-y:auto;height:calc(100vh - 120px)} +table{border-collapse:collapse;width:100%} +th,td{border:1px solid #ddd;padding:6px 10px;text-align:left;white-space:nowrap} +th{background:#f0f0f0;cursor:pointer;user-select:none;position:sticky;top:0;z-index:1} +th:hover{background:#e0e0e0} +th .arrow{font-size:0.7em;margin-left:4px} +tr:hover{background:#e8f0fe} +input{padding:6px 10px;margin-bottom:12px;width:300px;border:1px solid #ccc;border-radius:4px} +button{padding:6px 14px;margin-left:8px;margin-bottom:12px;border:1px solid #ccc;border-radius:4px;background:#f0f0f0;cursor:pointer} +button:hover{background:#e0e0e0} +a{color:#1a73e8;text-decoration:none} +a:hover{text-decoration:underline} +.num{text-align:right} +</style></head><body> +<h2>Exec Summary</h2> +<input type="text" id="filter" placeholder="Filter by function name..."><button id="csvBtn">Export CSV</button> +<div id="container"> +<table><thead><tr> +<th data-col="0">LSN <span class="arrow"></span></th> +<th data-col="1">Index <span class="arrow"></span></th> +<th data-col="2">Action ID <span class="arrow"></span></th> +<th data-col="3">Function <span class="arrow"></span></th> +<th data-col="4">In Attachments <span class="arrow"></span></th> +<th data-col="5">In Bytes <span class="arrow"></span></th> +<th data-col="6">Out Attachments <span class="arrow"></span></th> +<th data-col="7">Out Bytes <span class="arrow"></span></th> +<th data-col="8">Wall(s) <span class="arrow"></span></th> +<th data-col="9">CPU(s) <span class="arrow"></span></th> +<th data-col="10">Submitted <span class="arrow"></span></th> +<th data-col="11">Started <span class="arrow"></span></th> +<th data-col="12">Location <span class="arrow"></span></th> +</tr></thead><tbody> +<tr id="spacerTop"><td colspan="13"></td></tr> +<tr id="spacerBot"><td colspan="13"></td></tr> +</tbody></table></div> +<script> +const DATA=[ +)")); + + std::string_view ResultExt = ".result.yaml"; + std::string_view ActionExt = ".action.yaml"; + + for (const ActionSummaryEntry& Entry : Sorted) + { + std::string SafeName = EscapeJson(EscapeHtml(Entry.FunctionName)); + std::string ActionIdStr = Entry.ActionId.ToHexString(); + std::string ActionLink; + if (!ActionExt.empty()) + { + ActionLink = EscapeJson(fmt::format(" <a href=\"{}{}\">[action]</a>", Entry.Lsn, ActionExt)); + } + + // Indices: 0=lsn, 1=idx, 2=actionId, 3=fn, 4=inAtt, 5=inBytes, 6=outAtt, 7=outBytes, + // 8=wall, 9=cpu, 10=niceBytesIn, 11=niceBytesOut, 12=actionLink, + // 13=submittedTicks, 14=startedTicks, 15=submittedDisplay, 16=startedDisplay, + // 17=location + Html.Append( + fmt::format("[{},{},\"{}\",\"{}\",{},{},{},{},{:.6f},{:.6f},\"{}\",\"{}\",\"{}\",{},{},\"{}\",\"{}\",\"{}\"],\n", + Entry.Lsn, + Entry.RecordingIndex, + ActionIdStr, + SafeName, + Entry.InputAttachments, + Entry.InputBytes, + Entry.OutputAttachments, + Entry.OutputBytes, + Entry.WallSeconds, + Entry.CpuSeconds, + EscapeJson(NiceBytes(Entry.InputBytes)), + EscapeJson(NiceBytes(Entry.OutputBytes)), + ActionLink, + Entry.SubmittedTicks, + Entry.StartedTicks, + FormatTimestamp(Entry.SubmittedTicks), + FormatTimestamp(Entry.StartedTicks), + EscapeJson(EscapeHtml(Entry.ExecutionLocation)))); + } + + Html.Append(fmt::format(R"(]; +const RESULT_EXT="{}"; +)", + ResultExt)); + + Html.Append(std::string_view(R"JS((function(){ +const ROW_H=33,BUF=20; +const container=document.getElementById("container"); +const tbody=container.querySelector("tbody"); +const headers=container.querySelectorAll("th"); +const filterInput=document.getElementById("filter"); +const spacerTop=document.getElementById("spacerTop"); +const spacerBot=document.getElementById("spacerBot"); +let view=[...DATA.keys()]; +let sortCol=-1,sortAsc=true; +const COLS=[ + {f:0,t:"n"},{f:1,t:"n"},{f:2,t:"s"},{f:3,t:"s"}, + {f:4,t:"n"},{f:5,t:"n"},{f:6,t:"n"},{f:7,t:"n"}, + {f:8,t:"n"},{f:9,t:"n"},{f:13,t:"n"},{f:14,t:"n"},{f:17,t:"s"} +]; +function rowHtml(i){ + const d=DATA[view[i]]; + const bg=i%2?' style="background:#f9f9f9"':''; + return '<tr'+bg+'>'+ + '<td class="num"><a href="'+d[0]+RESULT_EXT+'">'+d[0]+'</a></td>'+ + '<td class="num">'+d[1]+'</td>'+ + '<td><code>'+d[2]+'</code></td>'+ + '<td>'+d[3]+d[12]+'</td>'+ + '<td class="num">'+d[4]+'</td>'+ + '<td class="num">'+d[10]+'</td>'+ + '<td class="num">'+d[6]+'</td>'+ + '<td class="num">'+d[11]+'</td>'+ + '<td class="num">'+d[8].toFixed(2)+'</td>'+ + '<td class="num">'+d[9].toFixed(2)+'</td>'+ + '<td class="num">'+d[15]+'</td>'+ + '<td class="num">'+d[16]+'</td>'+ + '<td>'+d[17]+'</td></tr>'; +} +let lastFirst=-1,lastLast=-1; +function render(){ + const scrollTop=container.scrollTop; + const viewH=container.clientHeight; + let first=Math.floor(scrollTop/ROW_H)-BUF; + let last=Math.ceil((scrollTop+viewH)/ROW_H)+BUF; + if(first<0) first=0; + if(last>=view.length) last=view.length-1; + if(first===lastFirst&&last===lastLast) return; + lastFirst=first;lastLast=last; + const rows=[]; + for(let i=first;i<=last;i++) rows.push(rowHtml(i)); + spacerTop.style.height=(first*ROW_H)+'px'; + spacerBot.style.height=((view.length-1-last)*ROW_H)+'px'; + const mid=rows.join(''); + const topTr='<tr id="spacerTop"><td colspan="13" style="border:0;padding:0;height:'+spacerTop.style.height+'"></td></tr>'; + const botTr='<tr id="spacerBot"><td colspan="13" style="border:0;padding:0;height:'+spacerBot.style.height+'"></td></tr>'; + tbody.innerHTML=topTr+mid+botTr; +} +function applySort(){ + if(sortCol<0) return; + const c=COLS[sortCol]; + view.sort((a,b)=>{ + const va=DATA[a][c.f],vb=DATA[b][c.f]; + if(c.t==="n") return sortAsc?va-vb:vb-va; + return sortAsc?(va<vb?-1:va>vb?1:0):(va>vb?-1:va<vb?1:0); + }); +} +function rebuild(){ + const q=filterInput.value.toLowerCase(); + view=[]; + for(let i=0;i<DATA.length;i++){ + if(!q||DATA[i][3].toLowerCase().includes(q)) view.push(i); + } + applySort(); + lastFirst=lastLast=-1; + render(); +} +headers.forEach(th=>{ + th.addEventListener("click",()=>{ + const col=parseInt(th.dataset.col); + if(sortCol===col){sortAsc=!sortAsc}else{sortCol=col;sortAsc=true} + headers.forEach(h=>h.querySelector(".arrow").textContent=""); + th.querySelector(".arrow").textContent=sortAsc?"\u25B2":"\u25BC"; + applySort(); + lastFirst=lastLast=-1; + render(); + }); +}); +filterInput.addEventListener("input",()=>rebuild()); +let ticking=false; +container.addEventListener("scroll",()=>{ + if(!ticking){ticking=true;requestAnimationFrame(()=>{render();ticking=false})} +}); +rebuild(); +document.getElementById("csvBtn").addEventListener("click",()=>{ + const H=["LSN","Index","Action ID","Function","In Attachments","In Bytes","Out Attachments","Out Bytes","Wall(s)","CPU(s)","Submitted","Started","Location"]; + const esc=v=>{const s=String(v);return s.includes(',')||s.includes('"')||s.includes('\n')?'"'+s.replace(/"/g,'""')+'"':s}; + const rows=[H.join(",")]; + for(let i=0;i<view.length;i++){ + const d=DATA[view[i]]; + rows.push([d[0],d[1],d[2],d[3],d[4],d[5],d[6],d[7],d[8],d[9],d[15],d[16],d[17]].map(esc).join(",")); + } + const blob=new Blob([rows.join("\n")],{type:"text/csv"}); + const a=document.createElement("a"); + a.href=URL.createObjectURL(blob); + a.download="summary.csv"; + a.click(); + URL.revokeObjectURL(a.href); +}); +})(); +</script></body></html> +)JS")); + + std::filesystem::path HtmlPath = m_OutputPath / "summary.html"; + std::string_view HtmlStr = Html; + zen::WriteFile(HtmlPath, IoBuffer(IoBuffer::Clone, HtmlStr.data(), HtmlStr.size())); + + ZEN_CONSOLE("wrote HTML summary to {}", HtmlPath.string()); + } + } + + if (FailedWorkCounter) + { + return 1; + } + + return 0; +} + +int +ExecCommand::LocalMessagingExecute() +{ + // Non-HTTP work submission path + + // To be reimplemented using final transport + + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +int +ExecCommand::HttpExecute() +{ + ZEN_ASSERT(m_ChunkResolver); + ChunkResolver& Resolver = *m_ChunkResolver; + + std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); + + zen::compute::ComputeServiceSession ComputeSession(Resolver); + ComputeSession.AddRemoteRunner(Resolver, TempPath, m_HostName); + + return ExecUsingSession(ComputeSession); +} + +int +ExecCommand::BeaconExecute() +{ + ZEN_ASSERT(m_ChunkResolver); + ChunkResolver& Resolver = *m_ChunkResolver; + std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); + + zen::compute::ComputeServiceSession ComputeSession(Resolver); + + if (!m_OrchestratorUrl.empty()) + { + ZEN_CONSOLE_INFO("using orchestrator at {}", m_OrchestratorUrl); + ComputeSession.SetOrchestratorEndpoint(m_OrchestratorUrl); + ComputeSession.SetOrchestratorBasePath(TempPath); + } + else + { + ZEN_CONSOLE_INFO("note: using hard-coded local worker path"); + ComputeSession.AddRemoteRunner(Resolver, TempPath, "http://localhost:8558"); + } + + return ExecUsingSession(ComputeSession); +} + +////////////////////////////////////////////////////////////////////////// + +void +ExecCommand::RegisterWorkerFunctionsFromDescription(const CbObject& WorkerDesc, const IoHash& WorkerId) +{ + const Guid WorkerBuildSystemVersion = WorkerDesc["buildsystem_version"sv].AsUuid(); + + for (auto& Item : WorkerDesc["functions"sv]) + { + CbObjectView Function = Item.AsObjectView(); + + std::string_view FunctionName = Function["name"sv].AsString(); + const Guid FunctionVersion = Function["version"sv].AsUuid(); + + m_FunctionList.emplace_back(FunctionDefinition{.FunctionName = std::string{FunctionName}, + .FunctionVersion = FunctionVersion, + .BuildSystemVersion = WorkerBuildSystemVersion, + .WorkerId = WorkerId}); + } +} + +void +ExecCommand::EmitFunctionListOnce(const std::vector<FunctionDefinition>& FunctionList) +{ + if (m_FunctionListEmittedOnce == false) + { + EmitFunctionList(FunctionList); + + m_FunctionListEmittedOnce = true; + } +} + +int +ExecCommand::DumpWorkItems() +{ + std::atomic<int> EmittedCount{0}; + + eastl::hash_map<IoHash, uint64_t> SeenAttachments; // Attachment CID -> count of references + + m_RecordingReader->IterateActions( + [&](CbObject ActionObject, const IoHash& ActionId) { + eastl::hash_map<IoHash, CompressedBuffer> Attachments; + + uint64_t AttachmentBytes = 0; + uint64_t UncompressedAttachmentBytes = 0; + + ActionObject.IterateAttachments([&](const zen::CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + IoBuffer AttachmentData = m_ChunkResolver->FindChunkByCid(AttachmentCid); + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); + Attachments[AttachmentCid] = CompressedData; + + AttachmentBytes += CompressedData.GetCompressedSize(); + UncompressedAttachmentBytes += CompressedData.DecodeRawSize(); + + if (auto [Iter, Inserted] = SeenAttachments.insert({AttachmentCid, 1}); !Inserted) + { + ++Iter->second; + } + }); + + zen::ExtendableStringBuilder<1024> ObjStr; + +# if 0 + zen::CompactBinaryToJson(ActionObject, ObjStr); + ZEN_CONSOLE("work item {} ({} attachments): {}", ActionId, Attachments.size(), ObjStr); +# else + zen::CompactBinaryToYaml(ActionObject, ObjStr); + ZEN_CONSOLE("work item {} ({} attachments, {}->{} bytes):\n{}", + ActionId, + Attachments.size(), + AttachmentBytes, + UncompressedAttachmentBytes, + ObjStr); +# endif + + ++EmittedCount; + }, + 1); + + ZEN_CONSOLE("emitted: {} actions", EmittedCount.load()); + + eastl::map<uint64_t, std::vector<IoHash>> ReferenceHistogram; + + for (const auto& [K, V] : SeenAttachments) + { + if (V > 1) + { + ReferenceHistogram[V].push_back(K); + } + } + + for (const auto& [RefCount, Cids] : ReferenceHistogram) + { + ZEN_CONSOLE("{} attachments with {} references", Cids.size(), RefCount); + } + + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +int +ExecCommand::BuildActionsLog() +{ + ZEN_ASSERT(m_ChunkResolver); + ChunkResolver& Resolver = *m_ChunkResolver; + + if (m_RecordingPath.empty()) + { + throw OptionParseException("need to specify recording path", m_Options.help()); + } + + if (std::filesystem::exists(m_RecordingLogPath)) + { + throw OptionParseException(fmt::format("recording log directory '{}' already exists!", m_RecordingLogPath), m_Options.help()); + } + + ZEN_NOT_IMPLEMENTED("build log generation not implemented yet!"); + + std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); + + zen::compute::ComputeServiceSession ComputeSession(Resolver); + ComputeSession.StartRecording(Resolver, m_RecordingLogPath); + + return ExecUsingSession(ComputeSession); +} + +void +ExecCommand::EmitFunctionList(const std::vector<FunctionDefinition>& FunctionList) +{ + ZEN_CONSOLE("=== Known functions:\n==========================="); + + ZEN_CONSOLE("{:30} {:36} {:36} {}", "function", "version", "build system", "worker id"); + + for (const FunctionDefinition& Func : FunctionList) + { + ZEN_CONSOLE("{:30} {:36} {:36} {}", Func.FunctionName, Func.FunctionVersion, Func.BuildSystemVersion, Func.WorkerId); + } + + ZEN_CONSOLE("==========================="); +} + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zen/cmds/exec_cmd.h b/src/zen/cmds/exec_cmd.h new file mode 100644 index 000000000..6311354c0 --- /dev/null +++ b/src/zen/cmds/exec_cmd.h @@ -0,0 +1,101 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +#include <zencompute/recordingreader.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/guid.h> +#include <zencore/iohash.h> + +#include <filesystem> +#include <functional> +#include <unordered_map> + +namespace zen { +class CbPackage; +class CbObject; +struct IoHash; +class ChunkResolver; +} // namespace zen + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen::compute { +class ComputeServiceSession; +} + +namespace zen { + +/** + * Zen CLI command for executing functions from a recording + * + * Mostly for testing and debugging purposes + */ + +class ExecCommand : public ZenCmdBase +{ +public: + ExecCommand(); + ~ExecCommand(); + + static constexpr char Name[] = "exec"; + static constexpr char Description[] = "Execute functions from a recording"; + + virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::string m_OrchestratorUrl; + std::filesystem::path m_BeaconPath; + std::filesystem::path m_RecordingPath; + std::filesystem::path m_RecordingLogPath; + int m_Offset = 0; + int m_Stride = 1; + int m_Limit = 0; + bool m_Quiet = false; + std::string m_Mode{"http"}; + std::filesystem::path m_OutputPath; + bool m_Binary = false; + + struct FunctionDefinition + { + std::string FunctionName; + zen::Guid FunctionVersion; + zen::Guid BuildSystemVersion; + zen::IoHash WorkerId; + }; + + bool m_FunctionListEmittedOnce = false; + void EmitFunctionListOnce(const std::vector<FunctionDefinition>& FunctionList); + void EmitFunctionList(const std::vector<FunctionDefinition>& FunctionList); + + std::unordered_map<zen::IoHash, zen::CbPackage> m_WorkerMap; + std::vector<FunctionDefinition> m_FunctionList; + bool m_VerboseLogging = false; + bool m_QuietLogging = false; + bool m_DumpActions = false; + + zen::ChunkResolver* m_ChunkResolver = nullptr; + zen::compute::RecordingReaderBase* m_RecordingReader = nullptr; + + void RegisterWorkerFunctionsFromDescription(const zen::CbObject& WorkerDesc, const zen::IoHash& WorkerId); + + int ExecUsingSession(zen::compute::ComputeServiceSession& ComputeSession); + + // Execution modes + + int DumpWorkItems(); + int HttpExecute(); + int InProcessExecute(); + int LocalMessagingExecute(); + int BeaconExecute(); + int BuildActionsLog(); +}; + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zen/cmds/info_cmd.h b/src/zen/cmds/info_cmd.h index 231565bfd..dc108b8a2 100644 --- a/src/zen/cmds/info_cmd.h +++ b/src/zen/cmds/info_cmd.h @@ -9,6 +9,9 @@ namespace zen { class InfoCommand : public ZenCmdBase { public: + static constexpr char Name[] = "info"; + static constexpr char Description[] = "Show high level Zen server information"; + InfoCommand(); ~InfoCommand(); @@ -17,7 +20,7 @@ public: // virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"info", "Show high level zen store information"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp index 030cc8b66..c6b250fdf 100644 --- a/src/zen/cmds/print_cmd.cpp +++ b/src/zen/cmds/print_cmd.cpp @@ -84,7 +84,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - MakeSafeAbsolutePathÍnPlace(m_Filename); + MakeSafeAbsolutePathInPlace(m_Filename); Fc = ReadFile(m_Filename); } @@ -244,7 +244,7 @@ PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar if (m_Filename.empty()) throw OptionParseException("'--source' is required", m_Options.help()); - MakeSafeAbsolutePathÍnPlace(m_Filename); + MakeSafeAbsolutePathInPlace(m_Filename); FileContents Fc = ReadFile(m_Filename); IoBuffer Data = Fc.Flatten(); CbPackage Package; diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h index 6c1529b7c..f4a97e218 100644 --- a/src/zen/cmds/print_cmd.h +++ b/src/zen/cmds/print_cmd.h @@ -11,6 +11,9 @@ namespace zen { class PrintCommand : public ZenCmdBase { public: + static constexpr char Name[] = "print"; + static constexpr char Description[] = "Print compact binary object"; + PrintCommand(); ~PrintCommand(); @@ -19,7 +22,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"print", "Print compact binary object"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_Filename; bool m_ShowCbObjectTypeInfo = false; }; @@ -29,6 +32,9 @@ private: class PrintPackageCommand : public ZenCmdBase { public: + static constexpr char Name[] = "printpackage"; + static constexpr char Description[] = "Print compact binary package"; + PrintPackageCommand(); ~PrintPackageCommand(); @@ -37,7 +43,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"printpkg", "Print compact binary package"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_Filename; bool m_ShowCbObjectTypeInfo = false; }; diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 519b68126..db931e49a 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -41,12 +41,10 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { -namespace { +namespace projectstore_impl { using namespace std::literals; -#define ZEN_CLOUD_STORAGE "Cloud Storage" - void WriteAuthOptions(CbObjectWriter& Writer, std::string_view JupiterOpenIdProvider, std::string_view JupiterAccessToken, @@ -500,7 +498,7 @@ namespace { return {}; } -} // namespace +} // namespace projectstore_impl /////////////////////////////////////// @@ -522,6 +520,7 @@ DropProjectCommand::~DropProjectCommand() void DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -611,6 +610,7 @@ ProjectInfoCommand::~ProjectInfoCommand() void ProjectInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -697,6 +697,7 @@ CreateProjectCommand::~CreateProjectCommand() = default; void CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); using namespace std::literals; @@ -766,6 +767,7 @@ CreateOplogCommand::~CreateOplogCommand() = default; void CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); using namespace std::literals; @@ -990,6 +992,7 @@ ExportOplogCommand::~ExportOplogCommand() void ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; using namespace std::literals; ZEN_UNUSED(GlobalOptions); @@ -1470,6 +1473,20 @@ ImportOplogCommand::ImportOplogCommand() "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive", cxxopts::value(m_BoostWorkers), "<boostworkermemory>"); + m_Options.add_option( + "", + "", + "allow-partial-block-requests", + "Allow request for partial chunk blocks.\n" + " false = only full block requests allowed\n" + " mixed = multiple partial block ranges requests per block allowed to zen cache, single partial block range " + "request per block to host\n" + " zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests " + "allowed to host\n" + " true = multiple partial block ranges requests per block allowed to zen cache and host\n" + "Defaults to 'mixed'.", + cxxopts::value(m_AllowPartialBlockRequests), + "<allowpartialblockrequests>"); m_Options.parse_positional({"project", "oplog", "gcpath"}); m_Options.positional_help("[<projectid> <oplogid> [<gcpath>]]"); @@ -1482,6 +1499,7 @@ ImportOplogCommand::~ImportOplogCommand() void ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; using namespace std::literals; ZEN_UNUSED(GlobalOptions); @@ -1514,6 +1532,13 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg throw OptionParseException("'--oplog' is required", m_Options.help()); } + EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests); + if (Mode == EPartialBlockRequestMode::Invalid) + { + throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests), + m_Options.help()); + } + HttpClient Http(m_HostName); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -1651,6 +1676,9 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg { Writer.AddBool("boostworkermemory"sv, true); } + + Writer.AddString("partialblockrequestmode", m_AllowPartialBlockRequests); + if (!m_FileDirectoryPath.empty()) { Writer.BeginObject("file"sv); @@ -1766,6 +1794,7 @@ SnapshotOplogCommand::~SnapshotOplogCommand() void SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; using namespace std::literals; ZEN_UNUSED(GlobalOptions); @@ -1830,6 +1859,7 @@ ProjectStatsCommand::~ProjectStatsCommand() void ProjectStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -1882,6 +1912,7 @@ ProjectOpDetailsCommand::~ProjectOpDetailsCommand() void ProjectOpDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -1997,6 +2028,7 @@ OplogMirrorCommand::~OplogMirrorCommand() void OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -2264,6 +2296,7 @@ OplogValidateCommand::~OplogValidateCommand() void OplogValidateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -2415,6 +2448,7 @@ OplogDownloadCommand::~OplogDownloadCommand() void OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -2432,7 +2466,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a { m_SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + MakeSafeAbsolutePathInPlace(m_SystemRootDir); }; ParseSystemOptions(); @@ -2570,36 +2604,37 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a StorageInstance Storage; - ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2; + ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; ClientSettings.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u; - Storage.BuildStorageHttp = std::make_unique<HttpClient>(ResolveRes.HostUrl, ClientSettings); + Storage.BuildStorageHttp = std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings); + Storage.BuildStorageHost = ResolveRes.Cloud; BuildStorageCache::Statistics StorageCacheStats; std::atomic<bool> AbortFlag(false); - if (!ResolveRes.CacheUrl.empty()) + if (!ResolveRes.Cache.Address.empty()) { Storage.CacheHttp = std::make_unique<HttpClient>( - ResolveRes.CacheUrl, + ResolveRes.Cache.Address, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = ResolveRes.CacheAssumeHttp2, + .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, .MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u}, [&AbortFlag]() { return AbortFlag.load(); }); - Storage.CacheName = ResolveRes.CacheName; + Storage.CacheHost = ResolveRes.Cache; } if (!m_Quiet) { std::string StorageDescription = fmt::format("Cloud {}{}. SessionId {}. Namespace '{}', Bucket '{}'", - ResolveRes.HostName, - (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), + ResolveRes.Cloud.Name, + (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), Storage.BuildStorageHttp->GetSessionId(), m_Namespace, m_Bucket); @@ -2610,8 +2645,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a { std::string CacheDescription = fmt::format("Zen {}{}. SessionId {}. Namespace '{}', Bucket '{}'", - ResolveRes.CacheName, - (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl), + ResolveRes.Cache.Name, + (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address), Storage.CacheHttp->GetSessionId(), m_Namespace, m_Bucket); @@ -2627,11 +2662,10 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a Storage.BuildStorage = CreateJupiterBuildStorage(Log(), *Storage.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, m_AllowRedirect, StorageTempPath); - Storage.StorageName = ResolveRes.HostName; if (Storage.CacheHttp) { - Storage.BuildCacheStorage = CreateZenBuildStorageCache( + Storage.CacheStorage = CreateZenBuildStorageCache( *Storage.CacheHttp, StorageCacheStats, m_Namespace, diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index 56ef858f5..1ba98b39e 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -16,6 +16,9 @@ class ProjectStoreCommand : public ZenCmdBase class DropProjectCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-drop"; + static constexpr char Description[] = "Drop project or project oplog"; + DropProjectCommand(); ~DropProjectCommand(); @@ -23,7 +26,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-drop", "Drop project or project oplog"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -33,13 +36,16 @@ private: class ProjectInfoCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-info"; + static constexpr char Description[] = "Info on project or project oplog"; + ProjectInfoCommand(); ~ProjectInfoCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-info", "Info on project or project oplog"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -48,6 +54,9 @@ private: class CreateProjectCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-create"; + static constexpr char Description[] = "Create a project"; + CreateProjectCommand(); ~CreateProjectCommand(); @@ -55,7 +64,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-create", "Create project, the project must not already exist."}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectId; std::string m_RootDir; @@ -68,6 +77,9 @@ private: class CreateOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-create"; + static constexpr char Description[] = "Create a project oplog"; + CreateOplogCommand(); ~CreateOplogCommand(); @@ -75,7 +87,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-create", "Create oplog in an existing project, the oplog must not already exist."}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectId; std::string m_OplogId; @@ -86,6 +98,9 @@ private: class ExportOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-export"; + static constexpr char Description[] = "Export project store oplog"; + ExportOplogCommand(); ~ExportOplogCommand(); @@ -93,8 +108,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-export", - "Export project store oplog to cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -145,6 +159,9 @@ private: class ImportOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-import"; + static constexpr char Description[] = "Import project store oplog"; + ImportOplogCommand(); ~ImportOplogCommand(); @@ -152,8 +169,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-import", - "Import project store oplog from cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -193,19 +209,23 @@ private: bool m_BoostWorkerCount = false; bool m_BoostWorkerMemory = false; bool m_BoostWorkers = false; + + std::string m_AllowPartialBlockRequests = "true"; }; class SnapshotOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-snapshot"; + static constexpr char Description[] = "Snapshot project store oplog"; + SnapshotOplogCommand(); ~SnapshotOplogCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-snapshot", "Snapshot external file references in project store oplog into zen"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -214,26 +234,32 @@ private: class ProjectStatsCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-stats"; + static constexpr char Description[] = "Stats on project store"; + ProjectStatsCommand(); ~ProjectStatsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-stats", "Stats info on project store"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; class ProjectOpDetailsCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-op-details"; + static constexpr char Description[] = "Detail info on ops inside a project store oplog"; + ProjectOpDetailsCommand(); ~ProjectOpDetailsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-op-details", "Detail info on ops inside a project store oplog"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_Details = false; bool m_OpDetails = false; @@ -247,13 +273,16 @@ private: class OplogMirrorCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-mirror"; + static constexpr char Description[] = "Mirror project store oplog to file system"; + OplogMirrorCommand(); ~OplogMirrorCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-mirror", "Mirror oplog to file system"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -268,13 +297,16 @@ private: class OplogValidateCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-validate"; + static constexpr char Description[] = "Validate oplog for missing references"; + OplogValidateCommand(); ~OplogValidateCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-validate", "Validate oplog for missing references"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h index a6363b614..332a3126c 100644 --- a/src/zen/cmds/rpcreplay_cmd.h +++ b/src/zen/cmds/rpcreplay_cmd.h @@ -9,6 +9,9 @@ namespace zen { class RpcStartRecordingCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "rpc-record-start"; + static constexpr char Description[] = "Starts recording of cache rpc requests on a host"; + RpcStartRecordingCommand(); ~RpcStartRecordingCommand(); @@ -16,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"rpc-record-start", "Starts recording of cache rpc requests on a host"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_RecordingPath; }; @@ -24,6 +27,9 @@ private: class RpcStopRecordingCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "rpc-record-stop"; + static constexpr char Description[] = "Stops recording of cache rpc requests on a host"; + RpcStopRecordingCommand(); ~RpcStopRecordingCommand(); @@ -31,13 +37,16 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"rpc-record-stop", "Stops recording of cache rpc requests on a host"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; class RpcReplayCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "rpc-record-replay"; + static constexpr char Description[] = "Replays a previously recorded session of rpc requests"; + RpcReplayCommand(); ~RpcReplayCommand(); @@ -45,7 +54,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"rpc-record-replay", "Replays a previously recorded session of cache rpc requests to a target host"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_RecordingPath; bool m_OnHost = false; diff --git a/src/zen/cmds/run_cmd.h b/src/zen/cmds/run_cmd.h index 570a2e63a..300c08c5b 100644 --- a/src/zen/cmds/run_cmd.h +++ b/src/zen/cmds/run_cmd.h @@ -9,6 +9,9 @@ namespace zen { class RunCommand : public ZenCmdBase { public: + static constexpr char Name[] = "run"; + static constexpr char Description[] = "Run command with special options"; + RunCommand(); ~RunCommand(); @@ -17,7 +20,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"run", "Run executable"}; + cxxopts::Options m_Options{Name, Description}; int m_RunCount = 0; int m_RunTime = -1; std::string m_BaseDirectory; diff --git a/src/zen/cmds/serve_cmd.h b/src/zen/cmds/serve_cmd.h index ac74981f2..22f430948 100644 --- a/src/zen/cmds/serve_cmd.h +++ b/src/zen/cmds/serve_cmd.h @@ -11,6 +11,9 @@ namespace zen { class ServeCommand : public ZenCmdBase { public: + static constexpr char Name[] = "serve"; + static constexpr char Description[] = "Serve files from a directory"; + ServeCommand(); ~ServeCommand(); @@ -18,7 +21,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"serve", "Serve files from a tree"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h index dc103a196..df5df3066 100644 --- a/src/zen/cmds/status_cmd.h +++ b/src/zen/cmds/status_cmd.h @@ -11,6 +11,9 @@ namespace zen { class StatusCommand : public ZenCmdBase { public: + static constexpr char Name[] = "status"; + static constexpr char Description[] = "Show zen status"; + StatusCommand(); ~StatusCommand(); @@ -20,7 +23,7 @@ public: private: int GetLockFileEffectivePort() const; - cxxopts::Options m_Options{"status", "Show zen status"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; std::filesystem::path m_DataDir; }; diff --git a/src/zen/cmds/top_cmd.h b/src/zen/cmds/top_cmd.h index 74167ecfd..aeb196558 100644 --- a/src/zen/cmds/top_cmd.h +++ b/src/zen/cmds/top_cmd.h @@ -9,6 +9,9 @@ namespace zen { class TopCommand : public ZenCmdBase { public: + static constexpr char Name[] = "top"; + static constexpr char Description[] = "Monitor zen server activity"; + TopCommand(); ~TopCommand(); @@ -16,12 +19,15 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"top", "Show dev UI"}; + cxxopts::Options m_Options{Name, Description}; }; class PsCommand : public ZenCmdBase { public: + static constexpr char Name[] = "ps"; + static constexpr char Description[] = "Enumerate running zen server instances"; + PsCommand(); ~PsCommand(); @@ -29,7 +35,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"}; + cxxopts::Options m_Options{Name, Description}; }; } // namespace zen diff --git a/src/zen/cmds/trace_cmd.h b/src/zen/cmds/trace_cmd.h index a6c9742b7..6eb0ba22b 100644 --- a/src/zen/cmds/trace_cmd.h +++ b/src/zen/cmds/trace_cmd.h @@ -6,11 +6,12 @@ namespace zen { -/** Scrub storage - */ class TraceCommand : public ZenCmdBase { public: + static constexpr char Name[] = "trace"; + static constexpr char Description[] = "Control zen realtime tracing"; + TraceCommand(); ~TraceCommand(); @@ -18,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"trace", "Control zen realtime tracing"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_Stop = false; std::string m_TraceHost; diff --git a/src/zen/cmds/ui_cmd.cpp b/src/zen/cmds/ui_cmd.cpp new file mode 100644 index 000000000..da06ce305 --- /dev/null +++ b/src/zen/cmds/ui_cmd.cpp @@ -0,0 +1,236 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "ui_cmd.h" + +#include <zencore/except_fmt.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/process.h> +#include <zenutil/consoletui.h> +#include <zenutil/zenserverprocess.h> + +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +# include <shellapi.h> +#endif + +namespace zen { + +namespace { + + struct RunningServerInfo + { + uint16_t Port; + uint32_t Pid; + std::string SessionId; + std::string CmdLine; + }; + + static std::vector<RunningServerInfo> CollectRunningServers() + { + std::vector<RunningServerInfo> Servers; + ZenServerState State; + if (!State.InitializeReadOnly()) + return Servers; + + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + StringBuilder<25> SessionSB; + Entry.GetSessionId().ToString(SessionSB); + std::error_code CmdLineEc; + std::string CmdLine = GetProcessCommandLine(static_cast<int>(Entry.Pid.load()), CmdLineEc); + Servers.push_back({Entry.EffectiveListenPort.load(), Entry.Pid.load(), std::string(SessionSB.c_str()), std::move(CmdLine)}); + }); + + return Servers; + } + +} // namespace + +UiCommand::UiCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_options()("a,all", "Open dashboard for all running instances", cxxopts::value(m_All)->default_value("false")); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); + m_Options.add_option("", + "p", + "path", + "Dashboard path (default: /dashboard/)", + cxxopts::value(m_DashboardPath)->default_value("/dashboard/"), + "<path>"); + m_Options.parse_positional("path"); +} + +UiCommand::~UiCommand() +{ +} + +void +UiCommand::OpenBrowser(std::string_view HostName) +{ + // Allow shortcuts for specifying dashboard path, and ensure it is in a format we expect + // (leading slash, trailing slash if no file extension) + + if (!m_DashboardPath.empty()) + { + if (m_DashboardPath[0] != '/') + { + m_DashboardPath = "/dashboard/" + m_DashboardPath; + } + + if (m_DashboardPath.find_last_of('.') == std::string::npos && m_DashboardPath.back() != '/') + { + m_DashboardPath += '/'; + } + } + + bool Success = false; + + ExtendableStringBuilder<256> FullUrl; + FullUrl << HostName << m_DashboardPath; + +#if ZEN_PLATFORM_WINDOWS + HINSTANCE Result = ShellExecuteA(nullptr, "open", FullUrl.c_str(), nullptr, nullptr, SW_SHOWNORMAL); + Success = reinterpret_cast<intptr_t>(Result) > 32; +#else + // Validate URL doesn't contain shell metacharacters that could lead to command injection + std::string_view FullUrlView = FullUrl; + constexpr std::string_view DangerousChars = ";|&$`\\\"'<>(){}[]!#*?~\n\r"; + if (FullUrlView.find_first_of(DangerousChars) != std::string_view::npos) + { + throw OptionParseException(fmt::format("URL contains invalid characters: '{}'", FullUrl), m_Options.help()); + } + +# if ZEN_PLATFORM_MAC + std::string Command = fmt::format("open \"{}\"", FullUrl); +# elif ZEN_PLATFORM_LINUX + std::string Command = fmt::format("xdg-open \"{}\"", FullUrl); +# else + ZEN_NOT_IMPLEMENTED("Browser launching not implemented on this platform"); +# endif + + Success = system(Command.c_str()) == 0; +#endif + + if (!Success) + { + throw zen::runtime_error("Failed to launch browser for '{}'", FullUrl); + } + + ZEN_CONSOLE("Web browser launched for '{}' successfully", FullUrl); +} + +void +UiCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + using namespace std::literals; + + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return; + } + + // Resolve target server + uint16_t ServerPort = 0; + + if (m_HostName.empty()) + { + // Auto-discover running instances. + std::vector<RunningServerInfo> Servers = CollectRunningServers(); + + if (m_All) + { + if (Servers.empty()) + { + throw OptionParseException("No running Zen server instances found", m_Options.help()); + } + + for (const auto& Server : Servers) + { + OpenBrowser(fmt::format("http://localhost:{}", Server.Port)); + } + return; + } + + // If multiple are found and we have an interactive terminal, present a picker + // instead of silently using the first one. + if (Servers.size() > 1 && IsTuiAvailable()) + { + std::vector<std::string> Labels; + Labels.reserve(Servers.size() + 1); + Labels.push_back(fmt::format("(all {} instances)", Servers.size())); + + const int32_t Cols = static_cast<int32_t>(TuiConsoleColumns()); + constexpr int32_t kIndicator = 3; // " ▶ " or " " prefix + constexpr int32_t kSeparator = 2; // " " before cmdline + constexpr int32_t kEllipsis = 3; // "..." + + for (const auto& Server : Servers) + { + std::string Label = fmt::format("port {:<5} pid {:<7} session {}", Server.Port, Server.Pid, Server.SessionId); + + if (!Server.CmdLine.empty()) + { + int32_t Available = Cols - kIndicator - kSeparator - static_cast<int32_t>(Label.size()); + if (Available > kEllipsis) + { + Label += " "; + if (static_cast<int32_t>(Server.CmdLine.size()) <= Available) + { + Label += Server.CmdLine; + } + else + { + Label.append(Server.CmdLine, 0, static_cast<size_t>(Available - kEllipsis)); + Label += "..."; + } + } + } + + Labels.push_back(std::move(Label)); + } + + int SelectedIdx = TuiPickOne("Multiple Zen server instances found. Select one to open:", Labels); + if (SelectedIdx < 0) + return; // User cancelled + + if (SelectedIdx == 0) + { + // "All" selected + for (const auto& Server : Servers) + { + OpenBrowser(fmt::format("http://localhost:{}", Server.Port)); + } + return; + } + + ServerPort = Servers[SelectedIdx - 1].Port; + m_HostName = fmt::format("http://localhost:{}", ServerPort); + } + + if (m_HostName.empty()) + { + // Single or zero instances, or not an interactive terminal: + // fall back to default resolution (picks first instance or returns empty) + m_HostName = ResolveTargetHostSpec("", ServerPort); + } + } + else + { + if (m_All) + { + throw OptionParseException("--all cannot be used together with --hosturl", m_Options.help()); + } + m_HostName = ResolveTargetHostSpec(m_HostName, ServerPort); + } + + if (m_HostName.empty()) + { + throw OptionParseException("Unable to resolve server specification", m_Options.help()); + } + + OpenBrowser(m_HostName); +} + +} // namespace zen diff --git a/src/zen/cmds/ui_cmd.h b/src/zen/cmds/ui_cmd.h new file mode 100644 index 000000000..c74cdbbd0 --- /dev/null +++ b/src/zen/cmds/ui_cmd.h @@ -0,0 +1,32 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +#include <filesystem> + +namespace zen { + +class UiCommand : public ZenCmdBase +{ +public: + UiCommand(); + ~UiCommand(); + + static constexpr char Name[] = "ui"; + static constexpr char Description[] = "Launch web browser with zen server UI"; + + virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + void OpenBrowser(std::string_view HostName); + + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::string m_DashboardPath = "/dashboard/"; + bool m_All = false; +}; + +} // namespace zen diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h index 2e822d5fc..270db7f88 100644 --- a/src/zen/cmds/up_cmd.h +++ b/src/zen/cmds/up_cmd.h @@ -11,6 +11,9 @@ namespace zen { class UpCommand : public ZenCmdBase { public: + static constexpr char Name[] = "up"; + static constexpr char Description[] = "Bring zen server up"; + UpCommand(); ~UpCommand(); @@ -18,7 +21,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"up", "Bring up zen service"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; bool m_ShowConsole = false; bool m_ShowLog = false; @@ -28,6 +31,9 @@ private: class AttachCommand : public ZenCmdBase { public: + static constexpr char Name[] = "attach"; + static constexpr char Description[] = "Add a sponsor process to a running zen service"; + AttachCommand(); ~AttachCommand(); @@ -35,7 +41,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; int m_OwnerPid = 0; std::filesystem::path m_DataDir; @@ -44,6 +50,9 @@ private: class DownCommand : public ZenCmdBase { public: + static constexpr char Name[] = "down"; + static constexpr char Description[] = "Bring zen server down"; + DownCommand(); ~DownCommand(); @@ -51,7 +60,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"down", "Bring down zen service"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; bool m_ForceTerminate = false; std::filesystem::path m_ProgramBaseDir; diff --git a/src/zen/cmds/vfs_cmd.h b/src/zen/cmds/vfs_cmd.h index 5deaa02fa..9009c774b 100644 --- a/src/zen/cmds/vfs_cmd.h +++ b/src/zen/cmds/vfs_cmd.h @@ -9,6 +9,9 @@ namespace zen { class VfsCommand : public StorageCommand { public: + static constexpr char Name[] = "vfs"; + static constexpr char Description[] = "Manage virtual file system"; + VfsCommand(); ~VfsCommand(); @@ -16,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"vfs", "Manage virtual file system"}; + cxxopts::Options m_Options{Name, Description}; std::string m_Verb; std::string m_HostName; diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index adf0e61f0..10f5ad8e1 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -33,7 +33,7 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { -namespace { +namespace wipe_impl { static std::atomic<bool> AbortFlag = false; static std::atomic<bool> PauseFlag = false; static bool IsVerbose = false; @@ -49,10 +49,11 @@ namespace { : GetMediumWorkerPool(EWorkloadType::Burst); } -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \ +#undef ZEN_CONSOLE_VERBOSE +#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ + if (IsVerbose) \ + { \ + ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ } static void SignalCallbackHandler(int SigNum) @@ -505,7 +506,7 @@ namespace { } return CleanWipe; } -} // namespace +} // namespace wipe_impl WipeCommand::WipeCommand() { @@ -532,6 +533,7 @@ WipeCommand::~WipeCommand() = default; void WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace wipe_impl; ZEN_UNUSED(GlobalOptions); signal(SIGINT, SignalCallbackHandler); @@ -549,7 +551,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ProgressMode = (IsVerbose || m_PlainProgress) ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty; BoostWorkerThreads = m_BoostWorkerThreads; - MakeSafeAbsolutePathÍnPlace(m_Directory); + MakeSafeAbsolutePathInPlace(m_Directory); if (!IsDir(m_Directory)) { diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 6e6f5d863..af265d898 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -398,7 +398,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** } else { - MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + MakeSafeAbsolutePathInPlace(m_SystemRootDir); } std::filesystem::path StatePath = m_SystemRootDir / "workspaces"; @@ -815,7 +815,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (Results.size() != m_ChunkIds.size()) { throw std::runtime_error( - fmt::format("failed to get workspace share batch - invalid result count recevied (expected: {}, received: {}", + fmt::format("failed to get workspace share batch - invalid result count received (expected: {}, received: {}", m_ChunkIds.size(), Results.size())); } |